@trazum/cli 1.80.1 → 1.82.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env node
2
+ import { createHash } from 'node:crypto';
2
3
  import { readdirSync, readFileSync } from 'node:fs';
3
4
  import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
4
5
  import { cpus, tmpdir } from 'node:os';
@@ -111,6 +112,12 @@ import {
111
112
  parseUsageLine,
112
113
  plannedCalls,
113
114
  claudeCodeRecords,
115
+ ESTIMATE_ERROR_BAND_PCT,
116
+ bandFor,
117
+ foreignTokenizer,
118
+ measuredForeignError,
119
+ heliconeRecords,
120
+ litellmRecords,
114
121
  otelRecords,
115
122
  ownRate,
116
123
  positionAt,
@@ -144,9 +151,11 @@ import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cac
144
151
  import { dayOf, formatGap, median, spanDays } from './time.js';
145
152
  import type {
146
153
  BucketedReport,
154
+ EvalReport,
147
155
  FleetSource,
148
156
  HistoryRun,
149
157
  MeasuredUsage,
158
+ PruneReport,
150
159
  PlanDocument,
151
160
  StoredReport,
152
161
  VerifiedAction,
@@ -352,6 +361,7 @@ const VALUE_FLAGS = new Set([
352
361
  'gpu-usd-hour',
353
362
  'tokens-per-second',
354
363
  'utilization',
364
+ 'state',
355
365
  ]);
356
366
 
357
367
  function parseArgs(argv: string[], t: CliMessages): Args {
@@ -674,8 +684,10 @@ const COMMAND_FLAGS: Record<string, string[]> = {
674
684
  schema: [],
675
685
  rollup: ['json', 'html-out'],
676
686
  position: ['json', 'html-out', 'pricing', 'pricing-live'],
677
- 'from-claude-code': ['label', 'label-from-project', 'out', 'o'],
687
+ 'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
678
688
  'from-otel': ['label-from-service', 'out', 'o'],
689
+ 'from-litellm': ['out', 'o'],
690
+ 'from-helicone': ['out', 'o'],
679
691
  switch: ['to', 'migration-usd', 'cases'],
680
692
  ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
681
693
  pulse: ['json', 'max-stale-hours'],
@@ -787,11 +799,11 @@ function renderDiff(before: string, after: string, t: CliMessages): string {
787
799
  /**
788
800
  * The provider's name when the estimator was not calibrated for it.
789
801
  *
790
- * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and the
791
- * ±10% band descends from that. Printing the same band beside a GPT or Kimi
792
- * figure states a precision nobody has measured for that family — and since the
793
- * catalogue grew past Anthropic, that is most of it. Returns null when the model
794
- * is Anthropic's, where the band is at least the claim it was written for.
802
+ * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
803
+ * band descends from that. Printing one beside a GPT or Kimi figure states a
804
+ * precision nobody has measured for that family — and since the catalogue grew
805
+ * past Anthropic, that is most of it. Returns null when the model is
806
+ * Anthropic's, where the band is at least the claim it was written for.
795
807
  */
796
808
  function offFamilyName(modelId: string): string | null {
797
809
  const provider = getModel(modelId).provider;
@@ -799,6 +811,18 @@ function offFamilyName(modelId: string): string | null {
799
811
  return getModel(modelId).displayName;
800
812
  }
801
813
 
814
+ /**
815
+ * How far off the estimator has been measured on this model's family.
816
+ *
817
+ * Read from the catalogue's provider rather than from the display name, because
818
+ * the name is what a reader sees and the id is what was measured. Null on the
819
+ * calibrated family and on every family nobody has run — an unmeasured one gets
820
+ * a sentence saying so, never a borrowed figure.
821
+ */
822
+ function offFamilyError(modelId: string): number | null {
823
+ return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
824
+ }
825
+
802
826
  /**
803
827
  * Language codes as names, in the reader's language.
804
828
  *
@@ -881,7 +905,13 @@ function printReport(
881
905
  const n = (value: number): string => value.toLocaleString(t.numberLocale);
882
906
  const sourceNote =
883
907
  result.tokenSource === 'heuristic'
884
- ? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
908
+ ? c.dim(
909
+ t.report.estimated(
910
+ offFamilyName(result.usage.model),
911
+ bandFor(result.optimized),
912
+ offFamilyError(result.usage.model),
913
+ ),
914
+ )
885
915
  : c.dim(t.report.exactCount());
886
916
 
887
917
  /**
@@ -2600,6 +2630,122 @@ function projectLabelFor(file: string): string | undefined {
2600
2630
  return trimmed !== '' ? trimmed : folder;
2601
2631
  }
2602
2632
 
2633
+
2634
+ /**
2635
+ * `--state`: read the part of a transcript that is new since last time.
2636
+ *
2637
+ * A Claude Code transcript is append-only and can be enormous — the largest on
2638
+ * one real machine is 212 MB, and re-reading it to price the last thirty
2639
+ * seconds takes six and a half seconds. That is the whole cost of the status
2640
+ * line in `plugin/statusline`, and it is paid on every turn for a file whose
2641
+ * first two hundred megabytes cannot have changed.
2642
+ *
2643
+ * The state file ties three numbers together, and it is the third that makes
2644
+ * this exact rather than approximately right:
2645
+ *
2646
+ * - `offset`: where to start reading the transcript.
2647
+ * - `out`: how long the output file was when everything before `offset` had
2648
+ * been written. The tail after that is the last call as it looked last time,
2649
+ * and it is dropped and re-derived, because the converter's rule is that a
2650
+ * call arrives as several lines and the last one stands.
2651
+ * - `digest`: what the bytes just before `offset` were. A transcript that was
2652
+ * truncated, rotated or replaced would otherwise be resumed into the middle
2653
+ * of a different file, and the output would be a bill assembled from two
2654
+ * unrelated sessions. On a mismatch the whole thing is re-read, which is
2655
+ * slow and correct.
2656
+ */
2657
+ interface TranscriptState {
2658
+ schemaVersion: 1;
2659
+ files: Record<string, { offset: number; out: number; digest: string }>;
2660
+ }
2661
+
2662
+ /** How much of the run-up to `offset` is fingerprinted. */
2663
+ const STATE_DIGEST_BYTES = 4096;
2664
+
2665
+ const digestOf = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex');
2666
+
2667
+ const readState = async (path: string): Promise<TranscriptState> => {
2668
+ try {
2669
+ const parsed: unknown = JSON.parse(await readFile(path, 'utf8'));
2670
+ if (
2671
+ typeof parsed === 'object' &&
2672
+ parsed !== null &&
2673
+ (parsed as TranscriptState).schemaVersion === 1 &&
2674
+ typeof (parsed as TranscriptState).files === 'object'
2675
+ ) {
2676
+ return parsed as TranscriptState;
2677
+ }
2678
+ } catch {
2679
+ // A missing, unreadable or unrecognised state file is a cold start, not an
2680
+ // error: the answer it produces is the same one, computed the slow way.
2681
+ }
2682
+ return { schemaVersion: 1, files: {} };
2683
+ };
2684
+
2685
+ /**
2686
+ * The byte offset of a line index within a chunk, counted in bytes rather than
2687
+ * characters.
2688
+ *
2689
+ * `String.prototype.split` counts UTF-16 code units and a transcript is UTF-8
2690
+ * with prompts in it, so a character index used as a byte offset would land
2691
+ * mid-sequence on the first accented character and corrupt every resume after
2692
+ * it. The newlines are found in the Buffer.
2693
+ */
2694
+ const byteOffsetOfLine = (chunk: Buffer, line: number): number => {
2695
+ if (line <= 0) return 0;
2696
+ let seen = 0;
2697
+ for (let i = 0; i < chunk.length; i += 1) {
2698
+ if (chunk[i] !== 0x0a) continue;
2699
+ seen += 1;
2700
+ if (seen === line) return i + 1;
2701
+ }
2702
+ return chunk.length;
2703
+ };
2704
+
2705
+ /** What a resumable read decided, so the caller can report it honestly. */
2706
+ interface ResumedRead {
2707
+ text: string;
2708
+ /** Bytes skipped because a previous run had already settled them. */
2709
+ skipped: number;
2710
+ /** Where the output must be truncated to before appending. */
2711
+ truncateOutTo: number;
2712
+ /** Records the previous run had settled, and this one must not re-emit. */
2713
+ chunkStart: number;
2714
+ }
2715
+
2716
+ const resumableRead = async (
2717
+ file: string,
2718
+ state: TranscriptState,
2719
+ ): Promise<{ chunk: Buffer; read: ResumedRead }> => {
2720
+ const key = resolvePath(file);
2721
+ const entry = state.files[key];
2722
+ const handle = await open(key, 'r');
2723
+ try {
2724
+ const { size } = await handle.stat();
2725
+ let start = 0;
2726
+ let truncateOutTo = 0;
2727
+
2728
+ if (entry !== undefined && entry.offset > 0 && entry.offset <= size) {
2729
+ const runUp = Math.min(STATE_DIGEST_BYTES, entry.offset);
2730
+ const before = Buffer.alloc(runUp);
2731
+ await handle.read(before, 0, runUp, entry.offset - runUp);
2732
+ if (digestOf(before) === entry.digest) {
2733
+ start = entry.offset;
2734
+ truncateOutTo = entry.out;
2735
+ }
2736
+ }
2737
+
2738
+ const chunk = Buffer.alloc(size - start);
2739
+ if (chunk.length > 0) await handle.read(chunk, 0, chunk.length, start);
2740
+ return {
2741
+ chunk,
2742
+ read: { text: chunk.toString('utf8'), skipped: start, truncateOutTo, chunkStart: start },
2743
+ };
2744
+ } finally {
2745
+ await handle.close();
2746
+ }
2747
+ };
2748
+
2603
2749
  async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void> {
2604
2750
  const target = args.positional[0];
2605
2751
  if (target === undefined) throw new Error(t.fromClaudeCode.noPath());
@@ -2639,6 +2785,28 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
2639
2785
  * says so.
2640
2786
  */
2641
2787
  const labelFromProject = boolFlag(args, 'label-from-project', info.isDirectory());
2788
+ /**
2789
+ * `--state` is for one transcript, deliberately.
2790
+ *
2791
+ * The state ties a transcript offset to a length of the output file, and
2792
+ * with several transcripts appending to one output there is no single length
2793
+ * that means "everything settled": the second file's records sit after the
2794
+ * first file's unsettled tail. A directory is re-read in full, which is what
2795
+ * it was doing before and is the right cost for a walk that is not a status
2796
+ * line refreshing every turn.
2797
+ */
2798
+ const statePath = stringFlag(args, 'state');
2799
+ if (statePath !== undefined && info.isDirectory()) throw new Error(t.fromClaudeCode.stateNeedsFile());
2800
+ if (statePath !== undefined && stringFlag(args, 'out') === undefined && stringFlag(args, 'o') === undefined) {
2801
+ throw new Error(t.fromClaudeCode.stateNeedsOut());
2802
+ }
2803
+ const state = statePath !== undefined ? await readState(statePath) : undefined;
2804
+ let resumed: ResumedRead | undefined;
2805
+ let chunk: Buffer | undefined;
2806
+ let resumeLine = 0;
2807
+ let settledRecords = 0;
2808
+ let settledOutBytes = 0;
2809
+
2642
2810
  const lines: string[] = [];
2643
2811
  let records = 0;
2644
2812
  let collapsed = 0;
@@ -2650,7 +2818,15 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
2650
2818
  let disagreements = 0;
2651
2819
  let synthetic = 0;
2652
2820
  for (const file of files) {
2653
- const text = await readFile(file, 'utf8');
2821
+ let text: string;
2822
+ if (state !== undefined) {
2823
+ const read = await resumableRead(file, state);
2824
+ chunk = read.chunk;
2825
+ resumed = read.read;
2826
+ text = read.read.text;
2827
+ } else {
2828
+ text = await readFile(file, 'utf8');
2829
+ }
2654
2830
  const projectLabel = labelFromProject ? projectLabelFor(file) : undefined;
2655
2831
  const conversion = claudeCodeRecords(text, {
2656
2832
  ...(label !== undefined
@@ -2660,6 +2836,8 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
2660
2836
  : {}),
2661
2837
  });
2662
2838
  for (const record of conversion.records) lines.push(JSON.stringify(record));
2839
+ resumeLine = conversion.resume.line;
2840
+ settledRecords = conversion.resume.records;
2663
2841
  records += conversion.records.length;
2664
2842
  collapsed += conversion.collapsed;
2665
2843
  noRequestId += conversion.noRequestId;
@@ -2673,12 +2851,57 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
2673
2851
 
2674
2852
  const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
2675
2853
  if (out !== undefined) {
2676
- await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
2854
+ if (statePath !== undefined) {
2855
+ /**
2856
+ * Append rather than rewrite, and drop the tail first.
2857
+ *
2858
+ * The tail is the last call as the previous run recorded it, and this
2859
+ * run has just re-derived it from its own first line. Truncating is what
2860
+ * makes the two runs add up to exactly one reading of the transcript
2861
+ * rather than one and a bit.
2862
+ */
2863
+ const handle = await open(out, resumed!.truncateOutTo > 0 ? 'r+' : 'w');
2864
+ try {
2865
+ await handle.truncate(resumed!.truncateOutTo);
2866
+ const body = lines.join('\n') + (lines.length > 0 ? '\n' : '');
2867
+ if (body !== '') await handle.write(body, resumed!.truncateOutTo, 'utf8');
2868
+ settledOutBytes =
2869
+ resumed!.truncateOutTo +
2870
+ lines
2871
+ .slice(0, settledRecords)
2872
+ .reduce((sum, line) => sum + Buffer.byteLength(line + '\n', 'utf8'), 0);
2873
+ } finally {
2874
+ await handle.close();
2875
+ }
2876
+ } else {
2877
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
2878
+ }
2677
2879
  console.error(t.fromClaudeCode.written(out));
2678
2880
  } else {
2679
2881
  for (const line of lines) console.log(line);
2680
2882
  }
2681
2883
 
2884
+ if (statePath !== undefined) {
2885
+ const key = resolvePath(files[0]!);
2886
+ const nextOffset = resumed!.chunkStart + byteOffsetOfLine(chunk!, resumeLine);
2887
+ const runUp = Math.min(STATE_DIGEST_BYTES, nextOffset);
2888
+ const before = Buffer.alloc(runUp);
2889
+ if (runUp > 0) {
2890
+ const handle = await open(key, 'r');
2891
+ try {
2892
+ await handle.read(before, 0, runUp, nextOffset - runUp);
2893
+ } finally {
2894
+ await handle.close();
2895
+ }
2896
+ }
2897
+ const next: TranscriptState = {
2898
+ schemaVersion: 1,
2899
+ files: { [key]: { offset: nextOffset, out: settledOutBytes, digest: digestOf(before) } },
2900
+ };
2901
+ await writeFile(statePath, JSON.stringify(next, null, 2) + '\n', 'utf8');
2902
+ console.error(t.fromClaudeCode.resumed(resumed!.skipped, nextOffset));
2903
+ }
2904
+
2682
2905
  console.error(t.fromClaudeCode.summary(files.length, records));
2683
2906
  if (collapsed > 0) console.error(t.fromClaudeCode.collapsed(collapsed));
2684
2907
  if (streamed > 0) console.error(t.fromClaudeCode.streamed(streamed));
@@ -2887,6 +3110,169 @@ async function commandFromOtel(args: Args, t: CliMessages): Promise<void> {
2887
3110
  if (unparseable > 0) console.error(t.fromOtel.unparseable(unparseable));
2888
3111
  }
2889
3112
 
3113
+ /**
3114
+ * `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
3115
+ *
3116
+ * The same pure-converter pattern again, pointed at the gateway a great many
3117
+ * teams already put in front of every provider: `LiteLLM_SpendLogs` is the
3118
+ * export most likely to already exist on somebody's disk. `litellmRecords` in
3119
+ * core does the reading; this is the walk and the honesty.
3120
+ *
3121
+ * Three of the four counts it prints exist because the alternative is a
3122
+ * flattering silence. Rows naming no model are not priced by the route they
3123
+ * took, rows with no tokens are logged calls nobody can price, and a
3124
+ * `cache_hit` flag is not a token split — so the cache verdicts read "cannot
3125
+ * tell" rather than a fabricated one, exactly as `from-otel` does.
3126
+ *
3127
+ * The fourth is the one that matters most: LiteLLM prices the same calls with
3128
+ * its own table, and that figure is printed beside Trazum's and never merged
3129
+ * into it. Two price tables summed into one total is how a report becomes
3130
+ * quietly wrong.
3131
+ *
3132
+ * The row carries `messages`, `response`, `api_key`, `requester_ip_address`
3133
+ * and `end_user`. None of it is read; a fixture plants a marker in each and
3134
+ * greps the whole output.
3135
+ */
3136
+ async function commandFromLiteLlm(args: Args, t: CliMessages): Promise<void> {
3137
+ const target = args.positional[0];
3138
+ if (target === undefined) throw new Error(t.fromLiteLlm.noPath());
3139
+
3140
+ let info;
3141
+ try {
3142
+ info = await stat(target);
3143
+ } catch {
3144
+ throw new Error(t.fromLiteLlm.notFound(target));
3145
+ }
3146
+ const files: string[] = [];
3147
+ if (info.isDirectory()) {
3148
+ const entries = await readdir(target, { recursive: true, withFileTypes: true });
3149
+ for (const entry of entries) {
3150
+ if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
3151
+ files.push(join(entry.parentPath, entry.name));
3152
+ }
3153
+ }
3154
+ files.sort();
3155
+ if (files.length === 0) throw new Error(t.fromLiteLlm.noExports(target));
3156
+ } else {
3157
+ files.push(target);
3158
+ }
3159
+
3160
+ const lines: string[] = [];
3161
+ let rows = 0;
3162
+ let unnamedModel = 0;
3163
+ let noTokens = 0;
3164
+ let cacheFlagged = 0;
3165
+ let unparseable = 0;
3166
+ let reportedSpend = 0;
3167
+ let sawSpend = false;
3168
+ for (const file of files) {
3169
+ const conversion = litellmRecords(await readFile(file, 'utf8'));
3170
+ for (const record of conversion.records) lines.push(JSON.stringify(record));
3171
+ rows += conversion.rows;
3172
+ unnamedModel += conversion.unnamedModel;
3173
+ noTokens += conversion.noTokens;
3174
+ cacheFlagged += conversion.cacheFlagged;
3175
+ unparseable += conversion.unparseable;
3176
+ if (conversion.reportedSpendUsd !== null) {
3177
+ reportedSpend += conversion.reportedSpendUsd;
3178
+ sawSpend = true;
3179
+ }
3180
+ }
3181
+
3182
+ const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
3183
+ if (out !== undefined) {
3184
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
3185
+ console.error(t.fromLiteLlm.written(out));
3186
+ } else {
3187
+ for (const line of lines) console.log(line);
3188
+ }
3189
+
3190
+ console.error(t.fromLiteLlm.summary(files.length, rows));
3191
+ if (unnamedModel > 0) console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
3192
+ if (noTokens > 0) console.error(t.fromLiteLlm.noTokens(noTokens));
3193
+ if (cacheFlagged > 0) console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
3194
+ if (sawSpend) console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
3195
+ if (unparseable > 0) console.error(t.fromLiteLlm.unparseable(unparseable));
3196
+ }
3197
+
3198
+ /**
3199
+ * `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
3200
+ *
3201
+ * The third converter, and the one that needs three columns where the others
3202
+ * need one: Helicone carries `request_model`, `model_override` and
3203
+ * `response_model`, and they can disagree. The response wins, because a bill
3204
+ * is about what was billed rather than what was intended, and the
3205
+ * disagreements are counted so a substitution is something the reader sees.
3206
+ *
3207
+ * Two absences are stated rather than filled in. There is no cache token
3208
+ * split on the row, only a flag; and a Helicone request id names one call and
3209
+ * never a conversation, so the records carry no session and the
3210
+ * conversation-shaped findings stay unavailable. Both are printed, because a
3211
+ * gap that is not said reads as a gap that is not there.
3212
+ */
3213
+ async function commandFromHelicone(args: Args, t: CliMessages): Promise<void> {
3214
+ const target = args.positional[0];
3215
+ if (target === undefined) throw new Error(t.fromHelicone.noPath());
3216
+
3217
+ let info;
3218
+ try {
3219
+ info = await stat(target);
3220
+ } catch {
3221
+ throw new Error(t.fromHelicone.notFound(target));
3222
+ }
3223
+ const files: string[] = [];
3224
+ if (info.isDirectory()) {
3225
+ const entries = await readdir(target, { recursive: true, withFileTypes: true });
3226
+ for (const entry of entries) {
3227
+ if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
3228
+ files.push(join(entry.parentPath, entry.name));
3229
+ }
3230
+ }
3231
+ files.sort();
3232
+ if (files.length === 0) throw new Error(t.fromHelicone.noExports(target));
3233
+ } else {
3234
+ files.push(target);
3235
+ }
3236
+
3237
+ const lines: string[] = [];
3238
+ let rows = 0;
3239
+ let unnamedModel = 0;
3240
+ let disagreements = 0;
3241
+ let noTokens = 0;
3242
+ let cacheFlagged = 0;
3243
+ let unparseable = 0;
3244
+ for (const file of files) {
3245
+ const conversion = heliconeRecords(await readFile(file, 'utf8'));
3246
+ for (const record of conversion.records) lines.push(JSON.stringify(record));
3247
+ rows += conversion.rows;
3248
+ unnamedModel += conversion.unnamedModel;
3249
+ disagreements += conversion.modelDisagreements;
3250
+ noTokens += conversion.noTokens;
3251
+ cacheFlagged += conversion.cacheFlagged;
3252
+ unparseable += conversion.unparseable;
3253
+ }
3254
+
3255
+ const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
3256
+ if (out !== undefined) {
3257
+ await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
3258
+ console.error(t.fromHelicone.written(out));
3259
+ } else {
3260
+ for (const line of lines) console.log(line);
3261
+ }
3262
+
3263
+ console.error(t.fromHelicone.summary(files.length, rows));
3264
+ if (unnamedModel > 0) console.error(t.fromHelicone.unnamedModel(unnamedModel));
3265
+ if (disagreements > 0) console.error(t.fromHelicone.disagreements(disagreements));
3266
+ if (noTokens > 0) console.error(t.fromHelicone.noTokens(noTokens));
3267
+ if (cacheFlagged > 0) console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
3268
+ // Said on every run that produced records, not only when something is
3269
+ // missing: the absence is a property of the format, and a reader who has
3270
+ // just converted a month needs to know which questions this log cannot
3271
+ // answer before they go looking for the answers.
3272
+ if (rows > 0) console.error(t.fromHelicone.noSessions());
3273
+ if (unparseable > 0) console.error(t.fromHelicone.unparseable(unparseable));
3274
+ }
3275
+
2890
3276
  /**
2891
3277
  * `trazum position <usage.jsonl>` — where the month stands, measured.
2892
3278
  *
@@ -5487,6 +5873,7 @@ async function commandCheck(
5487
5873
  verdicts: [verdict],
5488
5874
  level,
5489
5875
  tokenSource: counter.source,
5876
+ band: bandFor(prompt),
5490
5877
  truncated: false,
5491
5878
  t,
5492
5879
  }),
@@ -5622,12 +6009,18 @@ async function checkEmbedded(
5622
6009
  const failures = verdicts.filter(isOverBudget);
5623
6010
  const ok = failures.length === 0 && extraction.declined.length === 0;
5624
6011
 
6012
+ // Widest across the prompts this report covers, never an average: a run over
6013
+ // prose and a CSV covers text the estimator is 6% and 33% out on, and a
6014
+ // figure between them would describe neither.
6015
+ const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
6016
+
5625
6017
  await writeMarkdown(args, () =>
5626
6018
  renderCheckMarkdown({
5627
6019
  target: path,
5628
6020
  verdicts,
5629
6021
  level,
5630
6022
  tokenSource: counter.source,
6023
+ band,
5631
6024
  truncated: false,
5632
6025
  t,
5633
6026
  }),
@@ -9448,12 +9841,102 @@ function reportProfileGaps(
9448
9841
  * exactly as `prune` does. A command that can spend somebody's money without
9449
9842
  * telling them first is a command they stop trusting.
9450
9843
  */
9844
+ /**
9845
+ * The routing measurement, without the text that produced it.
9846
+ *
9847
+ * `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
9848
+ * whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
9849
+ * `baseline[1]` and `optimized` (three model answers). It carried no
9850
+ * `schemaVersion` either.
9851
+ *
9852
+ * `docs/json-output.md` opens by promising that **nothing here carries a
9853
+ * session key or prompt text**, and this is the document most likely to be
9854
+ * piped into a dashboard, pasted into a ticket or dropped into a browser. A
9855
+ * tool whose headline is that prompts never leave the machine cannot have its
9856
+ * most portable artefact be the one that carries them.
9857
+ *
9858
+ * The per-case *shape* survives: the two similarity scores are the evidence
9859
+ * behind the verdict, and a reader checking a borderline call needs them. Only
9860
+ * the strings go. Anyone who wants the answers has the human-readable output,
9861
+ * which never left the terminal.
9862
+ *
9863
+ * A named function rather than an inline object so the stripping is a thing a
9864
+ * test can call without spending a provider call to reach it.
9865
+ */
9866
+ export function routeDocument(slice: unknown, result: EvalReport): unknown {
9867
+ return {
9868
+ schemaVersion: 1,
9869
+ slice,
9870
+ evaluation: {
9871
+ provider: result.provider,
9872
+ model: result.model,
9873
+ candidateModel: result.candidateModel,
9874
+ verdict: result.verdict,
9875
+ selfAgreement: result.selfAgreement,
9876
+ crossAgreement: result.crossAgreement,
9877
+ callsMade: result.callsMade,
9878
+ cases: result.cases.map((one: EvalReport['cases'][number]) => ({
9879
+ selfSimilarity: one.selfSimilarity,
9880
+ crossSimilarity: one.crossSimilarity,
9881
+ })),
9882
+ },
9883
+ };
9884
+ }
9885
+
9886
+ /**
9887
+ * The example-pruning measurement, likewise.
9888
+ *
9889
+ * `ExampleContribution.text` is *"the example itself, so a report can quote its
9890
+ * first line"* — true of the terminal rendering and wrong of a document that
9891
+ * promises to carry no prompt text. A few-shot example **is** prompt text, and
9892
+ * this was the second command emitting it.
9893
+ *
9894
+ * `index` and `tokens` say which example without quoting it, which is what a
9895
+ * machine reading this needs: it already has the prompt.
9896
+ */
9897
+ export function pruneDocument(report: PruneReport): unknown {
9898
+ return {
9899
+ schemaVersion: 1,
9900
+ provider: report.provider,
9901
+ model: report.model,
9902
+ selfAgreement: report.selfAgreement,
9903
+ recoverableTokens: report.recoverableTokens,
9904
+ callsMade: report.callsMade,
9905
+ contributions: report.contributions.map((one: PruneReport['contributions'][number]) => ({
9906
+ index: one.index,
9907
+ tokens: one.tokens,
9908
+ agreementWithout: one.agreementWithout,
9909
+ verdict: one.verdict,
9910
+ })),
9911
+ };
9912
+ }
9913
+
9451
9914
  async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
9915
+ /**
9916
+ * Under `--json`, stdout carries the document and nothing else.
9917
+ *
9918
+ * It carried the human preamble too — the slice it picked, the call count,
9919
+ * the "nothing has been spent yet" line — straight into the same stream as
9920
+ * the JSON, so `route --json > verdict.json` wrote a file no parser would
9921
+ * read. Found by running the command and feeding the file to the bridge
9922
+ * that reads it, which is the one thing the pure-function guards could not
9923
+ * do: `interchange.test.js` names `route` among the commands it cannot
9924
+ * drive, so nothing checked that its `--json` output was a document.
9925
+ *
9926
+ * The lines go to stderr rather than away. A reader running this in a
9927
+ * terminal still needs to see which slice was picked and what it will cost
9928
+ * before it costs it; a pipe needs the document. stderr gives both.
9929
+ */
9930
+ const asJson = boolFlag(args, 'json');
9931
+ const say = (line = ''): void => {
9932
+ if (asJson) console.error(line);
9933
+ else console.log(line);
9934
+ };
9452
9935
  const path = args.positional[0];
9453
9936
  if (path === undefined) {
9454
- console.log();
9455
- console.log(c.dim(wrap(t.route.noTarget(), 74, ' ')));
9456
- console.log();
9937
+ say();
9938
+ say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
9939
+ say();
9457
9940
  return;
9458
9941
  }
9459
9942
 
@@ -9476,18 +9959,18 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9476
9959
  const available = report.byLabel
9477
9960
  .map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
9478
9961
  .join(', ');
9479
- console.log();
9480
- console.log(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
9481
- console.log();
9962
+ say();
9963
+ say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
9964
+ say();
9482
9965
  return;
9483
9966
  }
9484
9967
  const slice = levers.slices.find(
9485
9968
  (s) => s.route !== null && (wanted === undefined || s.label === wanted),
9486
9969
  );
9487
9970
  if (!slice?.route) {
9488
- console.log();
9489
- console.log(c.dim(wrap(t.route.noRoute(), 74, ' ')));
9490
- console.log();
9971
+ say();
9972
+ say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
9973
+ say();
9491
9974
  return;
9492
9975
  }
9493
9976
 
@@ -9511,8 +9994,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9511
9994
 
9512
9995
  const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
9513
9996
  const worth = formatUsd(slice.route.savingUsd);
9514
- console.log();
9515
- console.log(
9997
+ say();
9998
+ say(
9516
9999
  ` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`,
9517
10000
  );
9518
10001
  /**
@@ -9525,20 +10008,20 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9525
10008
  * counts, so it is stated rather than guessed at.
9526
10009
  */
9527
10010
  if (slice.label === UNLABELLED) {
9528
- console.log(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
10011
+ say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
9529
10012
  }
9530
- console.log();
9531
- console.log(
10013
+ say();
10014
+ say(
9532
10015
  ` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`,
9533
10016
  );
9534
10017
 
9535
10018
  if (!boolFlag(args, 'yes')) {
9536
- console.log(` ${c.dim(t.route.dryRun())}`);
9537
- console.log();
10019
+ say(` ${c.dim(t.route.dryRun())}`);
10020
+ say();
9538
10021
  return;
9539
10022
  }
9540
10023
 
9541
- console.log(` ${c.dim(t.route.running(inputs.length))}`);
10024
+ say(` ${c.dim(t.route.running(inputs.length))}`);
9542
10025
  // Same prompt on both sides. The axis under test is the model, and passing the
9543
10026
  // prompt twice is what says so at the call site.
9544
10027
  const result = await evaluate(prompt, prompt, inputs, provider, {
@@ -9547,22 +10030,22 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9547
10030
  });
9548
10031
 
9549
10032
  if (boolFlag(args, 'json')) {
9550
- console.log(JSON.stringify({ slice, evaluation: result }, null, 2));
10033
+ console.log(JSON.stringify(routeDocument(slice, result), null, 2));
9551
10034
  return;
9552
10035
  }
9553
10036
 
9554
10037
  const asPct = (v: number): string => `${(v * 100).toFixed(0)}%`;
9555
- console.log();
9556
- console.log(
10038
+ say();
10039
+ say(
9557
10040
  ` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`,
9558
10041
  );
9559
- console.log();
10042
+ say();
9560
10043
  if (result.verdict === 'inconclusive') {
9561
- console.log(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
10044
+ say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
9562
10045
  } else if (result.verdict === 'diverges') {
9563
- console.log(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
10046
+ say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
9564
10047
  } else {
9565
- console.log(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
10048
+ say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
9566
10049
  }
9567
10050
  /**
9568
10051
  * Printed on every verdict including the good one. Agreement is not
@@ -9570,8 +10053,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
9570
10053
  * ever right, and a green tick that let somebody forget that would be the tool
9571
10054
  * overstating what it knows.
9572
10055
  */
9573
- console.log(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
9574
- console.log();
10056
+ say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
10057
+ say();
9575
10058
  }
9576
10059
 
9577
10060
  /**
@@ -9738,6 +10221,14 @@ interface PromptScan {
9738
10221
  truncated: boolean;
9739
10222
  /** The extensions actually walked, so an error can name them. */
9740
10223
  extensions: string[];
10224
+ /**
10225
+ * The widest measured band across the prompts this scan judged.
10226
+ *
10227
+ * Computed here because this is where the texts are, and widest rather than
10228
+ * averaged: a scan over a prose prompt and a CSV covers text the estimator
10229
+ * is 6% and 33% out on, and a figure between them describes neither.
10230
+ */
10231
+ band: number;
9741
10232
  }
9742
10233
 
9743
10234
  /**
@@ -9817,6 +10308,9 @@ async function scanPrompts(
9817
10308
  const verdicts: FileVerdict[] = [];
9818
10309
  const declined: Array<{ path: string; line: number; detail: string }> = [];
9819
10310
 
10311
+ // Every prompt this run judged, so the report's band is the widest of them
10312
+ // rather than one figure asserted over text of several kinds.
10313
+ const judgedTexts: string[] = [];
9820
10314
  for (const relativePath of files) {
9821
10315
  const text = capInput(
9822
10316
  await readFile(join(root, relativePath), 'utf8'),
@@ -9824,6 +10318,7 @@ async function scanPrompts(
9824
10318
  maxInputFlag(args, t),
9825
10319
  t,
9826
10320
  );
10321
+ judgedTexts.push(text);
9827
10322
  // Budgets are keyed on paths as written in the repository, so a pattern like
9828
10323
  // `prompts/**` has to be matched against the path including the root the
9829
10324
  // user passed — not against the name relative to it.
@@ -9857,7 +10352,7 @@ async function scanPrompts(
9857
10352
  throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
9858
10353
  }
9859
10354
 
9860
- return { verdicts, declined, truncated, extensions };
10355
+ return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
9861
10356
  }
9862
10357
 
9863
10358
  async function checkDirectory(
@@ -9889,7 +10384,7 @@ async function checkDirectory(
9889
10384
  only = checkable;
9890
10385
  }
9891
10386
 
9892
- const { verdicts, declined, truncated, extensions } = await scanPrompts(
10387
+ const { verdicts, declined, truncated, extensions, band } = await scanPrompts(
9893
10388
  root,
9894
10389
  args,
9895
10390
  flagBudget,
@@ -9967,6 +10462,7 @@ async function checkDirectory(
9967
10462
  renderCheckMarkdown({
9968
10463
  target: root,
9969
10464
  verdicts,
10465
+ band,
9970
10466
  level,
9971
10467
  tokenSource: counter.source,
9972
10468
  truncated,
@@ -10147,6 +10643,13 @@ function joinPosix(root: string, relativePath: string): string {
10147
10643
  * never "delete this", and nothing here edits the prompt.
10148
10644
  */
10149
10645
  async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10646
+ // Same as `route`: under `--json` stdout is the document, and the estimate
10647
+ // a reader has to see before spending goes to stderr rather than away.
10648
+ const asJson = boolFlag(args, 'json');
10649
+ const say = (line = ''): void => {
10650
+ if (asJson) console.error(line);
10651
+ else console.log(line);
10652
+ };
10150
10653
  const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
10151
10654
 
10152
10655
  const casesPath = stringFlag(args, 'cases');
@@ -10161,11 +10664,11 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10161
10664
 
10162
10665
  // Printed before the key is even looked up, so somebody weighing it up does not
10163
10666
  // need a configured provider to see the number.
10164
- console.log();
10165
- console.log(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
10667
+ say();
10668
+ say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
10166
10669
 
10167
10670
  if (!boolFlag(args, 'yes')) {
10168
- console.log(c.yellow(` ${t.prune.needsConsent()}`));
10671
+ say(c.yellow(` ${t.prune.needsConsent()}`));
10169
10672
  return;
10170
10673
  }
10171
10674
 
@@ -10177,15 +10680,15 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10177
10680
  });
10178
10681
 
10179
10682
  if (boolFlag(args, 'json')) {
10180
- console.log(JSON.stringify(report, null, 2));
10683
+ console.log(JSON.stringify(pruneDocument(report), null, 2));
10181
10684
  return;
10182
10685
  }
10183
10686
 
10184
10687
  const pct = (value: number): string => `${(value * 100).toFixed(0)}%`;
10185
- console.log();
10186
- console.log(sectionHeading(t.prune.heading(provider.model)));
10187
- console.log(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
10188
- console.log();
10688
+ say();
10689
+ say(sectionHeading(t.prune.heading(provider.model)));
10690
+ say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
10691
+ say();
10189
10692
 
10190
10693
  for (const contribution of report.contributions) {
10191
10694
  /**
@@ -10204,7 +10707,7 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10204
10707
  ? t.prune.verdictNeeded()
10205
10708
  : t.prune.verdictRecoverable();
10206
10709
 
10207
- console.log(
10710
+ say(
10208
10711
  ` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
10209
10712
  + ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`,
10210
10713
  );
@@ -10217,16 +10720,16 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
10217
10720
  */
10218
10721
  const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
10219
10722
  const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
10220
- console.log(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
10723
+ say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
10221
10724
  }
10222
10725
 
10223
- console.log();
10726
+ say();
10224
10727
  if (report.recoverableTokens > 0) {
10225
- console.log(` ${t.prune.recoverable(report.recoverableTokens)}`);
10728
+ say(` ${t.prune.recoverable(report.recoverableTokens)}`);
10226
10729
  }
10227
- console.log(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
10228
- console.log();
10229
- console.log(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
10730
+ say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
10731
+ say();
10732
+ say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
10230
10733
  }
10231
10734
 
10232
10735
  async function commandEval(
@@ -10545,6 +11048,18 @@ const DOCTOR_LIST_LIMIT = 8;
10545
11048
  const BLAME_DEFAULT_LIMIT = 20;
10546
11049
  const BLAME_MAX_LIMIT = 500;
10547
11050
 
11051
+ /**
11052
+ * The band for a report covering several prompts.
11053
+ *
11054
+ * The widest of them, and the empty case is the widest published band rather
11055
+ * than a narrow default: a report over nothing has measured nothing, and
11056
+ * guessing narrow there is the one direction that produces a false claim.
11057
+ */
11058
+ function widestBand(texts: readonly string[]): number {
11059
+ if (texts.length === 0) return ESTIMATE_ERROR_BAND_PCT;
11060
+ return Math.max(...texts.map((text) => bandFor(text)));
11061
+ }
11062
+
10548
11063
  interface BlameRow {
10549
11064
  revision: Revision;
10550
11065
  /** `null` when the file did not exist at that commit, or held no marked prompt. */
@@ -10643,17 +11158,35 @@ async function commandBlame(
10643
11158
  const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
10644
11159
  const truncatedHistory = revisions.length > limit;
10645
11160
 
11161
+ /*
11162
+ The band of this file's own text, read once and handed to both renderers.
11163
+
11164
+ `blame` walks one file across revisions, so the kind of text is a property
11165
+ of the file, and today's content is the closest thing to it that exists
11166
+ here. A file that is gone from the working tree — renamed away, or deleted
11167
+ in the commit being blamed — falls back to the widest published band, which
11168
+ is exactly what that constant is for: a caller that cannot see the text can
11169
+ only guess in the safe direction.
11170
+ */
11171
+ let band = ESTIMATE_ERROR_BAND_PCT;
11172
+ try {
11173
+ band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
11174
+ } catch {
11175
+ // Not on disk now. The wide band stands and says so by being wide.
11176
+ }
11177
+
10646
11178
  await writeMarkdown(args, () =>
10647
11179
  renderBlameMarkdown({
10648
11180
  repoPath,
10649
11181
  rows: shown,
10650
11182
  truncated: truncatedHistory,
10651
11183
  netCost: netCostOf(shown, args, config, pricing, t),
11184
+ band,
10652
11185
  t,
10653
11186
  }),
10654
11187
  );
10655
11188
 
10656
- printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
11189
+ printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
10657
11190
  }
10658
11191
 
10659
11192
  /**
@@ -10710,6 +11243,15 @@ function printBlame(
10710
11243
  pricing: PricingCatalogue;
10711
11244
  t: CliMessages;
10712
11245
  truncated: boolean;
11246
+ /**
11247
+ * The band of this file's own text, computed once by the caller.
11248
+ *
11249
+ * `blame` walks one file across revisions, so the kind of text is a
11250
+ * property of the file rather than of any one revision. The caller has
11251
+ * today's content and this renderer does not, which is why it arrives
11252
+ * here rather than being read again.
11253
+ */
11254
+ band: number;
10713
11255
  },
10714
11256
  ): void {
10715
11257
  const { repoPath, args, config, pricing, t, truncated } = context;
@@ -10844,7 +11386,7 @@ function printBlame(
10844
11386
  );
10845
11387
  }
10846
11388
 
10847
- console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
11389
+ console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
10848
11390
  }
10849
11391
 
10850
11392
  // --------------------------------------------------------------------------
@@ -11274,6 +11816,12 @@ async function main(): Promise<void> {
11274
11816
  case 'from-otel':
11275
11817
  await commandFromOtel(args, t);
11276
11818
  break;
11819
+ case 'from-litellm':
11820
+ await commandFromLiteLlm(args, t);
11821
+ break;
11822
+ case 'from-helicone':
11823
+ await commandFromHelicone(args, t);
11824
+ break;
11277
11825
  case 'switch':
11278
11826
  await commandSwitch(args, pricing, t);
11279
11827
  break;