@trazum/cli 1.80.1 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/gateway-server.d.ts +38 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +125 -9
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +437 -376
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +68 -5
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +74 -6
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +545 -55
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +10 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +4 -4
- package/dist/markdown.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +160 -9
- package/src/i18n/en.ts +454 -378
- package/src/i18n/es.ts +85 -7
- package/src/i18n/types.ts +76 -6
- package/src/index.ts +602 -54
- package/src/markdown.ts +14 -4
package/src/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
2
3
|
import { readdirSync, readFileSync } from 'node:fs';
|
|
3
4
|
import { mkdir, mkdtemp, open, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
5
|
import { cpus, tmpdir } from 'node:os';
|
|
@@ -111,6 +112,12 @@ import {
|
|
|
111
112
|
parseUsageLine,
|
|
112
113
|
plannedCalls,
|
|
113
114
|
claudeCodeRecords,
|
|
115
|
+
ESTIMATE_ERROR_BAND_PCT,
|
|
116
|
+
bandFor,
|
|
117
|
+
foreignTokenizer,
|
|
118
|
+
measuredForeignError,
|
|
119
|
+
heliconeRecords,
|
|
120
|
+
litellmRecords,
|
|
114
121
|
otelRecords,
|
|
115
122
|
ownRate,
|
|
116
123
|
positionAt,
|
|
@@ -144,9 +151,11 @@ import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cac
|
|
|
144
151
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
145
152
|
import type {
|
|
146
153
|
BucketedReport,
|
|
154
|
+
EvalReport,
|
|
147
155
|
FleetSource,
|
|
148
156
|
HistoryRun,
|
|
149
157
|
MeasuredUsage,
|
|
158
|
+
PruneReport,
|
|
150
159
|
PlanDocument,
|
|
151
160
|
StoredReport,
|
|
152
161
|
VerifiedAction,
|
|
@@ -352,6 +361,7 @@ const VALUE_FLAGS = new Set([
|
|
|
352
361
|
'gpu-usd-hour',
|
|
353
362
|
'tokens-per-second',
|
|
354
363
|
'utilization',
|
|
364
|
+
'state',
|
|
355
365
|
]);
|
|
356
366
|
|
|
357
367
|
function parseArgs(argv: string[], t: CliMessages): Args {
|
|
@@ -674,8 +684,10 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
674
684
|
schema: [],
|
|
675
685
|
rollup: ['json', 'html-out'],
|
|
676
686
|
position: ['json', 'html-out', 'pricing', 'pricing-live'],
|
|
677
|
-
'from-claude-code': ['label', 'label-from-project', 'out', 'o'],
|
|
687
|
+
'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
|
|
678
688
|
'from-otel': ['label-from-service', 'out', 'o'],
|
|
689
|
+
'from-litellm': ['out', 'o'],
|
|
690
|
+
'from-helicone': ['out', 'o'],
|
|
679
691
|
switch: ['to', 'migration-usd', 'cases'],
|
|
680
692
|
ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
|
|
681
693
|
pulse: ['json', 'max-stale-hours'],
|
|
@@ -787,11 +799,11 @@ function renderDiff(before: string, after: string, t: CliMessages): string {
|
|
|
787
799
|
/**
|
|
788
800
|
* The provider's name when the estimator was not calibrated for it.
|
|
789
801
|
*
|
|
790
|
-
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and
|
|
791
|
-
*
|
|
792
|
-
*
|
|
793
|
-
*
|
|
794
|
-
*
|
|
802
|
+
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
|
|
803
|
+
* band descends from that. Printing one beside a GPT or Kimi figure states a
|
|
804
|
+
* precision nobody has measured for that family — and since the catalogue grew
|
|
805
|
+
* past Anthropic, that is most of it. Returns null when the model is
|
|
806
|
+
* Anthropic's, where the band is at least the claim it was written for.
|
|
795
807
|
*/
|
|
796
808
|
function offFamilyName(modelId: string): string | null {
|
|
797
809
|
const provider = getModel(modelId).provider;
|
|
@@ -799,6 +811,18 @@ function offFamilyName(modelId: string): string | null {
|
|
|
799
811
|
return getModel(modelId).displayName;
|
|
800
812
|
}
|
|
801
813
|
|
|
814
|
+
/**
|
|
815
|
+
* How far off the estimator has been measured on this model's family.
|
|
816
|
+
*
|
|
817
|
+
* Read from the catalogue's provider rather than from the display name, because
|
|
818
|
+
* the name is what a reader sees and the id is what was measured. Null on the
|
|
819
|
+
* calibrated family and on every family nobody has run — an unmeasured one gets
|
|
820
|
+
* a sentence saying so, never a borrowed figure.
|
|
821
|
+
*/
|
|
822
|
+
function offFamilyError(modelId: string): number | null {
|
|
823
|
+
return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
|
|
824
|
+
}
|
|
825
|
+
|
|
802
826
|
/**
|
|
803
827
|
* Language codes as names, in the reader's language.
|
|
804
828
|
*
|
|
@@ -881,7 +905,13 @@ function printReport(
|
|
|
881
905
|
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
882
906
|
const sourceNote =
|
|
883
907
|
result.tokenSource === 'heuristic'
|
|
884
|
-
? c.dim(
|
|
908
|
+
? c.dim(
|
|
909
|
+
t.report.estimated(
|
|
910
|
+
offFamilyName(result.usage.model),
|
|
911
|
+
bandFor(result.optimized),
|
|
912
|
+
offFamilyError(result.usage.model),
|
|
913
|
+
),
|
|
914
|
+
)
|
|
885
915
|
: c.dim(t.report.exactCount());
|
|
886
916
|
|
|
887
917
|
/**
|
|
@@ -2600,6 +2630,122 @@ function projectLabelFor(file: string): string | undefined {
|
|
|
2600
2630
|
return trimmed !== '' ? trimmed : folder;
|
|
2601
2631
|
}
|
|
2602
2632
|
|
|
2633
|
+
|
|
2634
|
+
/**
|
|
2635
|
+
* `--state`: read the part of a transcript that is new since last time.
|
|
2636
|
+
*
|
|
2637
|
+
* A Claude Code transcript is append-only and can be enormous — the largest on
|
|
2638
|
+
* one real machine is 212 MB, and re-reading it to price the last thirty
|
|
2639
|
+
* seconds takes six and a half seconds. That is the whole cost of the status
|
|
2640
|
+
* line in `plugin/statusline`, and it is paid on every turn for a file whose
|
|
2641
|
+
* first two hundred megabytes cannot have changed.
|
|
2642
|
+
*
|
|
2643
|
+
* The state file ties three numbers together, and it is the third that makes
|
|
2644
|
+
* this exact rather than approximately right:
|
|
2645
|
+
*
|
|
2646
|
+
* - `offset`: where to start reading the transcript.
|
|
2647
|
+
* - `out`: how long the output file was when everything before `offset` had
|
|
2648
|
+
* been written. The tail after that is the last call as it looked last time,
|
|
2649
|
+
* and it is dropped and re-derived, because the converter's rule is that a
|
|
2650
|
+
* call arrives as several lines and the last one stands.
|
|
2651
|
+
* - `digest`: what the bytes just before `offset` were. A transcript that was
|
|
2652
|
+
* truncated, rotated or replaced would otherwise be resumed into the middle
|
|
2653
|
+
* of a different file, and the output would be a bill assembled from two
|
|
2654
|
+
* unrelated sessions. On a mismatch the whole thing is re-read, which is
|
|
2655
|
+
* slow and correct.
|
|
2656
|
+
*/
|
|
2657
|
+
interface TranscriptState {
|
|
2658
|
+
schemaVersion: 1;
|
|
2659
|
+
files: Record<string, { offset: number; out: number; digest: string }>;
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
/** How much of the run-up to `offset` is fingerprinted. */
|
|
2663
|
+
const STATE_DIGEST_BYTES = 4096;
|
|
2664
|
+
|
|
2665
|
+
const digestOf = (bytes: Buffer): string => createHash('sha256').update(bytes).digest('hex');
|
|
2666
|
+
|
|
2667
|
+
const readState = async (path: string): Promise<TranscriptState> => {
|
|
2668
|
+
try {
|
|
2669
|
+
const parsed: unknown = JSON.parse(await readFile(path, 'utf8'));
|
|
2670
|
+
if (
|
|
2671
|
+
typeof parsed === 'object' &&
|
|
2672
|
+
parsed !== null &&
|
|
2673
|
+
(parsed as TranscriptState).schemaVersion === 1 &&
|
|
2674
|
+
typeof (parsed as TranscriptState).files === 'object'
|
|
2675
|
+
) {
|
|
2676
|
+
return parsed as TranscriptState;
|
|
2677
|
+
}
|
|
2678
|
+
} catch {
|
|
2679
|
+
// A missing, unreadable or unrecognised state file is a cold start, not an
|
|
2680
|
+
// error: the answer it produces is the same one, computed the slow way.
|
|
2681
|
+
}
|
|
2682
|
+
return { schemaVersion: 1, files: {} };
|
|
2683
|
+
};
|
|
2684
|
+
|
|
2685
|
+
/**
|
|
2686
|
+
* The byte offset of a line index within a chunk, counted in bytes rather than
|
|
2687
|
+
* characters.
|
|
2688
|
+
*
|
|
2689
|
+
* `String.prototype.split` counts UTF-16 code units and a transcript is UTF-8
|
|
2690
|
+
* with prompts in it, so a character index used as a byte offset would land
|
|
2691
|
+
* mid-sequence on the first accented character and corrupt every resume after
|
|
2692
|
+
* it. The newlines are found in the Buffer.
|
|
2693
|
+
*/
|
|
2694
|
+
const byteOffsetOfLine = (chunk: Buffer, line: number): number => {
|
|
2695
|
+
if (line <= 0) return 0;
|
|
2696
|
+
let seen = 0;
|
|
2697
|
+
for (let i = 0; i < chunk.length; i += 1) {
|
|
2698
|
+
if (chunk[i] !== 0x0a) continue;
|
|
2699
|
+
seen += 1;
|
|
2700
|
+
if (seen === line) return i + 1;
|
|
2701
|
+
}
|
|
2702
|
+
return chunk.length;
|
|
2703
|
+
};
|
|
2704
|
+
|
|
2705
|
+
/** What a resumable read decided, so the caller can report it honestly. */
|
|
2706
|
+
interface ResumedRead {
|
|
2707
|
+
text: string;
|
|
2708
|
+
/** Bytes skipped because a previous run had already settled them. */
|
|
2709
|
+
skipped: number;
|
|
2710
|
+
/** Where the output must be truncated to before appending. */
|
|
2711
|
+
truncateOutTo: number;
|
|
2712
|
+
/** Records the previous run had settled, and this one must not re-emit. */
|
|
2713
|
+
chunkStart: number;
|
|
2714
|
+
}
|
|
2715
|
+
|
|
2716
|
+
const resumableRead = async (
|
|
2717
|
+
file: string,
|
|
2718
|
+
state: TranscriptState,
|
|
2719
|
+
): Promise<{ chunk: Buffer; read: ResumedRead }> => {
|
|
2720
|
+
const key = resolvePath(file);
|
|
2721
|
+
const entry = state.files[key];
|
|
2722
|
+
const handle = await open(key, 'r');
|
|
2723
|
+
try {
|
|
2724
|
+
const { size } = await handle.stat();
|
|
2725
|
+
let start = 0;
|
|
2726
|
+
let truncateOutTo = 0;
|
|
2727
|
+
|
|
2728
|
+
if (entry !== undefined && entry.offset > 0 && entry.offset <= size) {
|
|
2729
|
+
const runUp = Math.min(STATE_DIGEST_BYTES, entry.offset);
|
|
2730
|
+
const before = Buffer.alloc(runUp);
|
|
2731
|
+
await handle.read(before, 0, runUp, entry.offset - runUp);
|
|
2732
|
+
if (digestOf(before) === entry.digest) {
|
|
2733
|
+
start = entry.offset;
|
|
2734
|
+
truncateOutTo = entry.out;
|
|
2735
|
+
}
|
|
2736
|
+
}
|
|
2737
|
+
|
|
2738
|
+
const chunk = Buffer.alloc(size - start);
|
|
2739
|
+
if (chunk.length > 0) await handle.read(chunk, 0, chunk.length, start);
|
|
2740
|
+
return {
|
|
2741
|
+
chunk,
|
|
2742
|
+
read: { text: chunk.toString('utf8'), skipped: start, truncateOutTo, chunkStart: start },
|
|
2743
|
+
};
|
|
2744
|
+
} finally {
|
|
2745
|
+
await handle.close();
|
|
2746
|
+
}
|
|
2747
|
+
};
|
|
2748
|
+
|
|
2603
2749
|
async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void> {
|
|
2604
2750
|
const target = args.positional[0];
|
|
2605
2751
|
if (target === undefined) throw new Error(t.fromClaudeCode.noPath());
|
|
@@ -2639,6 +2785,28 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
|
|
|
2639
2785
|
* says so.
|
|
2640
2786
|
*/
|
|
2641
2787
|
const labelFromProject = boolFlag(args, 'label-from-project', info.isDirectory());
|
|
2788
|
+
/**
|
|
2789
|
+
* `--state` is for one transcript, deliberately.
|
|
2790
|
+
*
|
|
2791
|
+
* The state ties a transcript offset to a length of the output file, and
|
|
2792
|
+
* with several transcripts appending to one output there is no single length
|
|
2793
|
+
* that means "everything settled": the second file's records sit after the
|
|
2794
|
+
* first file's unsettled tail. A directory is re-read in full, which is what
|
|
2795
|
+
* it was doing before and is the right cost for a walk that is not a status
|
|
2796
|
+
* line refreshing every turn.
|
|
2797
|
+
*/
|
|
2798
|
+
const statePath = stringFlag(args, 'state');
|
|
2799
|
+
if (statePath !== undefined && info.isDirectory()) throw new Error(t.fromClaudeCode.stateNeedsFile());
|
|
2800
|
+
if (statePath !== undefined && stringFlag(args, 'out') === undefined && stringFlag(args, 'o') === undefined) {
|
|
2801
|
+
throw new Error(t.fromClaudeCode.stateNeedsOut());
|
|
2802
|
+
}
|
|
2803
|
+
const state = statePath !== undefined ? await readState(statePath) : undefined;
|
|
2804
|
+
let resumed: ResumedRead | undefined;
|
|
2805
|
+
let chunk: Buffer | undefined;
|
|
2806
|
+
let resumeLine = 0;
|
|
2807
|
+
let settledRecords = 0;
|
|
2808
|
+
let settledOutBytes = 0;
|
|
2809
|
+
|
|
2642
2810
|
const lines: string[] = [];
|
|
2643
2811
|
let records = 0;
|
|
2644
2812
|
let collapsed = 0;
|
|
@@ -2650,7 +2818,15 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
|
|
|
2650
2818
|
let disagreements = 0;
|
|
2651
2819
|
let synthetic = 0;
|
|
2652
2820
|
for (const file of files) {
|
|
2653
|
-
|
|
2821
|
+
let text: string;
|
|
2822
|
+
if (state !== undefined) {
|
|
2823
|
+
const read = await resumableRead(file, state);
|
|
2824
|
+
chunk = read.chunk;
|
|
2825
|
+
resumed = read.read;
|
|
2826
|
+
text = read.read.text;
|
|
2827
|
+
} else {
|
|
2828
|
+
text = await readFile(file, 'utf8');
|
|
2829
|
+
}
|
|
2654
2830
|
const projectLabel = labelFromProject ? projectLabelFor(file) : undefined;
|
|
2655
2831
|
const conversion = claudeCodeRecords(text, {
|
|
2656
2832
|
...(label !== undefined
|
|
@@ -2660,6 +2836,8 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
|
|
|
2660
2836
|
: {}),
|
|
2661
2837
|
});
|
|
2662
2838
|
for (const record of conversion.records) lines.push(JSON.stringify(record));
|
|
2839
|
+
resumeLine = conversion.resume.line;
|
|
2840
|
+
settledRecords = conversion.resume.records;
|
|
2663
2841
|
records += conversion.records.length;
|
|
2664
2842
|
collapsed += conversion.collapsed;
|
|
2665
2843
|
noRequestId += conversion.noRequestId;
|
|
@@ -2673,12 +2851,57 @@ async function commandFromClaudeCode(args: Args, t: CliMessages): Promise<void>
|
|
|
2673
2851
|
|
|
2674
2852
|
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
2675
2853
|
if (out !== undefined) {
|
|
2676
|
-
|
|
2854
|
+
if (statePath !== undefined) {
|
|
2855
|
+
/**
|
|
2856
|
+
* Append rather than rewrite, and drop the tail first.
|
|
2857
|
+
*
|
|
2858
|
+
* The tail is the last call as the previous run recorded it, and this
|
|
2859
|
+
* run has just re-derived it from its own first line. Truncating is what
|
|
2860
|
+
* makes the two runs add up to exactly one reading of the transcript
|
|
2861
|
+
* rather than one and a bit.
|
|
2862
|
+
*/
|
|
2863
|
+
const handle = await open(out, resumed!.truncateOutTo > 0 ? 'r+' : 'w');
|
|
2864
|
+
try {
|
|
2865
|
+
await handle.truncate(resumed!.truncateOutTo);
|
|
2866
|
+
const body = lines.join('\n') + (lines.length > 0 ? '\n' : '');
|
|
2867
|
+
if (body !== '') await handle.write(body, resumed!.truncateOutTo, 'utf8');
|
|
2868
|
+
settledOutBytes =
|
|
2869
|
+
resumed!.truncateOutTo +
|
|
2870
|
+
lines
|
|
2871
|
+
.slice(0, settledRecords)
|
|
2872
|
+
.reduce((sum, line) => sum + Buffer.byteLength(line + '\n', 'utf8'), 0);
|
|
2873
|
+
} finally {
|
|
2874
|
+
await handle.close();
|
|
2875
|
+
}
|
|
2876
|
+
} else {
|
|
2877
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
2878
|
+
}
|
|
2677
2879
|
console.error(t.fromClaudeCode.written(out));
|
|
2678
2880
|
} else {
|
|
2679
2881
|
for (const line of lines) console.log(line);
|
|
2680
2882
|
}
|
|
2681
2883
|
|
|
2884
|
+
if (statePath !== undefined) {
|
|
2885
|
+
const key = resolvePath(files[0]!);
|
|
2886
|
+
const nextOffset = resumed!.chunkStart + byteOffsetOfLine(chunk!, resumeLine);
|
|
2887
|
+
const runUp = Math.min(STATE_DIGEST_BYTES, nextOffset);
|
|
2888
|
+
const before = Buffer.alloc(runUp);
|
|
2889
|
+
if (runUp > 0) {
|
|
2890
|
+
const handle = await open(key, 'r');
|
|
2891
|
+
try {
|
|
2892
|
+
await handle.read(before, 0, runUp, nextOffset - runUp);
|
|
2893
|
+
} finally {
|
|
2894
|
+
await handle.close();
|
|
2895
|
+
}
|
|
2896
|
+
}
|
|
2897
|
+
const next: TranscriptState = {
|
|
2898
|
+
schemaVersion: 1,
|
|
2899
|
+
files: { [key]: { offset: nextOffset, out: settledOutBytes, digest: digestOf(before) } },
|
|
2900
|
+
};
|
|
2901
|
+
await writeFile(statePath, JSON.stringify(next, null, 2) + '\n', 'utf8');
|
|
2902
|
+
console.error(t.fromClaudeCode.resumed(resumed!.skipped, nextOffset));
|
|
2903
|
+
}
|
|
2904
|
+
|
|
2682
2905
|
console.error(t.fromClaudeCode.summary(files.length, records));
|
|
2683
2906
|
if (collapsed > 0) console.error(t.fromClaudeCode.collapsed(collapsed));
|
|
2684
2907
|
if (streamed > 0) console.error(t.fromClaudeCode.streamed(streamed));
|
|
@@ -2887,6 +3110,169 @@ async function commandFromOtel(args: Args, t: CliMessages): Promise<void> {
|
|
|
2887
3110
|
if (unparseable > 0) console.error(t.fromOtel.unparseable(unparseable));
|
|
2888
3111
|
}
|
|
2889
3112
|
|
|
3113
|
+
/**
|
|
3114
|
+
* `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
|
|
3115
|
+
*
|
|
3116
|
+
* The same pure-converter pattern again, pointed at the gateway a great many
|
|
3117
|
+
* teams already put in front of every provider: `LiteLLM_SpendLogs` is the
|
|
3118
|
+
* export most likely to already exist on somebody's disk. `litellmRecords` in
|
|
3119
|
+
* core does the reading; this is the walk and the honesty.
|
|
3120
|
+
*
|
|
3121
|
+
* Three of the four counts it prints exist because the alternative is a
|
|
3122
|
+
* flattering silence. Rows naming no model are not priced by the route they
|
|
3123
|
+
* took, rows with no tokens are logged calls nobody can price, and a
|
|
3124
|
+
* `cache_hit` flag is not a token split — so the cache verdicts read "cannot
|
|
3125
|
+
* tell" rather than a fabricated one, exactly as `from-otel` does.
|
|
3126
|
+
*
|
|
3127
|
+
* The fourth is the one that matters most: LiteLLM prices the same calls with
|
|
3128
|
+
* its own table, and that figure is printed beside Trazum's and never merged
|
|
3129
|
+
* into it. Two price tables summed into one total is how a report becomes
|
|
3130
|
+
* quietly wrong.
|
|
3131
|
+
*
|
|
3132
|
+
* The row carries `messages`, `response`, `api_key`, `requester_ip_address`
|
|
3133
|
+
* and `end_user`. None of it is read; a fixture plants a marker in each and
|
|
3134
|
+
* greps the whole output.
|
|
3135
|
+
*/
|
|
3136
|
+
async function commandFromLiteLlm(args: Args, t: CliMessages): Promise<void> {
|
|
3137
|
+
const target = args.positional[0];
|
|
3138
|
+
if (target === undefined) throw new Error(t.fromLiteLlm.noPath());
|
|
3139
|
+
|
|
3140
|
+
let info;
|
|
3141
|
+
try {
|
|
3142
|
+
info = await stat(target);
|
|
3143
|
+
} catch {
|
|
3144
|
+
throw new Error(t.fromLiteLlm.notFound(target));
|
|
3145
|
+
}
|
|
3146
|
+
const files: string[] = [];
|
|
3147
|
+
if (info.isDirectory()) {
|
|
3148
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
3149
|
+
for (const entry of entries) {
|
|
3150
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
3151
|
+
files.push(join(entry.parentPath, entry.name));
|
|
3152
|
+
}
|
|
3153
|
+
}
|
|
3154
|
+
files.sort();
|
|
3155
|
+
if (files.length === 0) throw new Error(t.fromLiteLlm.noExports(target));
|
|
3156
|
+
} else {
|
|
3157
|
+
files.push(target);
|
|
3158
|
+
}
|
|
3159
|
+
|
|
3160
|
+
const lines: string[] = [];
|
|
3161
|
+
let rows = 0;
|
|
3162
|
+
let unnamedModel = 0;
|
|
3163
|
+
let noTokens = 0;
|
|
3164
|
+
let cacheFlagged = 0;
|
|
3165
|
+
let unparseable = 0;
|
|
3166
|
+
let reportedSpend = 0;
|
|
3167
|
+
let sawSpend = false;
|
|
3168
|
+
for (const file of files) {
|
|
3169
|
+
const conversion = litellmRecords(await readFile(file, 'utf8'));
|
|
3170
|
+
for (const record of conversion.records) lines.push(JSON.stringify(record));
|
|
3171
|
+
rows += conversion.rows;
|
|
3172
|
+
unnamedModel += conversion.unnamedModel;
|
|
3173
|
+
noTokens += conversion.noTokens;
|
|
3174
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
3175
|
+
unparseable += conversion.unparseable;
|
|
3176
|
+
if (conversion.reportedSpendUsd !== null) {
|
|
3177
|
+
reportedSpend += conversion.reportedSpendUsd;
|
|
3178
|
+
sawSpend = true;
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
|
|
3182
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
3183
|
+
if (out !== undefined) {
|
|
3184
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
3185
|
+
console.error(t.fromLiteLlm.written(out));
|
|
3186
|
+
} else {
|
|
3187
|
+
for (const line of lines) console.log(line);
|
|
3188
|
+
}
|
|
3189
|
+
|
|
3190
|
+
console.error(t.fromLiteLlm.summary(files.length, rows));
|
|
3191
|
+
if (unnamedModel > 0) console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
|
|
3192
|
+
if (noTokens > 0) console.error(t.fromLiteLlm.noTokens(noTokens));
|
|
3193
|
+
if (cacheFlagged > 0) console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
|
|
3194
|
+
if (sawSpend) console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
|
|
3195
|
+
if (unparseable > 0) console.error(t.fromLiteLlm.unparseable(unparseable));
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
/**
|
|
3199
|
+
* `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
|
|
3200
|
+
*
|
|
3201
|
+
* The third converter, and the one that needs three columns where the others
|
|
3202
|
+
* need one: Helicone carries `request_model`, `model_override` and
|
|
3203
|
+
* `response_model`, and they can disagree. The response wins, because a bill
|
|
3204
|
+
* is about what was billed rather than what was intended, and the
|
|
3205
|
+
* disagreements are counted so a substitution is something the reader sees.
|
|
3206
|
+
*
|
|
3207
|
+
* Two absences are stated rather than filled in. There is no cache token
|
|
3208
|
+
* split on the row, only a flag; and a Helicone request id names one call and
|
|
3209
|
+
* never a conversation, so the records carry no session and the
|
|
3210
|
+
* conversation-shaped findings stay unavailable. Both are printed, because a
|
|
3211
|
+
* gap that is not said reads as a gap that is not there.
|
|
3212
|
+
*/
|
|
3213
|
+
async function commandFromHelicone(args: Args, t: CliMessages): Promise<void> {
|
|
3214
|
+
const target = args.positional[0];
|
|
3215
|
+
if (target === undefined) throw new Error(t.fromHelicone.noPath());
|
|
3216
|
+
|
|
3217
|
+
let info;
|
|
3218
|
+
try {
|
|
3219
|
+
info = await stat(target);
|
|
3220
|
+
} catch {
|
|
3221
|
+
throw new Error(t.fromHelicone.notFound(target));
|
|
3222
|
+
}
|
|
3223
|
+
const files: string[] = [];
|
|
3224
|
+
if (info.isDirectory()) {
|
|
3225
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
3226
|
+
for (const entry of entries) {
|
|
3227
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
3228
|
+
files.push(join(entry.parentPath, entry.name));
|
|
3229
|
+
}
|
|
3230
|
+
}
|
|
3231
|
+
files.sort();
|
|
3232
|
+
if (files.length === 0) throw new Error(t.fromHelicone.noExports(target));
|
|
3233
|
+
} else {
|
|
3234
|
+
files.push(target);
|
|
3235
|
+
}
|
|
3236
|
+
|
|
3237
|
+
const lines: string[] = [];
|
|
3238
|
+
let rows = 0;
|
|
3239
|
+
let unnamedModel = 0;
|
|
3240
|
+
let disagreements = 0;
|
|
3241
|
+
let noTokens = 0;
|
|
3242
|
+
let cacheFlagged = 0;
|
|
3243
|
+
let unparseable = 0;
|
|
3244
|
+
for (const file of files) {
|
|
3245
|
+
const conversion = heliconeRecords(await readFile(file, 'utf8'));
|
|
3246
|
+
for (const record of conversion.records) lines.push(JSON.stringify(record));
|
|
3247
|
+
rows += conversion.rows;
|
|
3248
|
+
unnamedModel += conversion.unnamedModel;
|
|
3249
|
+
disagreements += conversion.modelDisagreements;
|
|
3250
|
+
noTokens += conversion.noTokens;
|
|
3251
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
3252
|
+
unparseable += conversion.unparseable;
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
3256
|
+
if (out !== undefined) {
|
|
3257
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
3258
|
+
console.error(t.fromHelicone.written(out));
|
|
3259
|
+
} else {
|
|
3260
|
+
for (const line of lines) console.log(line);
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3263
|
+
console.error(t.fromHelicone.summary(files.length, rows));
|
|
3264
|
+
if (unnamedModel > 0) console.error(t.fromHelicone.unnamedModel(unnamedModel));
|
|
3265
|
+
if (disagreements > 0) console.error(t.fromHelicone.disagreements(disagreements));
|
|
3266
|
+
if (noTokens > 0) console.error(t.fromHelicone.noTokens(noTokens));
|
|
3267
|
+
if (cacheFlagged > 0) console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
|
|
3268
|
+
// Said on every run that produced records, not only when something is
|
|
3269
|
+
// missing: the absence is a property of the format, and a reader who has
|
|
3270
|
+
// just converted a month needs to know which questions this log cannot
|
|
3271
|
+
// answer before they go looking for the answers.
|
|
3272
|
+
if (rows > 0) console.error(t.fromHelicone.noSessions());
|
|
3273
|
+
if (unparseable > 0) console.error(t.fromHelicone.unparseable(unparseable));
|
|
3274
|
+
}
|
|
3275
|
+
|
|
2890
3276
|
/**
|
|
2891
3277
|
* `trazum position <usage.jsonl>` — where the month stands, measured.
|
|
2892
3278
|
*
|
|
@@ -5487,6 +5873,7 @@ async function commandCheck(
|
|
|
5487
5873
|
verdicts: [verdict],
|
|
5488
5874
|
level,
|
|
5489
5875
|
tokenSource: counter.source,
|
|
5876
|
+
band: bandFor(prompt),
|
|
5490
5877
|
truncated: false,
|
|
5491
5878
|
t,
|
|
5492
5879
|
}),
|
|
@@ -5622,12 +6009,18 @@ async function checkEmbedded(
|
|
|
5622
6009
|
const failures = verdicts.filter(isOverBudget);
|
|
5623
6010
|
const ok = failures.length === 0 && extraction.declined.length === 0;
|
|
5624
6011
|
|
|
6012
|
+
// Widest across the prompts this report covers, never an average: a run over
|
|
6013
|
+
// prose and a CSV covers text the estimator is 6% and 33% out on, and a
|
|
6014
|
+
// figure between them would describe neither.
|
|
6015
|
+
const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
|
|
6016
|
+
|
|
5625
6017
|
await writeMarkdown(args, () =>
|
|
5626
6018
|
renderCheckMarkdown({
|
|
5627
6019
|
target: path,
|
|
5628
6020
|
verdicts,
|
|
5629
6021
|
level,
|
|
5630
6022
|
tokenSource: counter.source,
|
|
6023
|
+
band,
|
|
5631
6024
|
truncated: false,
|
|
5632
6025
|
t,
|
|
5633
6026
|
}),
|
|
@@ -9448,12 +9841,102 @@ function reportProfileGaps(
|
|
|
9448
9841
|
* exactly as `prune` does. A command that can spend somebody's money without
|
|
9449
9842
|
* telling them first is a command they stop trusting.
|
|
9450
9843
|
*/
|
|
9844
|
+
/**
|
|
9845
|
+
* The routing measurement, without the text that produced it.
|
|
9846
|
+
*
|
|
9847
|
+
* `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
|
|
9848
|
+
* whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
|
|
9849
|
+
* `baseline[1]` and `optimized` (three model answers). It carried no
|
|
9850
|
+
* `schemaVersion` either.
|
|
9851
|
+
*
|
|
9852
|
+
* `docs/json-output.md` opens by promising that **nothing here carries a
|
|
9853
|
+
* session key or prompt text**, and this is the document most likely to be
|
|
9854
|
+
* piped into a dashboard, pasted into a ticket or dropped into a browser. A
|
|
9855
|
+
* tool whose headline is that prompts never leave the machine cannot have its
|
|
9856
|
+
* most portable artefact be the one that carries them.
|
|
9857
|
+
*
|
|
9858
|
+
* The per-case *shape* survives: the two similarity scores are the evidence
|
|
9859
|
+
* behind the verdict, and a reader checking a borderline call needs them. Only
|
|
9860
|
+
* the strings go. Anyone who wants the answers has the human-readable output,
|
|
9861
|
+
* which never left the terminal.
|
|
9862
|
+
*
|
|
9863
|
+
* A named function rather than an inline object so the stripping is a thing a
|
|
9864
|
+
* test can call without spending a provider call to reach it.
|
|
9865
|
+
*/
|
|
9866
|
+
export function routeDocument(slice: unknown, result: EvalReport): unknown {
|
|
9867
|
+
return {
|
|
9868
|
+
schemaVersion: 1,
|
|
9869
|
+
slice,
|
|
9870
|
+
evaluation: {
|
|
9871
|
+
provider: result.provider,
|
|
9872
|
+
model: result.model,
|
|
9873
|
+
candidateModel: result.candidateModel,
|
|
9874
|
+
verdict: result.verdict,
|
|
9875
|
+
selfAgreement: result.selfAgreement,
|
|
9876
|
+
crossAgreement: result.crossAgreement,
|
|
9877
|
+
callsMade: result.callsMade,
|
|
9878
|
+
cases: result.cases.map((one: EvalReport['cases'][number]) => ({
|
|
9879
|
+
selfSimilarity: one.selfSimilarity,
|
|
9880
|
+
crossSimilarity: one.crossSimilarity,
|
|
9881
|
+
})),
|
|
9882
|
+
},
|
|
9883
|
+
};
|
|
9884
|
+
}
|
|
9885
|
+
|
|
9886
|
+
/**
|
|
9887
|
+
* The example-pruning measurement, likewise.
|
|
9888
|
+
*
|
|
9889
|
+
* `ExampleContribution.text` is *"the example itself, so a report can quote its
|
|
9890
|
+
* first line"* — true of the terminal rendering and wrong of a document that
|
|
9891
|
+
* promises to carry no prompt text. A few-shot example **is** prompt text, and
|
|
9892
|
+
* this was the second command emitting it.
|
|
9893
|
+
*
|
|
9894
|
+
* `index` and `tokens` say which example without quoting it, which is what a
|
|
9895
|
+
* machine reading this needs: it already has the prompt.
|
|
9896
|
+
*/
|
|
9897
|
+
export function pruneDocument(report: PruneReport): unknown {
|
|
9898
|
+
return {
|
|
9899
|
+
schemaVersion: 1,
|
|
9900
|
+
provider: report.provider,
|
|
9901
|
+
model: report.model,
|
|
9902
|
+
selfAgreement: report.selfAgreement,
|
|
9903
|
+
recoverableTokens: report.recoverableTokens,
|
|
9904
|
+
callsMade: report.callsMade,
|
|
9905
|
+
contributions: report.contributions.map((one: PruneReport['contributions'][number]) => ({
|
|
9906
|
+
index: one.index,
|
|
9907
|
+
tokens: one.tokens,
|
|
9908
|
+
agreementWithout: one.agreementWithout,
|
|
9909
|
+
verdict: one.verdict,
|
|
9910
|
+
})),
|
|
9911
|
+
};
|
|
9912
|
+
}
|
|
9913
|
+
|
|
9451
9914
|
async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
|
|
9915
|
+
/**
|
|
9916
|
+
* Under `--json`, stdout carries the document and nothing else.
|
|
9917
|
+
*
|
|
9918
|
+
* It carried the human preamble too — the slice it picked, the call count,
|
|
9919
|
+
* the "nothing has been spent yet" line — straight into the same stream as
|
|
9920
|
+
* the JSON, so `route --json > verdict.json` wrote a file no parser would
|
|
9921
|
+
* read. Found by running the command and feeding the file to the bridge
|
|
9922
|
+
* that reads it, which is the one thing the pure-function guards could not
|
|
9923
|
+
* do: `interchange.test.js` names `route` among the commands it cannot
|
|
9924
|
+
* drive, so nothing checked that its `--json` output was a document.
|
|
9925
|
+
*
|
|
9926
|
+
* The lines go to stderr rather than away. A reader running this in a
|
|
9927
|
+
* terminal still needs to see which slice was picked and what it will cost
|
|
9928
|
+
* before it costs it; a pipe needs the document. stderr gives both.
|
|
9929
|
+
*/
|
|
9930
|
+
const asJson = boolFlag(args, 'json');
|
|
9931
|
+
const say = (line = ''): void => {
|
|
9932
|
+
if (asJson) console.error(line);
|
|
9933
|
+
else console.log(line);
|
|
9934
|
+
};
|
|
9452
9935
|
const path = args.positional[0];
|
|
9453
9936
|
if (path === undefined) {
|
|
9454
|
-
|
|
9455
|
-
|
|
9456
|
-
|
|
9937
|
+
say();
|
|
9938
|
+
say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
9939
|
+
say();
|
|
9457
9940
|
return;
|
|
9458
9941
|
}
|
|
9459
9942
|
|
|
@@ -9476,18 +9959,18 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9476
9959
|
const available = report.byLabel
|
|
9477
9960
|
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
9478
9961
|
.join(', ');
|
|
9479
|
-
|
|
9480
|
-
|
|
9481
|
-
|
|
9962
|
+
say();
|
|
9963
|
+
say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
9964
|
+
say();
|
|
9482
9965
|
return;
|
|
9483
9966
|
}
|
|
9484
9967
|
const slice = levers.slices.find(
|
|
9485
9968
|
(s) => s.route !== null && (wanted === undefined || s.label === wanted),
|
|
9486
9969
|
);
|
|
9487
9970
|
if (!slice?.route) {
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9971
|
+
say();
|
|
9972
|
+
say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
9973
|
+
say();
|
|
9491
9974
|
return;
|
|
9492
9975
|
}
|
|
9493
9976
|
|
|
@@ -9511,8 +9994,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9511
9994
|
|
|
9512
9995
|
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
9513
9996
|
const worth = formatUsd(slice.route.savingUsd);
|
|
9514
|
-
|
|
9515
|
-
|
|
9997
|
+
say();
|
|
9998
|
+
say(
|
|
9516
9999
|
` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`,
|
|
9517
10000
|
);
|
|
9518
10001
|
/**
|
|
@@ -9525,20 +10008,20 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9525
10008
|
* counts, so it is stated rather than guessed at.
|
|
9526
10009
|
*/
|
|
9527
10010
|
if (slice.label === UNLABELLED) {
|
|
9528
|
-
|
|
10011
|
+
say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
9529
10012
|
}
|
|
9530
|
-
|
|
9531
|
-
|
|
10013
|
+
say();
|
|
10014
|
+
say(
|
|
9532
10015
|
` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`,
|
|
9533
10016
|
);
|
|
9534
10017
|
|
|
9535
10018
|
if (!boolFlag(args, 'yes')) {
|
|
9536
|
-
|
|
9537
|
-
|
|
10019
|
+
say(` ${c.dim(t.route.dryRun())}`);
|
|
10020
|
+
say();
|
|
9538
10021
|
return;
|
|
9539
10022
|
}
|
|
9540
10023
|
|
|
9541
|
-
|
|
10024
|
+
say(` ${c.dim(t.route.running(inputs.length))}`);
|
|
9542
10025
|
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
9543
10026
|
// prompt twice is what says so at the call site.
|
|
9544
10027
|
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
@@ -9547,22 +10030,22 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9547
10030
|
});
|
|
9548
10031
|
|
|
9549
10032
|
if (boolFlag(args, 'json')) {
|
|
9550
|
-
console.log(JSON.stringify(
|
|
10033
|
+
console.log(JSON.stringify(routeDocument(slice, result), null, 2));
|
|
9551
10034
|
return;
|
|
9552
10035
|
}
|
|
9553
10036
|
|
|
9554
10037
|
const asPct = (v: number): string => `${(v * 100).toFixed(0)}%`;
|
|
9555
|
-
|
|
9556
|
-
|
|
10038
|
+
say();
|
|
10039
|
+
say(
|
|
9557
10040
|
` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`,
|
|
9558
10041
|
);
|
|
9559
|
-
|
|
10042
|
+
say();
|
|
9560
10043
|
if (result.verdict === 'inconclusive') {
|
|
9561
|
-
|
|
10044
|
+
say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
9562
10045
|
} else if (result.verdict === 'diverges') {
|
|
9563
|
-
|
|
10046
|
+
say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
9564
10047
|
} else {
|
|
9565
|
-
|
|
10048
|
+
say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
9566
10049
|
}
|
|
9567
10050
|
/**
|
|
9568
10051
|
* Printed on every verdict including the good one. Agreement is not
|
|
@@ -9570,8 +10053,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9570
10053
|
* ever right, and a green tick that let somebody forget that would be the tool
|
|
9571
10054
|
* overstating what it knows.
|
|
9572
10055
|
*/
|
|
9573
|
-
|
|
9574
|
-
|
|
10056
|
+
say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
10057
|
+
say();
|
|
9575
10058
|
}
|
|
9576
10059
|
|
|
9577
10060
|
/**
|
|
@@ -9738,6 +10221,14 @@ interface PromptScan {
|
|
|
9738
10221
|
truncated: boolean;
|
|
9739
10222
|
/** The extensions actually walked, so an error can name them. */
|
|
9740
10223
|
extensions: string[];
|
|
10224
|
+
/**
|
|
10225
|
+
* The widest measured band across the prompts this scan judged.
|
|
10226
|
+
*
|
|
10227
|
+
* Computed here because this is where the texts are, and widest rather than
|
|
10228
|
+
* averaged: a scan over a prose prompt and a CSV covers text the estimator
|
|
10229
|
+
* is 6% and 33% out on, and a figure between them describes neither.
|
|
10230
|
+
*/
|
|
10231
|
+
band: number;
|
|
9741
10232
|
}
|
|
9742
10233
|
|
|
9743
10234
|
/**
|
|
@@ -9817,6 +10308,9 @@ async function scanPrompts(
|
|
|
9817
10308
|
const verdicts: FileVerdict[] = [];
|
|
9818
10309
|
const declined: Array<{ path: string; line: number; detail: string }> = [];
|
|
9819
10310
|
|
|
10311
|
+
// Every prompt this run judged, so the report's band is the widest of them
|
|
10312
|
+
// rather than one figure asserted over text of several kinds.
|
|
10313
|
+
const judgedTexts: string[] = [];
|
|
9820
10314
|
for (const relativePath of files) {
|
|
9821
10315
|
const text = capInput(
|
|
9822
10316
|
await readFile(join(root, relativePath), 'utf8'),
|
|
@@ -9824,6 +10318,7 @@ async function scanPrompts(
|
|
|
9824
10318
|
maxInputFlag(args, t),
|
|
9825
10319
|
t,
|
|
9826
10320
|
);
|
|
10321
|
+
judgedTexts.push(text);
|
|
9827
10322
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
9828
10323
|
// `prompts/**` has to be matched against the path including the root the
|
|
9829
10324
|
// user passed — not against the name relative to it.
|
|
@@ -9857,7 +10352,7 @@ async function scanPrompts(
|
|
|
9857
10352
|
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
9858
10353
|
}
|
|
9859
10354
|
|
|
9860
|
-
return { verdicts, declined, truncated, extensions };
|
|
10355
|
+
return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
|
|
9861
10356
|
}
|
|
9862
10357
|
|
|
9863
10358
|
async function checkDirectory(
|
|
@@ -9889,7 +10384,7 @@ async function checkDirectory(
|
|
|
9889
10384
|
only = checkable;
|
|
9890
10385
|
}
|
|
9891
10386
|
|
|
9892
|
-
const { verdicts, declined, truncated, extensions } = await scanPrompts(
|
|
10387
|
+
const { verdicts, declined, truncated, extensions, band } = await scanPrompts(
|
|
9893
10388
|
root,
|
|
9894
10389
|
args,
|
|
9895
10390
|
flagBudget,
|
|
@@ -9967,6 +10462,7 @@ async function checkDirectory(
|
|
|
9967
10462
|
renderCheckMarkdown({
|
|
9968
10463
|
target: root,
|
|
9969
10464
|
verdicts,
|
|
10465
|
+
band,
|
|
9970
10466
|
level,
|
|
9971
10467
|
tokenSource: counter.source,
|
|
9972
10468
|
truncated,
|
|
@@ -10147,6 +10643,13 @@ function joinPosix(root: string, relativePath: string): string {
|
|
|
10147
10643
|
* never "delete this", and nothing here edits the prompt.
|
|
10148
10644
|
*/
|
|
10149
10645
|
async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
10646
|
+
// Same as `route`: under `--json` stdout is the document, and the estimate
|
|
10647
|
+
// a reader has to see before spending goes to stderr rather than away.
|
|
10648
|
+
const asJson = boolFlag(args, 'json');
|
|
10649
|
+
const say = (line = ''): void => {
|
|
10650
|
+
if (asJson) console.error(line);
|
|
10651
|
+
else console.log(line);
|
|
10652
|
+
};
|
|
10150
10653
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
10151
10654
|
|
|
10152
10655
|
const casesPath = stringFlag(args, 'cases');
|
|
@@ -10161,11 +10664,11 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10161
10664
|
|
|
10162
10665
|
// Printed before the key is even looked up, so somebody weighing it up does not
|
|
10163
10666
|
// need a configured provider to see the number.
|
|
10164
|
-
|
|
10165
|
-
|
|
10667
|
+
say();
|
|
10668
|
+
say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
|
|
10166
10669
|
|
|
10167
10670
|
if (!boolFlag(args, 'yes')) {
|
|
10168
|
-
|
|
10671
|
+
say(c.yellow(` ${t.prune.needsConsent()}`));
|
|
10169
10672
|
return;
|
|
10170
10673
|
}
|
|
10171
10674
|
|
|
@@ -10177,15 +10680,15 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10177
10680
|
});
|
|
10178
10681
|
|
|
10179
10682
|
if (boolFlag(args, 'json')) {
|
|
10180
|
-
console.log(JSON.stringify(report, null, 2));
|
|
10683
|
+
console.log(JSON.stringify(pruneDocument(report), null, 2));
|
|
10181
10684
|
return;
|
|
10182
10685
|
}
|
|
10183
10686
|
|
|
10184
10687
|
const pct = (value: number): string => `${(value * 100).toFixed(0)}%`;
|
|
10185
|
-
|
|
10186
|
-
|
|
10187
|
-
|
|
10188
|
-
|
|
10688
|
+
say();
|
|
10689
|
+
say(sectionHeading(t.prune.heading(provider.model)));
|
|
10690
|
+
say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
|
|
10691
|
+
say();
|
|
10189
10692
|
|
|
10190
10693
|
for (const contribution of report.contributions) {
|
|
10191
10694
|
/**
|
|
@@ -10204,7 +10707,7 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10204
10707
|
? t.prune.verdictNeeded()
|
|
10205
10708
|
: t.prune.verdictRecoverable();
|
|
10206
10709
|
|
|
10207
|
-
|
|
10710
|
+
say(
|
|
10208
10711
|
` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
|
|
10209
10712
|
+ ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`,
|
|
10210
10713
|
);
|
|
@@ -10217,16 +10720,16 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10217
10720
|
*/
|
|
10218
10721
|
const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
|
|
10219
10722
|
const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
|
|
10220
|
-
|
|
10723
|
+
say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
|
|
10221
10724
|
}
|
|
10222
10725
|
|
|
10223
|
-
|
|
10726
|
+
say();
|
|
10224
10727
|
if (report.recoverableTokens > 0) {
|
|
10225
|
-
|
|
10728
|
+
say(` ${t.prune.recoverable(report.recoverableTokens)}`);
|
|
10226
10729
|
}
|
|
10227
|
-
|
|
10228
|
-
|
|
10229
|
-
|
|
10730
|
+
say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
|
|
10731
|
+
say();
|
|
10732
|
+
say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
10230
10733
|
}
|
|
10231
10734
|
|
|
10232
10735
|
async function commandEval(
|
|
@@ -10545,6 +11048,18 @@ const DOCTOR_LIST_LIMIT = 8;
|
|
|
10545
11048
|
const BLAME_DEFAULT_LIMIT = 20;
|
|
10546
11049
|
const BLAME_MAX_LIMIT = 500;
|
|
10547
11050
|
|
|
11051
|
+
/**
|
|
11052
|
+
* The band for a report covering several prompts.
|
|
11053
|
+
*
|
|
11054
|
+
* The widest of them, and the empty case is the widest published band rather
|
|
11055
|
+
* than a narrow default: a report over nothing has measured nothing, and
|
|
11056
|
+
* guessing narrow there is the one direction that produces a false claim.
|
|
11057
|
+
*/
|
|
11058
|
+
function widestBand(texts: readonly string[]): number {
|
|
11059
|
+
if (texts.length === 0) return ESTIMATE_ERROR_BAND_PCT;
|
|
11060
|
+
return Math.max(...texts.map((text) => bandFor(text)));
|
|
11061
|
+
}
|
|
11062
|
+
|
|
10548
11063
|
interface BlameRow {
|
|
10549
11064
|
revision: Revision;
|
|
10550
11065
|
/** `null` when the file did not exist at that commit, or held no marked prompt. */
|
|
@@ -10643,17 +11158,35 @@ async function commandBlame(
|
|
|
10643
11158
|
const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
|
|
10644
11159
|
const truncatedHistory = revisions.length > limit;
|
|
10645
11160
|
|
|
11161
|
+
/*
|
|
11162
|
+
The band of this file's own text, read once and handed to both renderers.
|
|
11163
|
+
|
|
11164
|
+
`blame` walks one file across revisions, so the kind of text is a property
|
|
11165
|
+
of the file, and today's content is the closest thing to it that exists
|
|
11166
|
+
here. A file that is gone from the working tree — renamed away, or deleted
|
|
11167
|
+
in the commit being blamed — falls back to the widest published band, which
|
|
11168
|
+
is exactly what that constant is for: a caller that cannot see the text can
|
|
11169
|
+
only guess in the safe direction.
|
|
11170
|
+
*/
|
|
11171
|
+
let band = ESTIMATE_ERROR_BAND_PCT;
|
|
11172
|
+
try {
|
|
11173
|
+
band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
|
|
11174
|
+
} catch {
|
|
11175
|
+
// Not on disk now. The wide band stands and says so by being wide.
|
|
11176
|
+
}
|
|
11177
|
+
|
|
10646
11178
|
await writeMarkdown(args, () =>
|
|
10647
11179
|
renderBlameMarkdown({
|
|
10648
11180
|
repoPath,
|
|
10649
11181
|
rows: shown,
|
|
10650
11182
|
truncated: truncatedHistory,
|
|
10651
11183
|
netCost: netCostOf(shown, args, config, pricing, t),
|
|
11184
|
+
band,
|
|
10652
11185
|
t,
|
|
10653
11186
|
}),
|
|
10654
11187
|
);
|
|
10655
11188
|
|
|
10656
|
-
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
|
|
11189
|
+
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
|
|
10657
11190
|
}
|
|
10658
11191
|
|
|
10659
11192
|
/**
|
|
@@ -10710,6 +11243,15 @@ function printBlame(
|
|
|
10710
11243
|
pricing: PricingCatalogue;
|
|
10711
11244
|
t: CliMessages;
|
|
10712
11245
|
truncated: boolean;
|
|
11246
|
+
/**
|
|
11247
|
+
* The band of this file's own text, computed once by the caller.
|
|
11248
|
+
*
|
|
11249
|
+
* `blame` walks one file across revisions, so the kind of text is a
|
|
11250
|
+
* property of the file rather than of any one revision. The caller has
|
|
11251
|
+
* today's content and this renderer does not, which is why it arrives
|
|
11252
|
+
* here rather than being read again.
|
|
11253
|
+
*/
|
|
11254
|
+
band: number;
|
|
10713
11255
|
},
|
|
10714
11256
|
): void {
|
|
10715
11257
|
const { repoPath, args, config, pricing, t, truncated } = context;
|
|
@@ -10844,7 +11386,7 @@ function printBlame(
|
|
|
10844
11386
|
);
|
|
10845
11387
|
}
|
|
10846
11388
|
|
|
10847
|
-
console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
|
|
11389
|
+
console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
|
|
10848
11390
|
}
|
|
10849
11391
|
|
|
10850
11392
|
// --------------------------------------------------------------------------
|
|
@@ -11274,6 +11816,12 @@ async function main(): Promise<void> {
|
|
|
11274
11816
|
case 'from-otel':
|
|
11275
11817
|
await commandFromOtel(args, t);
|
|
11276
11818
|
break;
|
|
11819
|
+
case 'from-litellm':
|
|
11820
|
+
await commandFromLiteLlm(args, t);
|
|
11821
|
+
break;
|
|
11822
|
+
case 'from-helicone':
|
|
11823
|
+
await commandFromHelicone(args, t);
|
|
11824
|
+
break;
|
|
11277
11825
|
case 'switch':
|
|
11278
11826
|
await commandSwitch(args, pricing, t);
|
|
11279
11827
|
break;
|