@trazum/cli 1.81.0 → 1.82.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/gateway-server.d.ts.map +1 -1
- package/dist/gateway-server.js +14 -0
- package/dist/gateway-server.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +61 -5
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +63 -5
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +68 -6
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.d.ts +54 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +389 -52
- package/dist/index.js.map +1 -1
- package/dist/markdown.d.ts +10 -0
- package/dist/markdown.d.ts.map +1 -1
- package/dist/markdown.js +4 -4
- package/dist/markdown.js.map +1 -1
- package/package.json +2 -2
- package/src/gateway-server.ts +14 -0
- package/src/i18n/en.ts +76 -7
- package/src/i18n/es.ts +78 -7
- package/src/i18n/types.ts +70 -6
- package/src/index.ts +404 -51
- package/src/markdown.ts +14 -4
package/src/index.ts
CHANGED
|
@@ -112,6 +112,12 @@ import {
|
|
|
112
112
|
parseUsageLine,
|
|
113
113
|
plannedCalls,
|
|
114
114
|
claudeCodeRecords,
|
|
115
|
+
ESTIMATE_ERROR_BAND_PCT,
|
|
116
|
+
bandFor,
|
|
117
|
+
foreignTokenizer,
|
|
118
|
+
measuredForeignError,
|
|
119
|
+
heliconeRecords,
|
|
120
|
+
litellmRecords,
|
|
115
121
|
otelRecords,
|
|
116
122
|
ownRate,
|
|
117
123
|
positionAt,
|
|
@@ -145,9 +151,11 @@ import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cac
|
|
|
145
151
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
146
152
|
import type {
|
|
147
153
|
BucketedReport,
|
|
154
|
+
EvalReport,
|
|
148
155
|
FleetSource,
|
|
149
156
|
HistoryRun,
|
|
150
157
|
MeasuredUsage,
|
|
158
|
+
PruneReport,
|
|
151
159
|
PlanDocument,
|
|
152
160
|
StoredReport,
|
|
153
161
|
VerifiedAction,
|
|
@@ -678,6 +686,8 @@ const COMMAND_FLAGS: Record<string, string[]> = {
|
|
|
678
686
|
position: ['json', 'html-out', 'pricing', 'pricing-live'],
|
|
679
687
|
'from-claude-code': ['label', 'label-from-project', 'out', 'o', 'state'],
|
|
680
688
|
'from-otel': ['label-from-service', 'out', 'o'],
|
|
689
|
+
'from-litellm': ['out', 'o'],
|
|
690
|
+
'from-helicone': ['out', 'o'],
|
|
681
691
|
switch: ['to', 'migration-usd', 'cases'],
|
|
682
692
|
ownrate: ['gpu-usd-hour', 'tokens-per-second', 'utilization'],
|
|
683
693
|
pulse: ['json', 'max-stale-hours'],
|
|
@@ -789,11 +799,11 @@ function renderDiff(before: string, after: string, t: CliMessages): string {
|
|
|
789
799
|
/**
|
|
790
800
|
* The provider's name when the estimator was not calibrated for it.
|
|
791
801
|
*
|
|
792
|
-
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and
|
|
793
|
-
*
|
|
794
|
-
*
|
|
795
|
-
*
|
|
796
|
-
*
|
|
802
|
+
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and every
|
|
803
|
+
* band descends from that. Printing one beside a GPT or Kimi figure states a
|
|
804
|
+
* precision nobody has measured for that family — and since the catalogue grew
|
|
805
|
+
* past Anthropic, that is most of it. Returns null when the model is
|
|
806
|
+
* Anthropic's, where the band is at least the claim it was written for.
|
|
797
807
|
*/
|
|
798
808
|
function offFamilyName(modelId: string): string | null {
|
|
799
809
|
const provider = getModel(modelId).provider;
|
|
@@ -801,6 +811,18 @@ function offFamilyName(modelId: string): string | null {
|
|
|
801
811
|
return getModel(modelId).displayName;
|
|
802
812
|
}
|
|
803
813
|
|
|
814
|
+
/**
|
|
815
|
+
* How far off the estimator has been measured on this model's family.
|
|
816
|
+
*
|
|
817
|
+
* Read from the catalogue's provider rather than from the display name, because
|
|
818
|
+
* the name is what a reader sees and the id is what was measured. Null on the
|
|
819
|
+
* calibrated family and on every family nobody has run — an unmeasured one gets
|
|
820
|
+
* a sentence saying so, never a borrowed figure.
|
|
821
|
+
*/
|
|
822
|
+
function offFamilyError(modelId: string): number | null {
|
|
823
|
+
return measuredForeignError(foreignTokenizer(getModel(modelId).provider ?? null));
|
|
824
|
+
}
|
|
825
|
+
|
|
804
826
|
/**
|
|
805
827
|
* Language codes as names, in the reader's language.
|
|
806
828
|
*
|
|
@@ -883,7 +905,13 @@ function printReport(
|
|
|
883
905
|
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
884
906
|
const sourceNote =
|
|
885
907
|
result.tokenSource === 'heuristic'
|
|
886
|
-
? c.dim(
|
|
908
|
+
? c.dim(
|
|
909
|
+
t.report.estimated(
|
|
910
|
+
offFamilyName(result.usage.model),
|
|
911
|
+
bandFor(result.optimized),
|
|
912
|
+
offFamilyError(result.usage.model),
|
|
913
|
+
),
|
|
914
|
+
)
|
|
887
915
|
: c.dim(t.report.exactCount());
|
|
888
916
|
|
|
889
917
|
/**
|
|
@@ -3082,6 +3110,169 @@ async function commandFromOtel(args: Args, t: CliMessages): Promise<void> {
|
|
|
3082
3110
|
if (unparseable > 0) console.error(t.fromOtel.unparseable(unparseable));
|
|
3083
3111
|
}
|
|
3084
3112
|
|
|
3113
|
+
/**
|
|
3114
|
+
* `trazum from-litellm <file|dir>` — a LiteLLM spend log as a usage log.
|
|
3115
|
+
*
|
|
3116
|
+
* The same pure-converter pattern again, pointed at the gateway a great many
|
|
3117
|
+
* teams already put in front of every provider: `LiteLLM_SpendLogs` is the
|
|
3118
|
+
* export most likely to already exist on somebody's disk. `litellmRecords` in
|
|
3119
|
+
* core does the reading; this is the walk and the honesty.
|
|
3120
|
+
*
|
|
3121
|
+
* Three of the four counts it prints exist because the alternative is a
|
|
3122
|
+
* flattering silence. Rows naming no model are not priced by the route they
|
|
3123
|
+
* took, rows with no tokens are logged calls nobody can price, and a
|
|
3124
|
+
* `cache_hit` flag is not a token split — so the cache verdicts read "cannot
|
|
3125
|
+
* tell" rather than a fabricated one, exactly as `from-otel` does.
|
|
3126
|
+
*
|
|
3127
|
+
* The fourth is the one that matters most: LiteLLM prices the same calls with
|
|
3128
|
+
* its own table, and that figure is printed beside Trazum's and never merged
|
|
3129
|
+
* into it. Two price tables summed into one total is how a report becomes
|
|
3130
|
+
* quietly wrong.
|
|
3131
|
+
*
|
|
3132
|
+
* The row carries `messages`, `response`, `api_key`, `requester_ip_address`
|
|
3133
|
+
* and `end_user`. None of it is read; a fixture plants a marker in each and
|
|
3134
|
+
* greps the whole output.
|
|
3135
|
+
*/
|
|
3136
|
+
async function commandFromLiteLlm(args: Args, t: CliMessages): Promise<void> {
|
|
3137
|
+
const target = args.positional[0];
|
|
3138
|
+
if (target === undefined) throw new Error(t.fromLiteLlm.noPath());
|
|
3139
|
+
|
|
3140
|
+
let info;
|
|
3141
|
+
try {
|
|
3142
|
+
info = await stat(target);
|
|
3143
|
+
} catch {
|
|
3144
|
+
throw new Error(t.fromLiteLlm.notFound(target));
|
|
3145
|
+
}
|
|
3146
|
+
const files: string[] = [];
|
|
3147
|
+
if (info.isDirectory()) {
|
|
3148
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
3149
|
+
for (const entry of entries) {
|
|
3150
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
3151
|
+
files.push(join(entry.parentPath, entry.name));
|
|
3152
|
+
}
|
|
3153
|
+
}
|
|
3154
|
+
files.sort();
|
|
3155
|
+
if (files.length === 0) throw new Error(t.fromLiteLlm.noExports(target));
|
|
3156
|
+
} else {
|
|
3157
|
+
files.push(target);
|
|
3158
|
+
}
|
|
3159
|
+
|
|
3160
|
+
const lines: string[] = [];
|
|
3161
|
+
let rows = 0;
|
|
3162
|
+
let unnamedModel = 0;
|
|
3163
|
+
let noTokens = 0;
|
|
3164
|
+
let cacheFlagged = 0;
|
|
3165
|
+
let unparseable = 0;
|
|
3166
|
+
let reportedSpend = 0;
|
|
3167
|
+
let sawSpend = false;
|
|
3168
|
+
for (const file of files) {
|
|
3169
|
+
const conversion = litellmRecords(await readFile(file, 'utf8'));
|
|
3170
|
+
for (const record of conversion.records) lines.push(JSON.stringify(record));
|
|
3171
|
+
rows += conversion.rows;
|
|
3172
|
+
unnamedModel += conversion.unnamedModel;
|
|
3173
|
+
noTokens += conversion.noTokens;
|
|
3174
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
3175
|
+
unparseable += conversion.unparseable;
|
|
3176
|
+
if (conversion.reportedSpendUsd !== null) {
|
|
3177
|
+
reportedSpend += conversion.reportedSpendUsd;
|
|
3178
|
+
sawSpend = true;
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
|
|
3182
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
3183
|
+
if (out !== undefined) {
|
|
3184
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
3185
|
+
console.error(t.fromLiteLlm.written(out));
|
|
3186
|
+
} else {
|
|
3187
|
+
for (const line of lines) console.log(line);
|
|
3188
|
+
}
|
|
3189
|
+
|
|
3190
|
+
console.error(t.fromLiteLlm.summary(files.length, rows));
|
|
3191
|
+
if (unnamedModel > 0) console.error(t.fromLiteLlm.unnamedModel(unnamedModel));
|
|
3192
|
+
if (noTokens > 0) console.error(t.fromLiteLlm.noTokens(noTokens));
|
|
3193
|
+
if (cacheFlagged > 0) console.error(t.fromLiteLlm.cacheFlagged(cacheFlagged));
|
|
3194
|
+
if (sawSpend) console.error(t.fromLiteLlm.reportedSpend(formatUsd(reportedSpend)));
|
|
3195
|
+
if (unparseable > 0) console.error(t.fromLiteLlm.unparseable(unparseable));
|
|
3196
|
+
}
|
|
3197
|
+
|
|
3198
|
+
/**
|
|
3199
|
+
* `trazum from-helicone <file|dir>` — a Helicone request export as a usage log.
|
|
3200
|
+
*
|
|
3201
|
+
* The third converter, and the one that needs three columns where the others
|
|
3202
|
+
* need one: Helicone carries `request_model`, `model_override` and
|
|
3203
|
+
* `response_model`, and they can disagree. The response wins, because a bill
|
|
3204
|
+
* is about what was billed rather than what was intended, and the
|
|
3205
|
+
* disagreements are counted so a substitution is something the reader sees.
|
|
3206
|
+
*
|
|
3207
|
+
* Two absences are stated rather than filled in. There is no cache token
|
|
3208
|
+
* split on the row, only a flag; and a Helicone request id names one call and
|
|
3209
|
+
* never a conversation, so the records carry no session and the
|
|
3210
|
+
* conversation-shaped findings stay unavailable. Both are printed, because a
|
|
3211
|
+
* gap that is not said reads as a gap that is not there.
|
|
3212
|
+
*/
|
|
3213
|
+
async function commandFromHelicone(args: Args, t: CliMessages): Promise<void> {
|
|
3214
|
+
const target = args.positional[0];
|
|
3215
|
+
if (target === undefined) throw new Error(t.fromHelicone.noPath());
|
|
3216
|
+
|
|
3217
|
+
let info;
|
|
3218
|
+
try {
|
|
3219
|
+
info = await stat(target);
|
|
3220
|
+
} catch {
|
|
3221
|
+
throw new Error(t.fromHelicone.notFound(target));
|
|
3222
|
+
}
|
|
3223
|
+
const files: string[] = [];
|
|
3224
|
+
if (info.isDirectory()) {
|
|
3225
|
+
const entries = await readdir(target, { recursive: true, withFileTypes: true });
|
|
3226
|
+
for (const entry of entries) {
|
|
3227
|
+
if (entry.isFile() && /\.(json|jsonl|ndjson)$/i.test(entry.name)) {
|
|
3228
|
+
files.push(join(entry.parentPath, entry.name));
|
|
3229
|
+
}
|
|
3230
|
+
}
|
|
3231
|
+
files.sort();
|
|
3232
|
+
if (files.length === 0) throw new Error(t.fromHelicone.noExports(target));
|
|
3233
|
+
} else {
|
|
3234
|
+
files.push(target);
|
|
3235
|
+
}
|
|
3236
|
+
|
|
3237
|
+
const lines: string[] = [];
|
|
3238
|
+
let rows = 0;
|
|
3239
|
+
let unnamedModel = 0;
|
|
3240
|
+
let disagreements = 0;
|
|
3241
|
+
let noTokens = 0;
|
|
3242
|
+
let cacheFlagged = 0;
|
|
3243
|
+
let unparseable = 0;
|
|
3244
|
+
for (const file of files) {
|
|
3245
|
+
const conversion = heliconeRecords(await readFile(file, 'utf8'));
|
|
3246
|
+
for (const record of conversion.records) lines.push(JSON.stringify(record));
|
|
3247
|
+
rows += conversion.rows;
|
|
3248
|
+
unnamedModel += conversion.unnamedModel;
|
|
3249
|
+
disagreements += conversion.modelDisagreements;
|
|
3250
|
+
noTokens += conversion.noTokens;
|
|
3251
|
+
cacheFlagged += conversion.cacheFlagged;
|
|
3252
|
+
unparseable += conversion.unparseable;
|
|
3253
|
+
}
|
|
3254
|
+
|
|
3255
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o');
|
|
3256
|
+
if (out !== undefined) {
|
|
3257
|
+
await writeFile(out, lines.join('\n') + (lines.length > 0 ? '\n' : ''), 'utf8');
|
|
3258
|
+
console.error(t.fromHelicone.written(out));
|
|
3259
|
+
} else {
|
|
3260
|
+
for (const line of lines) console.log(line);
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3263
|
+
console.error(t.fromHelicone.summary(files.length, rows));
|
|
3264
|
+
if (unnamedModel > 0) console.error(t.fromHelicone.unnamedModel(unnamedModel));
|
|
3265
|
+
if (disagreements > 0) console.error(t.fromHelicone.disagreements(disagreements));
|
|
3266
|
+
if (noTokens > 0) console.error(t.fromHelicone.noTokens(noTokens));
|
|
3267
|
+
if (cacheFlagged > 0) console.error(t.fromHelicone.cacheFlagged(cacheFlagged));
|
|
3268
|
+
// Said on every run that produced records, not only when something is
|
|
3269
|
+
// missing: the absence is a property of the format, and a reader who has
|
|
3270
|
+
// just converted a month needs to know which questions this log cannot
|
|
3271
|
+
// answer before they go looking for the answers.
|
|
3272
|
+
if (rows > 0) console.error(t.fromHelicone.noSessions());
|
|
3273
|
+
if (unparseable > 0) console.error(t.fromHelicone.unparseable(unparseable));
|
|
3274
|
+
}
|
|
3275
|
+
|
|
3085
3276
|
/**
|
|
3086
3277
|
* `trazum position <usage.jsonl>` — where the month stands, measured.
|
|
3087
3278
|
*
|
|
@@ -5682,6 +5873,7 @@ async function commandCheck(
|
|
|
5682
5873
|
verdicts: [verdict],
|
|
5683
5874
|
level,
|
|
5684
5875
|
tokenSource: counter.source,
|
|
5876
|
+
band: bandFor(prompt),
|
|
5685
5877
|
truncated: false,
|
|
5686
5878
|
t,
|
|
5687
5879
|
}),
|
|
@@ -5817,12 +6009,18 @@ async function checkEmbedded(
|
|
|
5817
6009
|
const failures = verdicts.filter(isOverBudget);
|
|
5818
6010
|
const ok = failures.length === 0 && extraction.declined.length === 0;
|
|
5819
6011
|
|
|
6012
|
+
// Widest across the prompts this report covers, never an average: a run over
|
|
6013
|
+
// prose and a CSV covers text the estimator is 6% and 33% out on, and a
|
|
6014
|
+
// figure between them would describe neither.
|
|
6015
|
+
const band = widestBand(extraction.prompts.map((prompt) => prompt.text));
|
|
6016
|
+
|
|
5820
6017
|
await writeMarkdown(args, () =>
|
|
5821
6018
|
renderCheckMarkdown({
|
|
5822
6019
|
target: path,
|
|
5823
6020
|
verdicts,
|
|
5824
6021
|
level,
|
|
5825
6022
|
tokenSource: counter.source,
|
|
6023
|
+
band,
|
|
5826
6024
|
truncated: false,
|
|
5827
6025
|
t,
|
|
5828
6026
|
}),
|
|
@@ -9643,12 +9841,102 @@ function reportProfileGaps(
|
|
|
9643
9841
|
* exactly as `prune` does. A command that can spend somebody's money without
|
|
9644
9842
|
* telling them first is a command they stop trusting.
|
|
9645
9843
|
*/
|
|
9844
|
+
/**
|
|
9845
|
+
* The routing measurement, without the text that produced it.
|
|
9846
|
+
*
|
|
9847
|
+
* `route --json` printed `{ slice, evaluation }` — the whole `EvalReport`,
|
|
9848
|
+
* whose `cases[]` carries `input` (the caller's own case text), `baseline[0]`,
|
|
9849
|
+
* `baseline[1]` and `optimized` (three model answers). It carried no
|
|
9850
|
+
* `schemaVersion` either.
|
|
9851
|
+
*
|
|
9852
|
+
* `docs/json-output.md` opens by promising that **nothing here carries a
|
|
9853
|
+
* session key or prompt text**, and this is the document most likely to be
|
|
9854
|
+
* piped into a dashboard, pasted into a ticket or dropped into a browser. A
|
|
9855
|
+
* tool whose headline is that prompts never leave the machine cannot have its
|
|
9856
|
+
* most portable artefact be the one that carries them.
|
|
9857
|
+
*
|
|
9858
|
+
* The per-case *shape* survives: the two similarity scores are the evidence
|
|
9859
|
+
* behind the verdict, and a reader checking a borderline call needs them. Only
|
|
9860
|
+
* the strings go. Anyone who wants the answers has the human-readable output,
|
|
9861
|
+
* which never left the terminal.
|
|
9862
|
+
*
|
|
9863
|
+
* A named function rather than an inline object so the stripping is a thing a
|
|
9864
|
+
* test can call without spending a provider call to reach it.
|
|
9865
|
+
*/
|
|
9866
|
+
export function routeDocument(slice: unknown, result: EvalReport): unknown {
|
|
9867
|
+
return {
|
|
9868
|
+
schemaVersion: 1,
|
|
9869
|
+
slice,
|
|
9870
|
+
evaluation: {
|
|
9871
|
+
provider: result.provider,
|
|
9872
|
+
model: result.model,
|
|
9873
|
+
candidateModel: result.candidateModel,
|
|
9874
|
+
verdict: result.verdict,
|
|
9875
|
+
selfAgreement: result.selfAgreement,
|
|
9876
|
+
crossAgreement: result.crossAgreement,
|
|
9877
|
+
callsMade: result.callsMade,
|
|
9878
|
+
cases: result.cases.map((one: EvalReport['cases'][number]) => ({
|
|
9879
|
+
selfSimilarity: one.selfSimilarity,
|
|
9880
|
+
crossSimilarity: one.crossSimilarity,
|
|
9881
|
+
})),
|
|
9882
|
+
},
|
|
9883
|
+
};
|
|
9884
|
+
}
|
|
9885
|
+
|
|
9886
|
+
/**
|
|
9887
|
+
* The example-pruning measurement, likewise.
|
|
9888
|
+
*
|
|
9889
|
+
* `ExampleContribution.text` is *"the example itself, so a report can quote its
|
|
9890
|
+
* first line"* — true of the terminal rendering and wrong of a document that
|
|
9891
|
+
* promises to carry no prompt text. A few-shot example **is** prompt text, and
|
|
9892
|
+
* this was the second command emitting it.
|
|
9893
|
+
*
|
|
9894
|
+
* `index` and `tokens` say which example without quoting it, which is what a
|
|
9895
|
+
* machine reading this needs: it already has the prompt.
|
|
9896
|
+
*/
|
|
9897
|
+
export function pruneDocument(report: PruneReport): unknown {
|
|
9898
|
+
return {
|
|
9899
|
+
schemaVersion: 1,
|
|
9900
|
+
provider: report.provider,
|
|
9901
|
+
model: report.model,
|
|
9902
|
+
selfAgreement: report.selfAgreement,
|
|
9903
|
+
recoverableTokens: report.recoverableTokens,
|
|
9904
|
+
callsMade: report.callsMade,
|
|
9905
|
+
contributions: report.contributions.map((one: PruneReport['contributions'][number]) => ({
|
|
9906
|
+
index: one.index,
|
|
9907
|
+
tokens: one.tokens,
|
|
9908
|
+
agreementWithout: one.agreementWithout,
|
|
9909
|
+
verdict: one.verdict,
|
|
9910
|
+
})),
|
|
9911
|
+
};
|
|
9912
|
+
}
|
|
9913
|
+
|
|
9646
9914
|
async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
|
|
9915
|
+
/**
|
|
9916
|
+
* Under `--json`, stdout carries the document and nothing else.
|
|
9917
|
+
*
|
|
9918
|
+
* It carried the human preamble too — the slice it picked, the call count,
|
|
9919
|
+
* the "nothing has been spent yet" line — straight into the same stream as
|
|
9920
|
+
* the JSON, so `route --json > verdict.json` wrote a file no parser would
|
|
9921
|
+
* read. Found by running the command and feeding the file to the bridge
|
|
9922
|
+
* that reads it, which is the one thing the pure-function guards could not
|
|
9923
|
+
* do: `interchange.test.js` names `route` among the commands it cannot
|
|
9924
|
+
* drive, so nothing checked that its `--json` output was a document.
|
|
9925
|
+
*
|
|
9926
|
+
* The lines go to stderr rather than away. A reader running this in a
|
|
9927
|
+
* terminal still needs to see which slice was picked and what it will cost
|
|
9928
|
+
* before it costs it; a pipe needs the document. stderr gives both.
|
|
9929
|
+
*/
|
|
9930
|
+
const asJson = boolFlag(args, 'json');
|
|
9931
|
+
const say = (line = ''): void => {
|
|
9932
|
+
if (asJson) console.error(line);
|
|
9933
|
+
else console.log(line);
|
|
9934
|
+
};
|
|
9647
9935
|
const path = args.positional[0];
|
|
9648
9936
|
if (path === undefined) {
|
|
9649
|
-
|
|
9650
|
-
|
|
9651
|
-
|
|
9937
|
+
say();
|
|
9938
|
+
say(c.dim(wrap(t.route.noTarget(), 74, ' ')));
|
|
9939
|
+
say();
|
|
9652
9940
|
return;
|
|
9653
9941
|
}
|
|
9654
9942
|
|
|
@@ -9671,18 +9959,18 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9671
9959
|
const available = report.byLabel
|
|
9672
9960
|
.map((r) => (r.label === UNLABELLED ? t.profile.unlabelled() : r.label))
|
|
9673
9961
|
.join(', ');
|
|
9674
|
-
|
|
9675
|
-
|
|
9676
|
-
|
|
9962
|
+
say();
|
|
9963
|
+
say(c.dim(wrap(t.route.labelNotFound(wanted, available), 74, ' ')));
|
|
9964
|
+
say();
|
|
9677
9965
|
return;
|
|
9678
9966
|
}
|
|
9679
9967
|
const slice = levers.slices.find(
|
|
9680
9968
|
(s) => s.route !== null && (wanted === undefined || s.label === wanted),
|
|
9681
9969
|
);
|
|
9682
9970
|
if (!slice?.route) {
|
|
9683
|
-
|
|
9684
|
-
|
|
9685
|
-
|
|
9971
|
+
say();
|
|
9972
|
+
say(c.dim(wrap(t.route.noRoute(), 74, ' ')));
|
|
9973
|
+
say();
|
|
9686
9974
|
return;
|
|
9687
9975
|
}
|
|
9688
9976
|
|
|
@@ -9706,8 +9994,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9706
9994
|
|
|
9707
9995
|
const label = slice.label === UNLABELLED ? t.profile.unlabelled() : slice.label;
|
|
9708
9996
|
const worth = formatUsd(slice.route.savingUsd);
|
|
9709
|
-
|
|
9710
|
-
|
|
9997
|
+
say();
|
|
9998
|
+
say(
|
|
9711
9999
|
` ${c.bold(t.route.picked(label, slice.modelName, slice.route.candidate.displayName, worth, `${(slice.shareOfBill * 100).toFixed(1)}%`))}`,
|
|
9712
10000
|
);
|
|
9713
10001
|
/**
|
|
@@ -9720,20 +10008,20 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9720
10008
|
* counts, so it is stated rather than guessed at.
|
|
9721
10009
|
*/
|
|
9722
10010
|
if (slice.label === UNLABELLED) {
|
|
9723
|
-
|
|
10011
|
+
say(` ${c.yellow('!')} ${c.dim(wrap(t.route.unlabelledSlice(), 74, ' '))}`);
|
|
9724
10012
|
}
|
|
9725
|
-
|
|
9726
|
-
|
|
10013
|
+
say();
|
|
10014
|
+
say(
|
|
9727
10015
|
` ${c.dim(wrap(t.route.willSpend(inputs.length * 3, provider.model, candidate.model), 74, ' '))}`,
|
|
9728
10016
|
);
|
|
9729
10017
|
|
|
9730
10018
|
if (!boolFlag(args, 'yes')) {
|
|
9731
|
-
|
|
9732
|
-
|
|
10019
|
+
say(` ${c.dim(t.route.dryRun())}`);
|
|
10020
|
+
say();
|
|
9733
10021
|
return;
|
|
9734
10022
|
}
|
|
9735
10023
|
|
|
9736
|
-
|
|
10024
|
+
say(` ${c.dim(t.route.running(inputs.length))}`);
|
|
9737
10025
|
// Same prompt on both sides. The axis under test is the model, and passing the
|
|
9738
10026
|
// prompt twice is what says so at the call site.
|
|
9739
10027
|
const result = await evaluate(prompt, prompt, inputs, provider, {
|
|
@@ -9742,22 +10030,22 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9742
10030
|
});
|
|
9743
10031
|
|
|
9744
10032
|
if (boolFlag(args, 'json')) {
|
|
9745
|
-
console.log(JSON.stringify(
|
|
10033
|
+
console.log(JSON.stringify(routeDocument(slice, result), null, 2));
|
|
9746
10034
|
return;
|
|
9747
10035
|
}
|
|
9748
10036
|
|
|
9749
10037
|
const asPct = (v: number): string => `${(v * 100).toFixed(0)}%`;
|
|
9750
|
-
|
|
9751
|
-
|
|
10038
|
+
say();
|
|
10039
|
+
say(
|
|
9752
10040
|
` ${c.dim(wrap(t.route.agreement(asPct(result.crossAgreement), asPct(result.selfAgreement)), 74, ' '))}`,
|
|
9753
10041
|
);
|
|
9754
|
-
|
|
10042
|
+
say();
|
|
9755
10043
|
if (result.verdict === 'inconclusive') {
|
|
9756
|
-
|
|
10044
|
+
say(` ${c.bold(wrap(t.route.inconclusive(), 74, ' '))}`);
|
|
9757
10045
|
} else if (result.verdict === 'diverges') {
|
|
9758
|
-
|
|
10046
|
+
say(` ${c.yellow('!')} ${c.bold(wrap(t.route.diverges(worth), 74, ' '))}`);
|
|
9759
10047
|
} else {
|
|
9760
|
-
|
|
10048
|
+
say(` ${c.green('✓')} ${c.bold(wrap(t.route.holds(worth), 74, ' '))}`);
|
|
9761
10049
|
}
|
|
9762
10050
|
/**
|
|
9763
10051
|
* Printed on every verdict including the good one. Agreement is not
|
|
@@ -9765,8 +10053,8 @@ async function commandRoute(args: Args, pricing: PricingCatalogue, t: CliMessage
|
|
|
9765
10053
|
* ever right, and a green tick that let somebody forget that would be the tool
|
|
9766
10054
|
* overstating what it knows.
|
|
9767
10055
|
*/
|
|
9768
|
-
|
|
9769
|
-
|
|
10056
|
+
say(` ${c.dim(wrap(t.route.yours(), 74, ' '))}`);
|
|
10057
|
+
say();
|
|
9770
10058
|
}
|
|
9771
10059
|
|
|
9772
10060
|
/**
|
|
@@ -9933,6 +10221,14 @@ interface PromptScan {
|
|
|
9933
10221
|
truncated: boolean;
|
|
9934
10222
|
/** The extensions actually walked, so an error can name them. */
|
|
9935
10223
|
extensions: string[];
|
|
10224
|
+
/**
|
|
10225
|
+
* The widest measured band across the prompts this scan judged.
|
|
10226
|
+
*
|
|
10227
|
+
* Computed here because this is where the texts are, and widest rather than
|
|
10228
|
+
* averaged: a scan over a prose prompt and a CSV covers text the estimator
|
|
10229
|
+
* is 6% and 33% out on, and a figure between them describes neither.
|
|
10230
|
+
*/
|
|
10231
|
+
band: number;
|
|
9936
10232
|
}
|
|
9937
10233
|
|
|
9938
10234
|
/**
|
|
@@ -10012,6 +10308,9 @@ async function scanPrompts(
|
|
|
10012
10308
|
const verdicts: FileVerdict[] = [];
|
|
10013
10309
|
const declined: Array<{ path: string; line: number; detail: string }> = [];
|
|
10014
10310
|
|
|
10311
|
+
// Every prompt this run judged, so the report's band is the widest of them
|
|
10312
|
+
// rather than one figure asserted over text of several kinds.
|
|
10313
|
+
const judgedTexts: string[] = [];
|
|
10015
10314
|
for (const relativePath of files) {
|
|
10016
10315
|
const text = capInput(
|
|
10017
10316
|
await readFile(join(root, relativePath), 'utf8'),
|
|
@@ -10019,6 +10318,7 @@ async function scanPrompts(
|
|
|
10019
10318
|
maxInputFlag(args, t),
|
|
10020
10319
|
t,
|
|
10021
10320
|
);
|
|
10321
|
+
judgedTexts.push(text);
|
|
10022
10322
|
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
10023
10323
|
// `prompts/**` has to be matched against the path including the root the
|
|
10024
10324
|
// user passed — not against the name relative to it.
|
|
@@ -10052,7 +10352,7 @@ async function scanPrompts(
|
|
|
10052
10352
|
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
10053
10353
|
}
|
|
10054
10354
|
|
|
10055
|
-
return { verdicts, declined, truncated, extensions };
|
|
10355
|
+
return { verdicts, declined, truncated, extensions, band: widestBand(judgedTexts) };
|
|
10056
10356
|
}
|
|
10057
10357
|
|
|
10058
10358
|
async function checkDirectory(
|
|
@@ -10084,7 +10384,7 @@ async function checkDirectory(
|
|
|
10084
10384
|
only = checkable;
|
|
10085
10385
|
}
|
|
10086
10386
|
|
|
10087
|
-
const { verdicts, declined, truncated, extensions } = await scanPrompts(
|
|
10387
|
+
const { verdicts, declined, truncated, extensions, band } = await scanPrompts(
|
|
10088
10388
|
root,
|
|
10089
10389
|
args,
|
|
10090
10390
|
flagBudget,
|
|
@@ -10162,6 +10462,7 @@ async function checkDirectory(
|
|
|
10162
10462
|
renderCheckMarkdown({
|
|
10163
10463
|
target: root,
|
|
10164
10464
|
verdicts,
|
|
10465
|
+
band,
|
|
10165
10466
|
level,
|
|
10166
10467
|
tokenSource: counter.source,
|
|
10167
10468
|
truncated,
|
|
@@ -10342,6 +10643,13 @@ function joinPosix(root: string, relativePath: string): string {
|
|
|
10342
10643
|
* never "delete this", and nothing here edits the prompt.
|
|
10343
10644
|
*/
|
|
10344
10645
|
async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
10646
|
+
// Same as `route`: under `--json` stdout is the document, and the estimate
|
|
10647
|
+
// a reader has to see before spending goes to stderr rather than away.
|
|
10648
|
+
const asJson = boolFlag(args, 'json');
|
|
10649
|
+
const say = (line = ''): void => {
|
|
10650
|
+
if (asJson) console.error(line);
|
|
10651
|
+
else console.log(line);
|
|
10652
|
+
};
|
|
10345
10653
|
const prompt = await readInput(args.positional[0], t, maxInputFlag(args, t));
|
|
10346
10654
|
|
|
10347
10655
|
const casesPath = stringFlag(args, 'cases');
|
|
@@ -10356,11 +10664,11 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10356
10664
|
|
|
10357
10665
|
// Printed before the key is even looked up, so somebody weighing it up does not
|
|
10358
10666
|
// need a configured provider to see the number.
|
|
10359
|
-
|
|
10360
|
-
|
|
10667
|
+
say();
|
|
10668
|
+
say(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
|
|
10361
10669
|
|
|
10362
10670
|
if (!boolFlag(args, 'yes')) {
|
|
10363
|
-
|
|
10671
|
+
say(c.yellow(` ${t.prune.needsConsent()}`));
|
|
10364
10672
|
return;
|
|
10365
10673
|
}
|
|
10366
10674
|
|
|
@@ -10372,15 +10680,15 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10372
10680
|
});
|
|
10373
10681
|
|
|
10374
10682
|
if (boolFlag(args, 'json')) {
|
|
10375
|
-
console.log(JSON.stringify(report, null, 2));
|
|
10683
|
+
console.log(JSON.stringify(pruneDocument(report), null, 2));
|
|
10376
10684
|
return;
|
|
10377
10685
|
}
|
|
10378
10686
|
|
|
10379
10687
|
const pct = (value: number): string => `${(value * 100).toFixed(0)}%`;
|
|
10380
|
-
|
|
10381
|
-
|
|
10382
|
-
|
|
10383
|
-
|
|
10688
|
+
say();
|
|
10689
|
+
say(sectionHeading(t.prune.heading(provider.model)));
|
|
10690
|
+
say(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
|
|
10691
|
+
say();
|
|
10384
10692
|
|
|
10385
10693
|
for (const contribution of report.contributions) {
|
|
10386
10694
|
/**
|
|
@@ -10399,7 +10707,7 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10399
10707
|
? t.prune.verdictNeeded()
|
|
10400
10708
|
: t.prune.verdictRecoverable();
|
|
10401
10709
|
|
|
10402
|
-
|
|
10710
|
+
say(
|
|
10403
10711
|
` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
|
|
10404
10712
|
+ ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`,
|
|
10405
10713
|
);
|
|
@@ -10412,16 +10720,16 @@ async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
|
10412
10720
|
*/
|
|
10413
10721
|
const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
|
|
10414
10722
|
const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
|
|
10415
|
-
|
|
10723
|
+
say(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
|
|
10416
10724
|
}
|
|
10417
10725
|
|
|
10418
|
-
|
|
10726
|
+
say();
|
|
10419
10727
|
if (report.recoverableTokens > 0) {
|
|
10420
|
-
|
|
10728
|
+
say(` ${t.prune.recoverable(report.recoverableTokens)}`);
|
|
10421
10729
|
}
|
|
10422
|
-
|
|
10423
|
-
|
|
10424
|
-
|
|
10730
|
+
say(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
|
|
10731
|
+
say();
|
|
10732
|
+
say(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
10425
10733
|
}
|
|
10426
10734
|
|
|
10427
10735
|
async function commandEval(
|
|
@@ -10740,6 +11048,18 @@ const DOCTOR_LIST_LIMIT = 8;
|
|
|
10740
11048
|
const BLAME_DEFAULT_LIMIT = 20;
|
|
10741
11049
|
const BLAME_MAX_LIMIT = 500;
|
|
10742
11050
|
|
|
11051
|
+
/**
|
|
11052
|
+
* The band for a report covering several prompts.
|
|
11053
|
+
*
|
|
11054
|
+
* The widest of them, and the empty case is the widest published band rather
|
|
11055
|
+
* than a narrow default: a report over nothing has measured nothing, and
|
|
11056
|
+
* guessing narrow there is the one direction that produces a false claim.
|
|
11057
|
+
*/
|
|
11058
|
+
function widestBand(texts: readonly string[]): number {
|
|
11059
|
+
if (texts.length === 0) return ESTIMATE_ERROR_BAND_PCT;
|
|
11060
|
+
return Math.max(...texts.map((text) => bandFor(text)));
|
|
11061
|
+
}
|
|
11062
|
+
|
|
10743
11063
|
interface BlameRow {
|
|
10744
11064
|
revision: Revision;
|
|
10745
11065
|
/** `null` when the file did not exist at that commit, or held no marked prompt. */
|
|
@@ -10838,17 +11158,35 @@ async function commandBlame(
|
|
|
10838
11158
|
const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
|
|
10839
11159
|
const truncatedHistory = revisions.length > limit;
|
|
10840
11160
|
|
|
11161
|
+
/*
|
|
11162
|
+
The band of this file's own text, read once and handed to both renderers.
|
|
11163
|
+
|
|
11164
|
+
`blame` walks one file across revisions, so the kind of text is a property
|
|
11165
|
+
of the file, and today's content is the closest thing to it that exists
|
|
11166
|
+
here. A file that is gone from the working tree — renamed away, or deleted
|
|
11167
|
+
in the commit being blamed — falls back to the widest published band, which
|
|
11168
|
+
is exactly what that constant is for: a caller that cannot see the text can
|
|
11169
|
+
only guess in the safe direction.
|
|
11170
|
+
*/
|
|
11171
|
+
let band = ESTIMATE_ERROR_BAND_PCT;
|
|
11172
|
+
try {
|
|
11173
|
+
band = bandFor(await readFile(resolvePath(process.cwd(), target), 'utf8'));
|
|
11174
|
+
} catch {
|
|
11175
|
+
// Not on disk now. The wide band stands and says so by being wide.
|
|
11176
|
+
}
|
|
11177
|
+
|
|
10841
11178
|
await writeMarkdown(args, () =>
|
|
10842
11179
|
renderBlameMarkdown({
|
|
10843
11180
|
repoPath,
|
|
10844
11181
|
rows: shown,
|
|
10845
11182
|
truncated: truncatedHistory,
|
|
10846
11183
|
netCost: netCostOf(shown, args, config, pricing, t),
|
|
11184
|
+
band,
|
|
10847
11185
|
t,
|
|
10848
11186
|
}),
|
|
10849
11187
|
);
|
|
10850
11188
|
|
|
10851
|
-
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
|
|
11189
|
+
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory, band });
|
|
10852
11190
|
}
|
|
10853
11191
|
|
|
10854
11192
|
/**
|
|
@@ -10905,6 +11243,15 @@ function printBlame(
|
|
|
10905
11243
|
pricing: PricingCatalogue;
|
|
10906
11244
|
t: CliMessages;
|
|
10907
11245
|
truncated: boolean;
|
|
11246
|
+
/**
|
|
11247
|
+
* The band of this file's own text, computed once by the caller.
|
|
11248
|
+
*
|
|
11249
|
+
* `blame` walks one file across revisions, so the kind of text is a
|
|
11250
|
+
* property of the file rather than of any one revision. The caller has
|
|
11251
|
+
* today's content and this renderer does not, which is why it arrives
|
|
11252
|
+
* here rather than being read again.
|
|
11253
|
+
*/
|
|
11254
|
+
band: number;
|
|
10908
11255
|
},
|
|
10909
11256
|
): void {
|
|
10910
11257
|
const { repoPath, args, config, pricing, t, truncated } = context;
|
|
@@ -11039,7 +11386,7 @@ function printBlame(
|
|
|
11039
11386
|
);
|
|
11040
11387
|
}
|
|
11041
11388
|
|
|
11042
|
-
console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
|
|
11389
|
+
console.log(`\n${c.dim(t.blame.estimateNote(context.band))}\n`);
|
|
11043
11390
|
}
|
|
11044
11391
|
|
|
11045
11392
|
// --------------------------------------------------------------------------
|
|
@@ -11469,6 +11816,12 @@ async function main(): Promise<void> {
|
|
|
11469
11816
|
case 'from-otel':
|
|
11470
11817
|
await commandFromOtel(args, t);
|
|
11471
11818
|
break;
|
|
11819
|
+
case 'from-litellm':
|
|
11820
|
+
await commandFromLiteLlm(args, t);
|
|
11821
|
+
break;
|
|
11822
|
+
case 'from-helicone':
|
|
11823
|
+
await commandFromHelicone(args, t);
|
|
11824
|
+
break;
|
|
11472
11825
|
case 'switch':
|
|
11473
11826
|
await commandSwitch(args, pricing, t);
|
|
11474
11827
|
break;
|