@trazum/cli 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +134 -0
- package/dist/git.d.ts +81 -0
- package/dist/git.d.ts.map +1 -0
- package/dist/git.js +225 -0
- package/dist/git.js.map +1 -0
- package/dist/i18n/en.d.ts +10 -0
- package/dist/i18n/en.d.ts.map +1 -0
- package/dist/i18n/en.js +694 -0
- package/dist/i18n/en.js.map +1 -0
- package/dist/i18n/es.d.ts +4 -0
- package/dist/i18n/es.d.ts.map +1 -0
- package/dist/i18n/es.js +694 -0
- package/dist/i18n/es.js.map +1 -0
- package/dist/i18n/index.d.ts +34 -0
- package/dist/i18n/index.d.ts.map +1 -0
- package/dist/i18n/index.js +46 -0
- package/dist/i18n/index.js.map +1 -0
- package/dist/i18n/types.d.ts +363 -0
- package/dist/i18n/types.d.ts.map +1 -0
- package/dist/i18n/types.js +2 -0
- package/dist/i18n/types.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2824 -0
- package/dist/index.js.map +1 -0
- package/dist/markdown.d.ts +246 -0
- package/dist/markdown.d.ts.map +1 -0
- package/dist/markdown.js +492 -0
- package/dist/markdown.js.map +1 -0
- package/dist/suggest-cache.d.ts +119 -0
- package/dist/suggest-cache.d.ts.map +1 -0
- package/dist/suggest-cache.js +225 -0
- package/dist/suggest-cache.js.map +1 -0
- package/package.json +49 -0
- package/src/git.ts +294 -0
- package/src/i18n/en.ts +825 -0
- package/src/i18n/es.ts +838 -0
- package/src/i18n/index.ts +57 -0
- package/src/i18n/types.ts +372 -0
- package/src/index.ts +3873 -0
- package/src/markdown.ts +717 -0
- package/src/suggest-cache.ts +268 -0
package/src/index.ts
ADDED
|
@@ -0,0 +1,3873 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFile, stat, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join, resolve as resolvePath } from 'node:path';
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
BASELINE_FILENAME,
|
|
7
|
+
BASELINE_VERSION,
|
|
8
|
+
MAX_BASELINE_BYTES,
|
|
9
|
+
breaches,
|
|
10
|
+
compareToBaseline,
|
|
11
|
+
formatBaseline,
|
|
12
|
+
moneyIsComparable,
|
|
13
|
+
parseBaseline,
|
|
14
|
+
DEFAULT_USAGE,
|
|
15
|
+
LOCALES,
|
|
16
|
+
PRICING_LAST_REVIEWED,
|
|
17
|
+
reviewAgeDays,
|
|
18
|
+
RULES,
|
|
19
|
+
comparePrompts,
|
|
20
|
+
countTokensAnthropic,
|
|
21
|
+
getModel,
|
|
22
|
+
applyRewrites,
|
|
23
|
+
computeSavings,
|
|
24
|
+
profilePrompt,
|
|
25
|
+
toPromptfoo,
|
|
26
|
+
PHRASE_LANGUAGES,
|
|
27
|
+
estimateTokens,
|
|
28
|
+
formatUsd,
|
|
29
|
+
formatSignedUsd,
|
|
30
|
+
getMessages,
|
|
31
|
+
listModels,
|
|
32
|
+
nearestName,
|
|
33
|
+
optimize,
|
|
34
|
+
toOtlpMetrics,
|
|
35
|
+
providerFromEnv,
|
|
36
|
+
reorderForCache,
|
|
37
|
+
sharedPrefixes,
|
|
38
|
+
cacheableMinimum,
|
|
39
|
+
findExamples,
|
|
40
|
+
plannedCalls,
|
|
41
|
+
pruneExamples,
|
|
42
|
+
extractPrompts,
|
|
43
|
+
promptId,
|
|
44
|
+
hasMarker,
|
|
45
|
+
SOURCE_EXTENSIONS,
|
|
46
|
+
detectFromSource,
|
|
47
|
+
evaluate,
|
|
48
|
+
refineWithLlm,
|
|
49
|
+
rejectionText,
|
|
50
|
+
suggestRewrites,
|
|
51
|
+
reviewExamples,
|
|
52
|
+
withExactTokenCounts,
|
|
53
|
+
} from '@trazum/core';
|
|
54
|
+
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
55
|
+
import type {
|
|
56
|
+
BaselineBreach,
|
|
57
|
+
BaselineChange,
|
|
58
|
+
BaselineComparison,
|
|
59
|
+
BaselineDocument,
|
|
60
|
+
Advisory,
|
|
61
|
+
ExampleReview,
|
|
62
|
+
PromptComparison,
|
|
63
|
+
ReorderResult,
|
|
64
|
+
ExtractedPrompt,
|
|
65
|
+
DeclinedPrompt,
|
|
66
|
+
Locale,
|
|
67
|
+
OptimizationResult,
|
|
68
|
+
RuleId,
|
|
69
|
+
RejectedReason,
|
|
70
|
+
PromptProfile,
|
|
71
|
+
RuleLevel,
|
|
72
|
+
SharedPrefix,
|
|
73
|
+
SuggestResult,
|
|
74
|
+
UsageProfile,
|
|
75
|
+
} from '@trazum/core';
|
|
76
|
+
// Everything that reads the filesystem, on its own entry point so the web
|
|
77
|
+
// bundle cannot reach it. See packages/core/src/node.ts.
|
|
78
|
+
import {
|
|
79
|
+
CONFIG_FILENAME,
|
|
80
|
+
DEFAULT_EXTENSIONS,
|
|
81
|
+
budgetFor,
|
|
82
|
+
BUNDLED_CATALOGUE,
|
|
83
|
+
SAFE_FETCH_INIT,
|
|
84
|
+
applyPricingOverlay,
|
|
85
|
+
catalogueFromOverlay,
|
|
86
|
+
checkedEndpoint,
|
|
87
|
+
openrouterOverlay,
|
|
88
|
+
detectHost,
|
|
89
|
+
loadConfig,
|
|
90
|
+
walkPrompts,
|
|
91
|
+
} from '@trazum/core/node';
|
|
92
|
+
import type { HostEnvironment, PricingCatalogue, ResolvedBudget, TrazumConfig } from '@trazum/core/node';
|
|
93
|
+
|
|
94
|
+
import {
|
|
95
|
+
contentAt,
|
|
96
|
+
gitAvailable,
|
|
97
|
+
namesByRevision,
|
|
98
|
+
pathInRepository,
|
|
99
|
+
repositoryRoot,
|
|
100
|
+
revisionsFor,
|
|
101
|
+
} from './git.js';
|
|
102
|
+
import type { Revision } from './git.js';
|
|
103
|
+
import { detectLocale, getCliMessages } from './i18n/index.js';
|
|
104
|
+
import {
|
|
105
|
+
MAX_SUMMARY_CHARS,
|
|
106
|
+
fitWithin,
|
|
107
|
+
renderBlameMarkdown,
|
|
108
|
+
renderCheckMarkdown,
|
|
109
|
+
renderDiffMarkdown,
|
|
110
|
+
renderRankMarkdown,
|
|
111
|
+
} from './markdown.js';
|
|
112
|
+
import type { CliMessages } from './i18n/index.js';
|
|
113
|
+
|
|
114
|
+
// --------------------------------------------------------------------------
|
|
115
|
+
// Presentation
|
|
116
|
+
// --------------------------------------------------------------------------
|
|
117
|
+
|
|
118
|
+
const useColor = process.stdout.isTTY && !process.env.NO_COLOR;
|
|
119
|
+
const c = {
|
|
120
|
+
bold: (s: string) => (useColor ? `\u001b[1m${s}\u001b[22m` : s),
|
|
121
|
+
dim: (s: string) => (useColor ? `\u001b[2m${s}\u001b[22m` : s),
|
|
122
|
+
green: (s: string) => (useColor ? `\u001b[32m${s}\u001b[39m` : s),
|
|
123
|
+
red: (s: string) => (useColor ? `\u001b[31m${s}\u001b[39m` : s),
|
|
124
|
+
yellow: (s: string) => (useColor ? `\u001b[33m${s}\u001b[39m` : s),
|
|
125
|
+
cyan: (s: string) => (useColor ? `\u001b[36m${s}\u001b[39m` : s),
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
// --------------------------------------------------------------------------
|
|
129
|
+
// Argument parsing
|
|
130
|
+
// --------------------------------------------------------------------------
|
|
131
|
+
|
|
132
|
+
interface Args {
|
|
133
|
+
command: string;
|
|
134
|
+
positional: string[];
|
|
135
|
+
flags: Map<string, string | boolean>;
|
|
136
|
+
/**
|
|
137
|
+
* How a flag was spelled, when that differs from the key it is stored under.
|
|
138
|
+
*
|
|
139
|
+
* Only `--no-x` differs today, and it exists so an error quotes what was
|
|
140
|
+
* actually typed. Telling somebody "unknown option --nonsense" when they
|
|
141
|
+
* wrote `--no-nonsense` sends them looking for a flag they never used.
|
|
142
|
+
*/
|
|
143
|
+
asTyped: Map<string, string>;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const VALUE_FLAGS = new Set([
|
|
147
|
+
'level',
|
|
148
|
+
'model',
|
|
149
|
+
'calls',
|
|
150
|
+
'output-tokens',
|
|
151
|
+
'cache-hit-rate',
|
|
152
|
+
'disable',
|
|
153
|
+
'max-tokens',
|
|
154
|
+
'cases',
|
|
155
|
+
'concurrency',
|
|
156
|
+
'max-growth',
|
|
157
|
+
'export',
|
|
158
|
+
'limit',
|
|
159
|
+
'locale',
|
|
160
|
+
'config',
|
|
161
|
+
'markdown-out',
|
|
162
|
+
'otlp-out',
|
|
163
|
+
'pricing',
|
|
164
|
+
'prompt',
|
|
165
|
+
'out',
|
|
166
|
+
'o',
|
|
167
|
+
]);
|
|
168
|
+
|
|
169
|
+
function parseArgs(argv: string[], t: CliMessages): Args {
|
|
170
|
+
const flags = new Map<string, string | boolean>();
|
|
171
|
+
const asTyped = new Map<string, string>();
|
|
172
|
+
const positional: string[] = [];
|
|
173
|
+
|
|
174
|
+
for (let i = 0; i < argv.length; i++) {
|
|
175
|
+
const arg = argv[i]!;
|
|
176
|
+
// The POSIX escape: everything after `--` is a path, whatever it looks
|
|
177
|
+
// like. Without it there is no way to name a file called `-x.txt` or
|
|
178
|
+
// `--output=…` on the command line at all — the parser sees a flag and
|
|
179
|
+
// refuses before the path reaches the code that knows what to do with it.
|
|
180
|
+
if (arg === '--') {
|
|
181
|
+
positional.push(...argv.slice(i + 1));
|
|
182
|
+
break;
|
|
183
|
+
}
|
|
184
|
+
if (!arg.startsWith('-') || arg === '-') {
|
|
185
|
+
positional.push(arg);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const typed = arg.replace(/^--?/, '');
|
|
189
|
+
let name = typed;
|
|
190
|
+
|
|
191
|
+
// `--no-batch` stores `batch: false`. This exists because a config file can
|
|
192
|
+
// switch a boolean on, and a setting that cannot be switched back off from
|
|
193
|
+
// the command line is one you have to edit the repository to escape.
|
|
194
|
+
let value: string | boolean = true;
|
|
195
|
+
if (name.startsWith('no-') && !VALUE_FLAGS.has(name)) {
|
|
196
|
+
name = name.slice(3);
|
|
197
|
+
value = false;
|
|
198
|
+
asTyped.set(name, typed);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
if (VALUE_FLAGS.has(name)) {
|
|
202
|
+
if (value === false) throw new Error(t.errors.cannotNegate(name));
|
|
203
|
+
const given = argv[++i];
|
|
204
|
+
if (given === undefined) throw new Error(t.errors.optionNeedsValue(name));
|
|
205
|
+
flags.set(name === 'o' ? 'out' : name, given);
|
|
206
|
+
} else {
|
|
207
|
+
flags.set(name, value);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
return { command: positional[0] ?? '', positional: positional.slice(1), flags, asTyped };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Reads a boolean flag, honouring `--no-` and a project default.
|
|
216
|
+
*
|
|
217
|
+
* `flags.has(name)` is the wrong test once negation exists: `--no-batch` stores
|
|
218
|
+
* the key with the value `false`, and `has` would report it as set.
|
|
219
|
+
*/
|
|
220
|
+
function boolFlag(args: Args, name: string, fallback = false): boolean {
|
|
221
|
+
const raw = args.flags.get(name);
|
|
222
|
+
return typeof raw === 'boolean' ? raw : fallback;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Reads `--locale` before the rest of the parsing, so even a parse error is
|
|
227
|
+
* reported in the language the user asked for.
|
|
228
|
+
*/
|
|
229
|
+
function localeFromArgv(argv: string[]): Locale {
|
|
230
|
+
const index = argv.indexOf('--locale');
|
|
231
|
+
const flag = index >= 0 ? argv[index + 1] : undefined;
|
|
232
|
+
return detectLocale(flag);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function stringFlag(args: Args, name: string): string | undefined {
|
|
236
|
+
const raw = args.flags.get(name);
|
|
237
|
+
return typeof raw === 'string' ? raw : undefined;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function numberFlag(args: Args, name: string, fallback: number, t: CliMessages): number {
|
|
241
|
+
const raw = args.flags.get(name);
|
|
242
|
+
if (raw === undefined || typeof raw === 'boolean') return fallback;
|
|
243
|
+
const value = Number(raw);
|
|
244
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
245
|
+
throw new Error(t.errors.mustBeNonNegative(name, raw));
|
|
246
|
+
}
|
|
247
|
+
return value;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Resolves the rule level: flag, then config, then `safe`.
|
|
252
|
+
*
|
|
253
|
+
* The layering order is the same for every setting in this file — the command
|
|
254
|
+
* line beats the project, and the project beats the built-in default. A config
|
|
255
|
+
* file that could override an explicit flag would make the flag a suggestion.
|
|
256
|
+
*/
|
|
257
|
+
function levelFlag(args: Args, config: TrazumConfig, t: CliMessages): RuleLevel {
|
|
258
|
+
const level = (args.flags.get('level') ?? config.level ?? 'safe') as RuleLevel;
|
|
259
|
+
if (level !== 'safe' && level !== 'aggressive') {
|
|
260
|
+
throw new Error(t.errors.badLevel(String(level)));
|
|
261
|
+
}
|
|
262
|
+
return level;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* Usage profile from flags over config over detection over the built-in default.
|
|
267
|
+
*
|
|
268
|
+
* `detected` is what the source file said — an SDK import, a base URL, a quoted
|
|
269
|
+
* model id. It beats the default because reading the code is better than
|
|
270
|
+
* assuming, and loses to config because being told is better than reading.
|
|
271
|
+
*/
|
|
272
|
+
function usageFrom(
|
|
273
|
+
args: Args,
|
|
274
|
+
config: TrazumConfig,
|
|
275
|
+
t: CliMessages,
|
|
276
|
+
detected?: string,
|
|
277
|
+
): UsageProfile {
|
|
278
|
+
const fromConfig = config.usage ?? {};
|
|
279
|
+
const model = stringFlag(args, 'model') ?? fromConfig.model ?? detected ?? DEFAULT_USAGE.model;
|
|
280
|
+
return {
|
|
281
|
+
model,
|
|
282
|
+
callsPerMonth: numberFlag(
|
|
283
|
+
args,
|
|
284
|
+
'calls',
|
|
285
|
+
fromConfig.callsPerMonth ?? DEFAULT_USAGE.callsPerMonth,
|
|
286
|
+
t,
|
|
287
|
+
),
|
|
288
|
+
avgOutputTokens: numberFlag(
|
|
289
|
+
args,
|
|
290
|
+
'output-tokens',
|
|
291
|
+
fromConfig.avgOutputTokens ?? DEFAULT_USAGE.avgOutputTokens,
|
|
292
|
+
t,
|
|
293
|
+
),
|
|
294
|
+
cacheHitRate: numberFlag(
|
|
295
|
+
args,
|
|
296
|
+
'cache-hit-rate',
|
|
297
|
+
fromConfig.cacheHitRate ?? DEFAULT_USAGE.cacheHitRate,
|
|
298
|
+
t,
|
|
299
|
+
),
|
|
300
|
+
batchEligible: boolFlag(args, 'batch', fromConfig.batchEligible ?? false),
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* Prices for this run: `--pricing` beats the config's overlay, which beats the
|
|
306
|
+
* bundled catalogue — the same layering as every other setting.
|
|
307
|
+
*/
|
|
308
|
+
/**
|
|
309
|
+
* OpenRouter's public catalogue. Overridable for an operator behind a mirror.
|
|
310
|
+
*
|
|
311
|
+
* Not a secret and not a credential: the models endpoint is unauthenticated,
|
|
312
|
+
* which is why this can be a flag rather than a key.
|
|
313
|
+
*/
|
|
314
|
+
const OPENROUTER_MODELS_URL =
|
|
315
|
+
process.env.TRAZUM_OPENROUTER_URL ?? 'https://openrouter.ai/api/v1/models';
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Prices from a live source, and the reasoning for why this is opt-in.
|
|
319
|
+
*
|
|
320
|
+
* The bundled catalogue is a table somebody typed, so it is stale the day after
|
|
321
|
+
* it is written and it only ever covered the providers whoever typed it reached
|
|
322
|
+
* for. `--pricing-live` replaces the price half of it with today's figures for
|
|
323
|
+
* hundreds of models across dozens of providers.
|
|
324
|
+
*
|
|
325
|
+
* **Opt-in, because it is a network call.** Rule 1 of this project is that no
|
|
326
|
+
* feature makes a network call a prerequisite for optimising a prompt. This is
|
|
327
|
+
* the CLI reaching out on request and handing the core a value; the core never
|
|
328
|
+
* fetches anything, which is what keeps `optimize()` free, offline and
|
|
329
|
+
* deterministic.
|
|
330
|
+
*
|
|
331
|
+
* Through `checkedEndpoint` and `SAFE_FETCH_INIT` like every other outbound
|
|
332
|
+
* call here: URL validated before the request, redirects refused, so an
|
|
333
|
+
* endpoint that passes the check cannot answer `302` and send the request
|
|
334
|
+
* somewhere on the metadata network.
|
|
335
|
+
*/
|
|
336
|
+
async function livePricing(source: string, t: CliMessages): Promise<PricingCatalogue> {
|
|
337
|
+
const endpoint = checkedEndpoint(source, { name: 'openrouter' });
|
|
338
|
+
|
|
339
|
+
let payload: unknown;
|
|
340
|
+
try {
|
|
341
|
+
const response = await fetch(endpoint, { ...SAFE_FETCH_INIT, method: 'GET' });
|
|
342
|
+
if (!response.ok) throw new Error(`${response.status} ${response.statusText}`);
|
|
343
|
+
payload = await response.json();
|
|
344
|
+
} catch (error) {
|
|
345
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
346
|
+
throw new Error(t.errors.livePricingFailed(endpoint, detail));
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const known = new Set(BUNDLED_CATALOGUE.models.map((model) => model.id));
|
|
350
|
+
const { overlay, skipped } = openrouterOverlay(payload, {
|
|
351
|
+
knownIds: known,
|
|
352
|
+
lastReviewed: new Date().toISOString().slice(0, 10),
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
const catalogue = applyPricingOverlay(BUNDLED_CATALOGUE, overlay, endpoint);
|
|
356
|
+
|
|
357
|
+
// Said out loud, on stderr so it never lands in `--json`. A price feed that
|
|
358
|
+
// silently dropped a third of its entries would leave somebody wondering why
|
|
359
|
+
// their model is still missing.
|
|
360
|
+
console.error(
|
|
361
|
+
t.pricing.liveLoaded(catalogue.addedModels.length, catalogue.overriddenModels.length, skipped.length),
|
|
362
|
+
);
|
|
363
|
+
|
|
364
|
+
return catalogue;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async function pricingFor(
|
|
368
|
+
args: Args,
|
|
369
|
+
loaded: { pricing: PricingCatalogue },
|
|
370
|
+
t: CliMessages,
|
|
371
|
+
): Promise<PricingCatalogue> {
|
|
372
|
+
const flag = stringFlag(args, 'pricing');
|
|
373
|
+
if (flag) {
|
|
374
|
+
const raw = await readFile(flag, 'utf8');
|
|
375
|
+
return catalogueFromOverlay(raw, flag);
|
|
376
|
+
}
|
|
377
|
+
// A file beats the network: somebody who wrote prices down meant them.
|
|
378
|
+
if (boolFlag(args, 'pricing-live')) return livePricing(OPENROUTER_MODELS_URL, t);
|
|
379
|
+
return loaded.pricing;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Rules to disable: the flag replaces the config list rather than adding to it. */
|
|
383
|
+
function disabledRules(args: Args, config: TrazumConfig): RuleId[] | undefined {
|
|
384
|
+
const flag = stringFlag(args, 'disable');
|
|
385
|
+
if (flag !== undefined) {
|
|
386
|
+
return flag.split(',').map((id) => id.trim()).filter(Boolean) as RuleId[];
|
|
387
|
+
}
|
|
388
|
+
return config.disable;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Flags each command accepts. An unrecognised flag used to be accepted
|
|
393
|
+
* silently, which on a gate command means CI passing while the author believes
|
|
394
|
+
* a threshold is set — `--max-growh 5` would have been ignored and the build
|
|
395
|
+
* gone green. Silence is the wrong answer for a typo.
|
|
396
|
+
*/
|
|
397
|
+
const GLOBAL_FLAGS = ['help', 'h', 'locale', 'json', 'config', 'pricing', 'pricing-live'];
|
|
398
|
+
const COMMAND_FLAGS: Record<string, string[]> = {
|
|
399
|
+
optimize: [
|
|
400
|
+
'level', 'model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch',
|
|
401
|
+
'disable', 'llm', 'exact-tokens', 'diff', 'reorder', 'out', 'o',
|
|
402
|
+
'tokens-only', 'cost', 'prompt', 'suggest', 'apply-suggestions',
|
|
403
|
+
'cache-suggestions',
|
|
404
|
+
],
|
|
405
|
+
check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
|
|
406
|
+
baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
|
|
407
|
+
eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
|
|
408
|
+
prune: ['cases', 'concurrency', 'json', 'yes'],
|
|
409
|
+
diff: ['level', 'model', 'calls', 'output-tokens', 'batch', 'max-growth', 'optimized', 'markdown-out', 'all', 'prompt'],
|
|
410
|
+
models: [],
|
|
411
|
+
rank: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'markdown-out'],
|
|
412
|
+
where: [],
|
|
413
|
+
rules: [],
|
|
414
|
+
blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
|
|
415
|
+
doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
|
|
416
|
+
};
|
|
417
|
+
|
|
418
|
+
function rejectUnknownFlags(args: Args, t: CliMessages): void {
|
|
419
|
+
const known = COMMAND_FLAGS[args.command];
|
|
420
|
+
if (!known) return;
|
|
421
|
+
const allowed = [...known, ...GLOBAL_FLAGS];
|
|
422
|
+
|
|
423
|
+
for (const name of args.flags.keys()) {
|
|
424
|
+
// `out` is stored under its long name even when given as `-o`, and a
|
|
425
|
+
// negated boolean under its base name, so both validate against the list.
|
|
426
|
+
if (allowed.includes(name)) continue;
|
|
427
|
+
|
|
428
|
+
// Quoted as typed, so `--no-nonsense` is not reported as `--nonsense`.
|
|
429
|
+
const spelled = args.asTyped.get(name) ?? name;
|
|
430
|
+
const nearest = nearestName(name, allowed);
|
|
431
|
+
throw new Error(
|
|
432
|
+
nearest
|
|
433
|
+
? t.errors.unknownFlagDidYouMean(spelled, nearest)
|
|
434
|
+
: t.errors.unknownFlag(spelled, allowed.slice().sort().join(', ')),
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// --------------------------------------------------------------------------
|
|
440
|
+
// Line-by-line diff
|
|
441
|
+
// --------------------------------------------------------------------------
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Largest diff this will attempt, in lines per side.
|
|
445
|
+
*
|
|
446
|
+
* The alignment table is quadratic: at 6,000 lines it is 36 million cells and
|
|
447
|
+
* roughly 288 MB before anything else runs. There is no prompt worth reading a
|
|
448
|
+
* 6,000-line diff of, so past this the diff is declined rather than the process
|
|
449
|
+
* being taken down by someone passing a large file.
|
|
450
|
+
*/
|
|
451
|
+
const MAX_DIFF_LINES = 2500;
|
|
452
|
+
|
|
453
|
+
/** Longest common subsequence, used to align the two versions. */
|
|
454
|
+
function lcsTable(a: string[], b: string[]): number[][] {
|
|
455
|
+
const table: number[][] = Array.from({ length: a.length + 1 }, () =>
|
|
456
|
+
new Array<number>(b.length + 1).fill(0),
|
|
457
|
+
);
|
|
458
|
+
for (let i = a.length - 1; i >= 0; i--) {
|
|
459
|
+
for (let j = b.length - 1; j >= 0; j--) {
|
|
460
|
+
table[i]![j] =
|
|
461
|
+
a[i] === b[j] ? table[i + 1]![j + 1]! + 1 : Math.max(table[i + 1]![j]!, table[i]![j + 1]!);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
return table;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
function renderDiff(before: string, after: string, t: CliMessages): string {
|
|
468
|
+
const a = before.split('\n');
|
|
469
|
+
const b = after.split('\n');
|
|
470
|
+
|
|
471
|
+
if (a.length > MAX_DIFF_LINES || b.length > MAX_DIFF_LINES) {
|
|
472
|
+
return c.dim(t.report.diffTooLarge(Math.max(a.length, b.length), MAX_DIFF_LINES));
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
const table = lcsTable(a, b);
|
|
476
|
+
const lines: string[] = [];
|
|
477
|
+
|
|
478
|
+
let i = 0;
|
|
479
|
+
let j = 0;
|
|
480
|
+
while (i < a.length && j < b.length) {
|
|
481
|
+
if (a[i] === b[j]) {
|
|
482
|
+
lines.push(c.dim(` ${a[i]}`));
|
|
483
|
+
i++;
|
|
484
|
+
j++;
|
|
485
|
+
} else if (table[i + 1]![j]! >= table[i]![j + 1]!) {
|
|
486
|
+
lines.push(c.red(`- ${a[i]}`));
|
|
487
|
+
i++;
|
|
488
|
+
} else {
|
|
489
|
+
lines.push(c.green(`+ ${b[j]}`));
|
|
490
|
+
j++;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
while (i < a.length) lines.push(c.red(`- ${a[i++]}`));
|
|
494
|
+
while (j < b.length) lines.push(c.green(`+ ${b[j++]}`));
|
|
495
|
+
|
|
496
|
+
return lines.join('\n');
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
// --------------------------------------------------------------------------
|
|
500
|
+
// Report
|
|
501
|
+
// --------------------------------------------------------------------------
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* The provider's name when the estimator was not calibrated for it.
|
|
505
|
+
*
|
|
506
|
+
* `estimateTokens` is a heuristic tuned against Claude's tokenizer, and the
|
|
507
|
+
* ±15% band descends from that. Printing the same band beside a GPT or Kimi
|
|
508
|
+
* figure states a precision nobody has measured for that family — and since the
|
|
509
|
+
* catalogue grew past Anthropic, that is most of it. Returns null when the model
|
|
510
|
+
* is Anthropic's, where the band is at least the claim it was written for.
|
|
511
|
+
*/
|
|
512
|
+
function offFamilyName(modelId: string): string | null {
|
|
513
|
+
const provider = getModel(modelId).provider;
|
|
514
|
+
if (provider === undefined || provider === 'anthropic') return null;
|
|
515
|
+
return getModel(modelId).displayName;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Language codes as names, in the reader's language.
|
|
520
|
+
*
|
|
521
|
+
* Built from `PHRASE_LANGUAGES` rather than written out, so a language added to
|
|
522
|
+
* the dictionaries appears here without anybody remembering to edit a sentence.
|
|
523
|
+
*/
|
|
524
|
+
function languageNames(codes: readonly string[], t: CliMessages): string {
|
|
525
|
+
const names = codes.map((code) => t.languages[code] ?? code);
|
|
526
|
+
if (names.length <= 1) return names[0] ?? '';
|
|
527
|
+
return `${names.slice(0, -1).join(', ')} ${t.languages.and} ${names[names.length - 1]}`;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function printReport(
|
|
531
|
+
result: OptimizationResult,
|
|
532
|
+
showDiff: boolean,
|
|
533
|
+
t: CliMessages,
|
|
534
|
+
examplesReview: ExampleReview | null = null,
|
|
535
|
+
reorder: ReorderResult | null = null,
|
|
536
|
+
tokensOnly = false,
|
|
537
|
+
host: HostEnvironment = { id: 'terminal', displayName: 'terminal', billing: 'unknown', evidence: null },
|
|
538
|
+
suggestions: { result: SuggestResult; applied: boolean; locale: Locale } | null = null,
|
|
539
|
+
): void {
|
|
540
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
541
|
+
const sourceNote =
|
|
542
|
+
result.tokenSource === 'heuristic'
|
|
543
|
+
? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
|
|
544
|
+
: c.dim(t.report.exactCount());
|
|
545
|
+
|
|
546
|
+
console.log();
|
|
547
|
+
console.log(c.bold(t.report.inputTokens()));
|
|
548
|
+
console.log(
|
|
549
|
+
` ${n(result.tokensBefore)} → ${c.green(n(result.tokensAfter))} ${c.bold(
|
|
550
|
+
`-${result.reductionPct.toFixed(1)}%`,
|
|
551
|
+
)}${sourceNote}`,
|
|
552
|
+
);
|
|
553
|
+
|
|
554
|
+
// Before the rules, because the rearrangement is the bigger change and the
|
|
555
|
+
// one the reader has to make a judgement about.
|
|
556
|
+
//
|
|
557
|
+
// Only when there is something to say. "Nothing could safely move" with no
|
|
558
|
+
// refusals underneath is a heading, a blank line and a shrug — the reader
|
|
559
|
+
// asked for a rearrangement, there was none available, and the token count
|
|
560
|
+
// above already told them nothing changed.
|
|
561
|
+
if (reorder !== null && (reorder.moved.length > 0 || reorder.declined.length > 0)) {
|
|
562
|
+
console.log();
|
|
563
|
+
console.log(c.bold(t.report.reorderHeading()));
|
|
564
|
+
if (reorder.moved.length === 0) {
|
|
565
|
+
console.log(` ${c.dim(t.report.reorderNothing())}`);
|
|
566
|
+
} else {
|
|
567
|
+
console.log(` ${t.report.reorderMoved(reorder.moved.length, n(reorder.tokensMoved))}`);
|
|
568
|
+
console.log(
|
|
569
|
+
` ${c.green(
|
|
570
|
+
t.report.reorderPrefix(n(reorder.prefixTokensBefore), n(reorder.prefixTokensAfter)),
|
|
571
|
+
)}`,
|
|
572
|
+
);
|
|
573
|
+
}
|
|
574
|
+
// Refusals are reported even when the move succeeded: a saving Trazum chose
|
|
575
|
+
// not to take is one the author cannot evaluate unless they are told.
|
|
576
|
+
if (reorder.declined.length > 0) {
|
|
577
|
+
console.log(` ${c.dim(t.report.reorderDeclined(reorder.declined.length))}`);
|
|
578
|
+
const SHOWN = 3;
|
|
579
|
+
for (const d of reorder.declined.slice(0, SHOWN)) {
|
|
580
|
+
const excerpt = truncate(d.text.trim().replace(/\s+/g, ' '), 48);
|
|
581
|
+
console.log(
|
|
582
|
+
` ${c.dim(
|
|
583
|
+
d.reason === 'uncovered-script'
|
|
584
|
+
? t.report.reorderDeclinedScript(d.script ?? '')
|
|
585
|
+
: d.reason === 'backward-reference'
|
|
586
|
+
? t.report.reorderDeclinedRef(d.phrase ?? '', excerpt)
|
|
587
|
+
: t.report.reorderDeclinedAfter(excerpt),
|
|
588
|
+
)}`,
|
|
589
|
+
);
|
|
590
|
+
}
|
|
591
|
+
// Say that the list was cut. A report that shows three of nine reads as
|
|
592
|
+
// "three" unless it admits otherwise.
|
|
593
|
+
if (reorder.declined.length > SHOWN) {
|
|
594
|
+
console.log(` ${c.dim(t.report.reorderDeclinedMore(reorder.declined.length - SHOWN))}`);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
if (reorder.moved.length > 0) console.log(` ${c.yellow(t.report.reorderReview())}`);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
if (result.rules.length > 0) {
|
|
601
|
+
console.log();
|
|
602
|
+
console.log(c.bold(t.report.rulesApplied()));
|
|
603
|
+
for (const rule of result.rules) {
|
|
604
|
+
const tag =
|
|
605
|
+
rule.level === 'aggressive'
|
|
606
|
+
? c.yellow(t.report.levelAggressive())
|
|
607
|
+
: c.dim(t.report.levelSafe());
|
|
608
|
+
console.log(` ${tag} ${rule.title} ${c.dim(t.report.ruleHits(rule.hits, rule.tokensSaved))}`);
|
|
609
|
+
|
|
610
|
+
// What the rule actually did. Shown under the aggressive level by
|
|
611
|
+
// default because that is the one whose advice is "read the diff", and
|
|
612
|
+
// a diff of everything at once is not something anyone reads.
|
|
613
|
+
const showChanges = showDiff || rule.level === 'aggressive';
|
|
614
|
+
if (showChanges) {
|
|
615
|
+
for (const change of rule.changes) {
|
|
616
|
+
const from = c.red(truncate(change.before, 46));
|
|
617
|
+
const to = change.after ? c.green(truncate(change.after, 30)) : c.dim('—');
|
|
618
|
+
console.log(` ${from} ${c.dim('→')} ${to}`);
|
|
619
|
+
}
|
|
620
|
+
if (rule.hits > rule.changes.length && rule.changes.length > 0) {
|
|
621
|
+
console.log(c.dim(` ${t.report.moreChanges(rule.hits - rule.changes.length)}`));
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
} else {
|
|
626
|
+
console.log();
|
|
627
|
+
console.log(c.dim(t.report.nothingToTrim()));
|
|
628
|
+
// Which languages the dictionaries actually cover. Only here, because this
|
|
629
|
+
// is the one branch where silence reads as "your prompt is already clean".
|
|
630
|
+
console.log(c.dim(t.report.dictionaryCoverage(languageNames(PHRASE_LANGUAGES, t))));
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
if (result.llm) {
|
|
634
|
+
console.log();
|
|
635
|
+
console.log(c.bold(t.report.llmPass()));
|
|
636
|
+
if (result.llm.applied) {
|
|
637
|
+
console.log(
|
|
638
|
+
` ${c.green(
|
|
639
|
+
t.report.llmApplied(
|
|
640
|
+
result.llm.provider,
|
|
641
|
+
result.llm.model,
|
|
642
|
+
result.llm.tokensBefore,
|
|
643
|
+
result.llm.tokensAfter,
|
|
644
|
+
),
|
|
645
|
+
)}`,
|
|
646
|
+
);
|
|
647
|
+
} else {
|
|
648
|
+
console.log(` ${c.yellow(t.report.llmRejected(result.llm.rejectedReason ?? ''))}`);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
// On a subscription there is no bill to reduce. Everything below this point
|
|
653
|
+
// would be arithmetic about tokens dressed as money, and "$184/month" told to
|
|
654
|
+
// somebody on a flat plan is wrong in the direction that matters most.
|
|
655
|
+
//
|
|
656
|
+
// What replaces it is the thing that *is* scarce there: the context window.
|
|
657
|
+
if (tokensOnly) {
|
|
658
|
+
printTokensOnly(result, host, t, n);
|
|
659
|
+
} else {
|
|
660
|
+
printMoney(result, t, n);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
// On a subscription, an advisory whose entire pitch is money is not weaker
|
|
664
|
+
// advice — it is not advice. "Use a cheaper model" saves nothing on a flat
|
|
665
|
+
// plan, and its detail text quotes dollars per month, so suppressing only the
|
|
666
|
+
// price tag beside the title left the money in the sentence underneath.
|
|
667
|
+
//
|
|
668
|
+
// The rest stay: an overflowing context window still fails the call, a
|
|
669
|
+
// contradiction is still wrong, redundant examples still cost tokens, and
|
|
670
|
+
// caching still buys latency and rate-limit headroom.
|
|
671
|
+
const MONEY_ONLY = new Set([
|
|
672
|
+
'model-downgrade',
|
|
673
|
+
'batch-api',
|
|
674
|
+
'output-dominated',
|
|
675
|
+
'promo-pricing',
|
|
676
|
+
'prompt-caching-not-worth-it',
|
|
677
|
+
]);
|
|
678
|
+
const advisories = tokensOnly
|
|
679
|
+
? result.advisories.filter((a) => !MONEY_ONLY.has(a.id))
|
|
680
|
+
: result.advisories;
|
|
681
|
+
|
|
682
|
+
if (advisories.length > 0) {
|
|
683
|
+
console.log();
|
|
684
|
+
console.log(c.bold(t.report.beyondShortening()));
|
|
685
|
+
|
|
686
|
+
// The amount goes in a column of its own rather than trailing the title.
|
|
687
|
+
// Four advisories worth $506, $422, $170 and nothing are meant to be
|
|
688
|
+
// compared, and comparing them meant reading to the end of four different
|
|
689
|
+
// sentences to find where the numbers were.
|
|
690
|
+
//
|
|
691
|
+
// The advisory itself still applies on a subscription — caching and a
|
|
692
|
+
// smaller model both buy back context and rate-limit headroom. Only the
|
|
693
|
+
// price tag is meaningless, so only the price tag goes.
|
|
694
|
+
const amountOf = (a: (typeof advisories)[number]): string =>
|
|
695
|
+
!tokensOnly && a.estimatedMonthlyUsd !== null ? formatUsd(a.estimatedMonthlyUsd) : '';
|
|
696
|
+
const width = Math.max(0, ...advisories.map((a) => amountOf(a).length));
|
|
697
|
+
// Indent the wrapped detail to the start of the title, so the prose forms
|
|
698
|
+
// one block instead of stepping around the numbers.
|
|
699
|
+
const gutter = ' '.repeat(4 + (width > 0 ? width + 2 : 0));
|
|
700
|
+
|
|
701
|
+
for (const advisory of advisories) {
|
|
702
|
+
const marker =
|
|
703
|
+
advisory.severity === 'warning'
|
|
704
|
+
? c.yellow('!')
|
|
705
|
+
: advisory.severity === 'opportunity'
|
|
706
|
+
? c.cyan('→')
|
|
707
|
+
: c.dim('·');
|
|
708
|
+
const amount = amountOf(advisory);
|
|
709
|
+
const column = width > 0 ? `${c.green(amount.padStart(width))} ` : '';
|
|
710
|
+
console.log(` ${marker} ${column}${c.bold(advisory.title)}`);
|
|
711
|
+
console.log(`${gutter}${c.dim(wrap(advisory.detail, 78 - gutter.length, gutter))}`);
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
// What to do first. The rules trimmed $1.25 and the top advisory is worth
|
|
715
|
+
// $506; leaving the reader to notice that by comparing four numbers in four
|
|
716
|
+
// sentences is how the most valuable line in the report gets skipped.
|
|
717
|
+
const best = advisories.find((a) => (a.estimatedMonthlyUsd ?? 0) > 0);
|
|
718
|
+
if (!tokensOnly && best?.estimatedMonthlyUsd) {
|
|
719
|
+
const ruleSaving = result.savings.monthlySavingsUsd;
|
|
720
|
+
const line = t.report.biggestLeverDetail(
|
|
721
|
+
best.title,
|
|
722
|
+
formatUsd(best.estimatedMonthlyUsd),
|
|
723
|
+
ruleSaving > 0 ? Math.round(best.estimatedMonthlyUsd / ruleSaving) : null,
|
|
724
|
+
);
|
|
725
|
+
console.log();
|
|
726
|
+
// Wrapped to the same width as everything else. An unwrapped closing line
|
|
727
|
+
// is the one that runs off a narrow terminal, and it is the line most
|
|
728
|
+
// worth reading.
|
|
729
|
+
console.log(` ${c.bold(t.report.biggestLever())} ${c.dim(wrap(line, 62, ' '))}`);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
printSuggestions(suggestions, t, n);
|
|
734
|
+
printRest(result, showDiff, t, examplesReview, n);
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
/** The cost section, for anyone billed by the token. */
|
|
738
|
+
function printMoney(result: OptimizationResult, t: CliMessages, n: (v: number) => string): void {
|
|
739
|
+
const { savings } = result;
|
|
740
|
+
console.log();
|
|
741
|
+
console.log(c.bold(t.report.costWith(savings.modelDisplayName)));
|
|
742
|
+
console.log(
|
|
743
|
+
` ${t.report.usageLine(
|
|
744
|
+
n(result.usage.callsPerMonth),
|
|
745
|
+
result.usage.avgOutputTokens,
|
|
746
|
+
result.usage.batchEligible,
|
|
747
|
+
)}`,
|
|
748
|
+
);
|
|
749
|
+
// Said, not assumed. Once prices can be overlaid locally, a figure from the
|
|
750
|
+
// bundled catalogue and a figure from somebody's JSON file look identical, and
|
|
751
|
+
// the reader has to be able to tell which one they are about to budget against.
|
|
752
|
+
const touched = [
|
|
753
|
+
...result.pricingSource.overriddenModels,
|
|
754
|
+
...result.pricingSource.addedModels,
|
|
755
|
+
];
|
|
756
|
+
if (touched.length > 0) {
|
|
757
|
+
console.log(
|
|
758
|
+
` ${c.yellow(t.report.pricingOverlaid(touched.join(', '), result.pricingSource.lastReviewed))}`,
|
|
759
|
+
);
|
|
760
|
+
}
|
|
761
|
+
console.log(
|
|
762
|
+
` ${formatUsd(savings.perMonth.before.totalUsd)} → ` +
|
|
763
|
+
`${c.green(formatUsd(savings.perMonth.after.totalUsd))} ` +
|
|
764
|
+
c.bold(
|
|
765
|
+
t.report.perMonthSaving(
|
|
766
|
+
formatUsd(savings.monthlySavingsUsd),
|
|
767
|
+
savings.monthlySavingsPct.toFixed(1),
|
|
768
|
+
),
|
|
769
|
+
),
|
|
770
|
+
);
|
|
771
|
+
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
/**
|
|
775
|
+
* What the saving buys when there is no bill: room.
|
|
776
|
+
*
|
|
777
|
+
* The context window is the scarce thing inside an agent — every token the
|
|
778
|
+
* system prompt holds is one the conversation cannot. That is a real saving and
|
|
779
|
+
* a measurable one, and it is the honest answer to "what did I gain" on a plan
|
|
780
|
+
* that costs the same either way.
|
|
781
|
+
*/
|
|
782
|
+
function printTokensOnly(
|
|
783
|
+
result: OptimizationResult,
|
|
784
|
+
host: HostEnvironment,
|
|
785
|
+
t: CliMessages,
|
|
786
|
+
n: (v: number) => string,
|
|
787
|
+
): void {
|
|
788
|
+
const model = getModel(result.usage.model);
|
|
789
|
+
const saved = result.tokensBefore - result.tokensAfter;
|
|
790
|
+
|
|
791
|
+
console.log();
|
|
792
|
+
console.log(c.bold(t.report.tokensOnlyHeading(host.displayName)));
|
|
793
|
+
// Only claim the host bills by subscription when it does. Forced with the
|
|
794
|
+
// flag on GitHub Actions, the first version said "GitHub Actions bills by
|
|
795
|
+
// subscription", which is simply false.
|
|
796
|
+
console.log(
|
|
797
|
+
` ${
|
|
798
|
+
host.billing === 'subscription'
|
|
799
|
+
? t.report.tokensOnlyWhy(host.displayName)
|
|
800
|
+
: t.report.tokensOnlyAsked()
|
|
801
|
+
}`,
|
|
802
|
+
);
|
|
803
|
+
console.log();
|
|
804
|
+
console.log(` ${c.green(t.report.tokensSaved(n(saved)))}`);
|
|
805
|
+
|
|
806
|
+
// Share of the window, which is what a saved token is actually worth here.
|
|
807
|
+
const share = (tokens: number): string =>
|
|
808
|
+
`${((tokens / model.contextWindow) * 100).toFixed(1)}%`;
|
|
809
|
+
console.log(
|
|
810
|
+
` ${c.dim(
|
|
811
|
+
t.report.windowUse(
|
|
812
|
+
share(result.tokensBefore),
|
|
813
|
+
share(result.tokensAfter),
|
|
814
|
+
model.displayName,
|
|
815
|
+
n(model.contextWindow),
|
|
816
|
+
),
|
|
817
|
+
)}`,
|
|
818
|
+
);
|
|
819
|
+
console.log(` ${c.dim(t.report.tokensOnlyCost())}`);
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
/**
|
|
823
|
+
* The proposed rewrites.
|
|
824
|
+
*
|
|
825
|
+
* A list, not a diff, because that is the shape of the decision: each line is
|
|
826
|
+
* one phrase and its replacement, and the reader is answering "yes" or "no" to
|
|
827
|
+
* that phrase rather than to a rewritten prompt.
|
|
828
|
+
*
|
|
829
|
+
* Rejections are summarised rather than listed one by one. "Four proposals did
|
|
830
|
+
* not survive checking" is the useful fact; which four is noise unless you are
|
|
831
|
+
* debugging the model, and `--json` has them for when you are.
|
|
832
|
+
*/
|
|
833
|
+
function printSuggestions(
|
|
834
|
+
suggestions: { result: SuggestResult; applied: boolean; locale: Locale } | null,
|
|
835
|
+
t: CliMessages,
|
|
836
|
+
n: (value: number) => string,
|
|
837
|
+
): void {
|
|
838
|
+
if (!suggestions) return;
|
|
839
|
+
const { result, applied, locale } = suggestions;
|
|
840
|
+
|
|
841
|
+
if (result.suggestions.length === 0) {
|
|
842
|
+
// Say so. A silent absence reads as "the flag did nothing".
|
|
843
|
+
console.log(`\n${c.bold(t.report.suggestHeading())}`);
|
|
844
|
+
console.log(` ${c.dim(t.report.suggestNothing(result.provider, result.model))}`);
|
|
845
|
+
if (result.rejected.length > 0) {
|
|
846
|
+
console.log(` ${c.dim(t.report.suggestRejected(result.rejected.length))}`);
|
|
847
|
+
}
|
|
848
|
+
return;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
const total = result.suggestions.reduce((sum, s) => sum + s.tokensSaved, 0);
|
|
852
|
+
console.log(`\n${c.bold(t.report.suggestHeading())}`);
|
|
853
|
+
console.log(
|
|
854
|
+
` ${c.dim(
|
|
855
|
+
applied
|
|
856
|
+
? t.report.suggestApplied(result.suggestions.length, n(total))
|
|
857
|
+
: t.report.suggestOffered(result.suggestions.length, n(total)),
|
|
858
|
+
)}`,
|
|
859
|
+
);
|
|
860
|
+
|
|
861
|
+
for (const s of result.suggestions) {
|
|
862
|
+
const after = s.after === '' ? c.dim(t.report.suggestRemoved()) : c.green(truncate(s.after, 40));
|
|
863
|
+
const times = s.offsets.length > 1 ? c.dim(` ×${s.offsets.length}`) : '';
|
|
864
|
+
console.log(
|
|
865
|
+
` ${c.red(truncate(s.before, 44))} ${c.dim('→')} ${after}` +
|
|
866
|
+
` ${c.dim(`~${n(s.tokensSaved)}`)}${times}`,
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
if (result.rejected.length > 0) {
|
|
871
|
+
console.log(` ${c.dim(t.report.suggestRejected(result.rejected.length))}`);
|
|
872
|
+
// The most common reason, named. Four rejections all saying "the model
|
|
873
|
+
// paraphrased what it quoted" is a fact about the model worth knowing.
|
|
874
|
+
const counts = new Map<string, number>();
|
|
875
|
+
for (const r of result.rejected) counts.set(r.reason, (counts.get(r.reason) ?? 0) + 1);
|
|
876
|
+
const [reason] = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]!;
|
|
877
|
+
console.log(` ${c.dim(rejectionText(reason as RejectedReason, locale))}`);
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
if (!applied) console.log(` ${c.dim(t.report.suggestHowToApply())}`);
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
function printRest(
|
|
884
|
+
result: OptimizationResult,
|
|
885
|
+
showDiff: boolean,
|
|
886
|
+
t: CliMessages,
|
|
887
|
+
examplesReview: ExampleReview | null,
|
|
888
|
+
n: (v: number) => string,
|
|
889
|
+
): void {
|
|
890
|
+
if (examplesReview && examplesReview.groups.length > 0) {
|
|
891
|
+
console.log();
|
|
892
|
+
console.log(c.bold(t.report.examplesReview()));
|
|
893
|
+
console.log(
|
|
894
|
+
c.dim(
|
|
895
|
+
` ${t.report.examplesReviewNote(
|
|
896
|
+
examplesReview.provider,
|
|
897
|
+
examplesReview.model,
|
|
898
|
+
examplesReview.exampleCount,
|
|
899
|
+
)}`,
|
|
900
|
+
),
|
|
901
|
+
);
|
|
902
|
+
for (const group of examplesReview.groups) {
|
|
903
|
+
console.log(
|
|
904
|
+
` ${c.yellow(t.report.exampleRedundant(group.redundant, group.keep))}` +
|
|
905
|
+
c.dim(` (~${group.tokens} tokens)`),
|
|
906
|
+
);
|
|
907
|
+
if (group.reason) console.log(` ${c.dim(group.reason)}`);
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if (showDiff) {
|
|
912
|
+
console.log();
|
|
913
|
+
console.log(c.bold(t.report.diff()));
|
|
914
|
+
console.log(renderDiff(result.original, result.optimized, t));
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
console.log();
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/** Shortens a snippet for the change list, keeping it on one line. */
|
|
921
|
+
function truncate(text: string, max: number): string {
|
|
922
|
+
const clean = text.replace(/\s+/g, ' ').trim();
|
|
923
|
+
return clean.length <= max ? clean : `${clean.slice(0, max - 1)}\u2026`;
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
/** Wraps a paragraph to a given width. */
|
|
927
|
+
function wrap(text: string, width: number, indent: string): string {
|
|
928
|
+
const words = text.split(/\s+/);
|
|
929
|
+
const lines: string[] = [];
|
|
930
|
+
let line = '';
|
|
931
|
+
for (const word of words) {
|
|
932
|
+
if (line.length + word.length + 1 > width) {
|
|
933
|
+
lines.push(line);
|
|
934
|
+
line = word;
|
|
935
|
+
} else {
|
|
936
|
+
line = line ? `${line} ${word}` : word;
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
if (line) lines.push(line);
|
|
940
|
+
return lines.join(`\n${indent}`);
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
// --------------------------------------------------------------------------
|
|
944
|
+
// Subcommands
|
|
945
|
+
// --------------------------------------------------------------------------
|
|
946
|
+
|
|
947
|
+
/**
|
|
948
|
+
* A provider's stand-in model, for when the code names who but not which.
|
|
949
|
+
*
|
|
950
|
+
* The same capability as the global default, so the figure is comparable with
|
|
951
|
+
* what Trazum would have printed anyway, and the cheapest at that capability so
|
|
952
|
+
* the guess errs downwards — overstating somebody's bill on a model they never
|
|
953
|
+
* chose is the worse direction to be wrong in.
|
|
954
|
+
*/
|
|
955
|
+
function defaultModelFor(provider: string, pricing: PricingCatalogue): string | null {
|
|
956
|
+
// Nearest capability, not an exact match. Matching exactly returned nothing
|
|
957
|
+
// for OpenAI and DeepSeek — neither has a `large` model, so the code fell
|
|
958
|
+
// through to the global default and printed "goes to openai / priced as
|
|
959
|
+
// Claude Opus 5" anyway. A ladder with different rungs is the normal case,
|
|
960
|
+
// not an edge one.
|
|
961
|
+
const RANK: Record<string, number> = { small: 0, mid: 1, large: 2, frontier: 3 };
|
|
962
|
+
const want = RANK[getModel(DEFAULT_USAGE.model).capability] ?? 2;
|
|
963
|
+
|
|
964
|
+
const candidates = pricing.models.filter(
|
|
965
|
+
(m) => m.provider === provider && m.recommendable !== false,
|
|
966
|
+
);
|
|
967
|
+
|
|
968
|
+
const best = candidates.reduce<(typeof candidates)[number] | null>((chosen, m) => {
|
|
969
|
+
if (chosen === null) return m;
|
|
970
|
+
const distance = Math.abs((RANK[m.capability] ?? 2) - want);
|
|
971
|
+
const chosenDistance = Math.abs((RANK[chosen.capability] ?? 2) - want);
|
|
972
|
+
if (distance !== chosenDistance) return distance < chosenDistance ? m : chosen;
|
|
973
|
+
// Same distance: the cheaper one, so the guess errs downwards. Overstating
|
|
974
|
+
// somebody's bill on a model they never chose is the worse way to be wrong.
|
|
975
|
+
return m.inputPerMTok < chosen.inputPerMTok ? m : chosen;
|
|
976
|
+
}, null);
|
|
977
|
+
|
|
978
|
+
return best?.id ?? null;
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
/**
|
|
982
|
+
* Reads a source file as the prompts it holds, rather than as one big prompt.
|
|
983
|
+
*
|
|
984
|
+
* Returns null for anything that is not a source file, which is the ordinary
|
|
985
|
+
* case: a `.txt` or `.md` prompt goes through untouched.
|
|
986
|
+
*
|
|
987
|
+
* For a source file it **refuses rather than guesses**. Optimising TypeScript
|
|
988
|
+
* as if it were prose does not produce a worse prompt, it produces broken code
|
|
989
|
+
* — `import OpenAI` came back as `Import OpenAI` from the capitalisation rule —
|
|
990
|
+
* and `-o` would write that over the file. A refusal with the marker syntax in
|
|
991
|
+
* it costs the reader one comment; the alternative cost them a compile.
|
|
992
|
+
*/
|
|
993
|
+
function sourceFileOf(
|
|
994
|
+
target: string,
|
|
995
|
+
raw: string,
|
|
996
|
+
pricing: PricingCatalogue,
|
|
997
|
+
wanted: string | undefined,
|
|
998
|
+
): { text: string; model?: string } | null {
|
|
999
|
+
const isSource = SOURCE_EXTENSIONS.some((ext) => target.toLowerCase().endsWith(ext));
|
|
1000
|
+
if (!isSource) return null;
|
|
1001
|
+
|
|
1002
|
+
// The catalogue in effect rather than the bundled one: an overlay can add a
|
|
1003
|
+
// model, and a detection that cannot see it would fall back for no reason.
|
|
1004
|
+
const detection = detectFromSource(raw, { models: pricing.models });
|
|
1005
|
+
// An import names who, never which — so a file that plainly calls OpenAI was
|
|
1006
|
+
// still being priced against Claude Opus 5. The provider's own stand-in is a
|
|
1007
|
+
// guess about which of their models rather than about whose, which is the
|
|
1008
|
+
// difference that matters. `trazum where` says which it picked and why.
|
|
1009
|
+
const model =
|
|
1010
|
+
detection.model ??
|
|
1011
|
+
(detection.provider !== null ? (defaultModelFor(detection.provider, pricing) ?? undefined) : undefined);
|
|
1012
|
+
|
|
1013
|
+
if (!hasMarker(raw)) {
|
|
1014
|
+
throw new Error(t_sourceNeedsMarker(target));
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
const { prompts, declined } = extractPrompts(raw);
|
|
1018
|
+
if (prompts.length === 0) {
|
|
1019
|
+
const why = declined[0];
|
|
1020
|
+
throw new Error(
|
|
1021
|
+
why
|
|
1022
|
+
? `${target}: the marker on line ${why.line} could not be read — ${why.detail}`
|
|
1023
|
+
: `${target}: nothing was extracted from the markers in this file.`,
|
|
1024
|
+
);
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
// One prompt is unambiguous. Several need naming, because optimising "the
|
|
1028
|
+
// first one" silently is how the wrong prompt ends up rewritten.
|
|
1029
|
+
const chosen =
|
|
1030
|
+
wanted !== undefined
|
|
1031
|
+
? prompts.find((p) => p.name === wanted || promptId(target, p) === wanted)
|
|
1032
|
+
: prompts.length === 1
|
|
1033
|
+
? prompts[0]
|
|
1034
|
+
: undefined;
|
|
1035
|
+
|
|
1036
|
+
if (!chosen) {
|
|
1037
|
+
const names = prompts.map((p) => promptId(target, p)).join('\n ');
|
|
1038
|
+
throw new Error(
|
|
1039
|
+
wanted !== undefined
|
|
1040
|
+
? `${target} has no marked prompt called "${wanted}". It holds:\n ${names}`
|
|
1041
|
+
: `${target} holds ${prompts.length} marked prompts. Name one with --prompt:\n ${names}`,
|
|
1042
|
+
);
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
return { text: chosen.text, ...(model ? { model } : {}) };
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/** Kept as a function so the sentence is in one place rather than two. */
|
|
1049
|
+
const t_sourceNeedsMarker = (target: string): string =>
|
|
1050
|
+
`${target} looks like source, not a prompt. Optimising it would rewrite your code — ` +
|
|
1051
|
+
'mark the prompt with a `// trazum:prompt` comment above the literal, or pass the ' +
|
|
1052
|
+
'prompt itself in a .txt file.';
|
|
1053
|
+
|
|
1054
|
+
/**
|
|
1055
|
+
* Says which provider a prompt is actually sent to, and how it knows.
|
|
1056
|
+
*
|
|
1057
|
+
* Trazum priced one vendor, so the default cost nothing. Pricing seven made it a
|
|
1058
|
+
* wrong number: a file calling OpenAI was billed against Claude Opus 5 without
|
|
1059
|
+
* comment. This reads what the code already says instead.
|
|
1060
|
+
*
|
|
1061
|
+
* Every answer names the line it came from. A detection this command cannot
|
|
1062
|
+
* justify is a guess, and the number that follows from it would be a guess too.
|
|
1063
|
+
*/
|
|
1064
|
+
async function commandWhere(
|
|
1065
|
+
args: Args,
|
|
1066
|
+
config: TrazumConfig,
|
|
1067
|
+
pricing: PricingCatalogue,
|
|
1068
|
+
t: CliMessages,
|
|
1069
|
+
): Promise<void> {
|
|
1070
|
+
const host = detectHost();
|
|
1071
|
+
|
|
1072
|
+
console.log();
|
|
1073
|
+
console.log(c.bold(t.where.hostHeading()));
|
|
1074
|
+
console.log(
|
|
1075
|
+
` ${host.displayName}${host.evidence ? c.dim(` (${host.evidence})`) : ''}`,
|
|
1076
|
+
);
|
|
1077
|
+
// The reason this is worth printing at all. Inside a flat plan the monthly
|
|
1078
|
+
// figure Trazum computes is arithmetic about tokens, not money anybody gets
|
|
1079
|
+
// back, and saying so is more useful than saying nothing.
|
|
1080
|
+
if (host.billing === 'subscription') {
|
|
1081
|
+
console.log(` ${c.yellow(t.where.subscription(host.displayName))}`);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
const target = args.positional[0];
|
|
1085
|
+
if (target === undefined) {
|
|
1086
|
+
console.log();
|
|
1087
|
+
console.log(c.dim(t.where.noTarget()));
|
|
1088
|
+
console.log();
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1092
|
+
const source = await readFile(target, 'utf8');
|
|
1093
|
+
const detection = detectFromSource(source, { models: pricing.models });
|
|
1094
|
+
|
|
1095
|
+
console.log();
|
|
1096
|
+
console.log(c.bold(t.where.sourceHeading(target)));
|
|
1097
|
+
|
|
1098
|
+
if (detection.conflicts.length > 0) {
|
|
1099
|
+
// Two answers is not a weaker version of one answer. Naming both and
|
|
1100
|
+
// declining is the only honest output here.
|
|
1101
|
+
console.log(` ${c.red(t.where.conflict())}`);
|
|
1102
|
+
for (const e of detection.evidence.slice(0, 4)) {
|
|
1103
|
+
console.log(` ${c.dim(t.where.evidenceLine(e.line ?? 0, e.kind, e.detail))}`);
|
|
1104
|
+
}
|
|
1105
|
+
console.log(` ${c.dim(t.where.conflictFallback())}`);
|
|
1106
|
+
} else if (detection.provider === null) {
|
|
1107
|
+
console.log(` ${c.dim(t.where.nothingFound())}`);
|
|
1108
|
+
} else {
|
|
1109
|
+
const model = detection.model ? getModel(detection.model) : null;
|
|
1110
|
+
console.log(
|
|
1111
|
+
` ${detection.provider}${model ? ` · ${model.displayName}` : c.dim(t.where.providerOnly())}`,
|
|
1112
|
+
);
|
|
1113
|
+
for (const e of detection.evidence.slice(0, 3)) {
|
|
1114
|
+
console.log(` ${c.dim(t.where.evidenceLine(e.line ?? 0, e.kind, e.detail))}`);
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
// What would actually be used, which is the question behind the question.
|
|
1119
|
+
// Flags beat config, config beats detection, detection beats the default —
|
|
1120
|
+
// and a reader deciding whether to pass --model needs to see which won.
|
|
1121
|
+
//
|
|
1122
|
+
// Knowing the provider but not the model is the common case: an import names
|
|
1123
|
+
// who, never which. Falling through to the built-in default there would print
|
|
1124
|
+
// "goes to openai" and "priced as Claude Opus 5" three lines apart, which is
|
|
1125
|
+
// the wrong number this command exists to catch, produced by the command
|
|
1126
|
+
// itself. A provider's own default is a guess, but it is a guess about which
|
|
1127
|
+
// of their models rather than about whose.
|
|
1128
|
+
const configured = config.usage?.model;
|
|
1129
|
+
const detected =
|
|
1130
|
+
detection.model ??
|
|
1131
|
+
(detection.provider !== null ? defaultModelFor(detection.provider, pricing) : null);
|
|
1132
|
+
|
|
1133
|
+
const effective = configured ?? detected ?? DEFAULT_USAGE.model;
|
|
1134
|
+
const reason = configured
|
|
1135
|
+
? t.where.fromConfig()
|
|
1136
|
+
: detection.model
|
|
1137
|
+
? t.where.fromDetection()
|
|
1138
|
+
: detected
|
|
1139
|
+
? t.where.fromProviderDefault(detection.provider ?? '')
|
|
1140
|
+
: t.where.fromDefault();
|
|
1141
|
+
|
|
1142
|
+
console.log();
|
|
1143
|
+
console.log(c.bold(t.where.pricedAs()));
|
|
1144
|
+
console.log(` ${getModel(effective).displayName} ${c.dim(reason)}`);
|
|
1145
|
+
console.log();
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
function commandModels(t: CliMessages, pricing: PricingCatalogue): void {
|
|
1149
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
1150
|
+
const col = t.models.columns;
|
|
1151
|
+
|
|
1152
|
+
console.log();
|
|
1153
|
+
console.log(c.bold(t.models.title()) + c.dim(t.models.unit()));
|
|
1154
|
+
console.log(
|
|
1155
|
+
c.dim(t.models.reviewedOn(pricing.lastReviewed, reviewAgeDays(pricing.lastReviewed, new Date()))),
|
|
1156
|
+
);
|
|
1157
|
+
console.log();
|
|
1158
|
+
|
|
1159
|
+
const rows = pricing.models.map((m) => ({
|
|
1160
|
+
id: m.id,
|
|
1161
|
+
input: m.promo ? `${m.promo.inputPerMTok} (→${m.inputPerMTok})` : String(m.inputPerMTok),
|
|
1162
|
+
output: m.promo ? `${m.promo.outputPerMTok} (→${m.outputPerMTok})` : String(m.outputPerMTok),
|
|
1163
|
+
context: `${n(m.contextWindow / 1000)}K`,
|
|
1164
|
+
// An unknown minimum prints as a dash, not as zero. Zero is a claim —
|
|
1165
|
+
// "caches from the first token" — and it is the wrong one.
|
|
1166
|
+
cache: m.cacheMinTokens === null ? '—' : n(m.cacheMinTokens),
|
|
1167
|
+
}));
|
|
1168
|
+
const widths = {
|
|
1169
|
+
id: Math.max(...rows.map((r) => r.id.length), col.model.length),
|
|
1170
|
+
input: Math.max(...rows.map((r) => r.input.length), col.input.length),
|
|
1171
|
+
output: Math.max(...rows.map((r) => r.output.length), col.output.length),
|
|
1172
|
+
context: Math.max(...rows.map((r) => r.context.length), col.context.length),
|
|
1173
|
+
cache: Math.max(...rows.map((r) => r.cache.length), col.cacheMin.length),
|
|
1174
|
+
};
|
|
1175
|
+
|
|
1176
|
+
console.log(
|
|
1177
|
+
c.bold(
|
|
1178
|
+
` ${col.model.padEnd(widths.id)} ${col.input.padStart(widths.input)} ` +
|
|
1179
|
+
`${col.output.padStart(widths.output)} ${col.context.padStart(widths.context)} ` +
|
|
1180
|
+
`${col.cacheMin.padStart(widths.cache)}`,
|
|
1181
|
+
),
|
|
1182
|
+
);
|
|
1183
|
+
for (const row of rows) {
|
|
1184
|
+
console.log(
|
|
1185
|
+
` ${row.id.padEnd(widths.id)} ${row.input.padStart(widths.input)} ` +
|
|
1186
|
+
`${row.output.padStart(widths.output)} ${row.context.padStart(widths.context)} ` +
|
|
1187
|
+
`${row.cache.padStart(widths.cache)}`,
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
console.log();
|
|
1192
|
+
console.log(c.dim(t.models.promoNote()));
|
|
1193
|
+
console.log(c.dim(t.models.cacheNote()));
|
|
1194
|
+
console.log(c.dim(t.models.batchNote()));
|
|
1195
|
+
console.log();
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
function commandRules(t: CliMessages, locale: Locale): void {
|
|
1199
|
+
// Rule copy lives in the core catalogue, so `trazum rules` and the report
|
|
1200
|
+
// never drift apart.
|
|
1201
|
+
const copy = getMessages(locale).rules;
|
|
1202
|
+
|
|
1203
|
+
console.log();
|
|
1204
|
+
console.log(c.bold(t.rules.title()));
|
|
1205
|
+
console.log(c.dim(t.rules.disableHint()));
|
|
1206
|
+
console.log();
|
|
1207
|
+
for (const rule of RULES) {
|
|
1208
|
+
const tag =
|
|
1209
|
+
rule.level === 'aggressive'
|
|
1210
|
+
? c.yellow(t.report.levelAggressive())
|
|
1211
|
+
: c.dim(t.report.levelSafe());
|
|
1212
|
+
console.log(` ${tag} ${c.bold(rule.id)} — ${copy[rule.id].title}`);
|
|
1213
|
+
console.log(` ${c.dim(wrap(copy[rule.id].rationale, 74, ' '))}`);
|
|
1214
|
+
console.log();
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
async function readInput(source: string | undefined, t: CliMessages): Promise<string> {
|
|
1219
|
+
if (!source) {
|
|
1220
|
+
throw new Error(t.errors.missingInputFile());
|
|
1221
|
+
}
|
|
1222
|
+
if (source === '-') {
|
|
1223
|
+
const chunks: Buffer[] = [];
|
|
1224
|
+
for await (const chunk of process.stdin) chunks.push(Buffer.from(chunk));
|
|
1225
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
1226
|
+
}
|
|
1227
|
+
return readFile(source, 'utf8');
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
async function commandOptimize(
|
|
1231
|
+
args: Args,
|
|
1232
|
+
config: TrazumConfig,
|
|
1233
|
+
pricing: PricingCatalogue,
|
|
1234
|
+
t: CliMessages,
|
|
1235
|
+
locale: Locale,
|
|
1236
|
+
): Promise<void> {
|
|
1237
|
+
const target = args.positional[0];
|
|
1238
|
+
const raw = await readInput(target, t);
|
|
1239
|
+
const level = levelFlag(args, config, t);
|
|
1240
|
+
|
|
1241
|
+
// A source file is not a prompt.
|
|
1242
|
+
//
|
|
1243
|
+
// Handed `src/prompts.ts`, this used to optimise the whole file — imports,
|
|
1244
|
+
// `const client = new OpenAI();`, all of it — count the code as tokens the
|
|
1245
|
+
// model would pay for, and then **rewrite the source**: the capitalisation
|
|
1246
|
+
// rule turned `import OpenAI` into `Import OpenAI`, which does not compile.
|
|
1247
|
+
// Writing that back over somebody's file is the worst thing in this
|
|
1248
|
+
// repository's history, and it was the default behaviour.
|
|
1249
|
+
const source =
|
|
1250
|
+
target !== undefined && target !== '-'
|
|
1251
|
+
? sourceFileOf(target, raw, pricing, stringFlag(args, 'prompt'))
|
|
1252
|
+
: null;
|
|
1253
|
+
const original = source ? source.text : raw;
|
|
1254
|
+
|
|
1255
|
+
// Detection sits between config and defaults, as everywhere: a flag beats
|
|
1256
|
+
// config, config beats what the code says, and what the code says beats a
|
|
1257
|
+
// built-in default that has no idea which provider you use.
|
|
1258
|
+
const usage = usageFrom(args, config, t, source?.model);
|
|
1259
|
+
|
|
1260
|
+
const disableRules = disabledRules(args, config) ?? [];
|
|
1261
|
+
for (const id of disableRules) {
|
|
1262
|
+
if (!RULES.some((r) => r.id === id)) {
|
|
1263
|
+
throw new Error(t.errors.unknownRuleInDisable(id));
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
// Reordering runs BEFORE the rules, and is opt-in.
|
|
1268
|
+
//
|
|
1269
|
+
// Before, because a rule that deletes a sentence changes which blocks exist;
|
|
1270
|
+
// reordering first means the rearrangement is decided on the prompt the author
|
|
1271
|
+
// wrote, which is the one they will review it against.
|
|
1272
|
+
//
|
|
1273
|
+
// Opt-in, and not part of `aggressive`, because every other transformation
|
|
1274
|
+
// here deletes text whose absence is local while this one moves text, and
|
|
1275
|
+
// order carries meaning. `aggressive` promises "read the diff"; this needs
|
|
1276
|
+
// "decide whether the order mattered", which is a different question.
|
|
1277
|
+
const reorder = boolFlag(args, 'reorder') ? reorderForCache(original, {
|
|
1278
|
+
// A prefix below the model's cacheable minimum caches nothing at all, so a
|
|
1279
|
+
// rearrangement that does not get it over the line buys nothing and there is
|
|
1280
|
+
// no reason to hand the author a diff for it.
|
|
1281
|
+
//
|
|
1282
|
+
// `undefined` when the catalogue does not know the minimum, which `reorder`
|
|
1283
|
+
// reads as "no floor to clear". That is the right way to be wrong here: the
|
|
1284
|
+
// author asked for the rearrangement explicitly, and withholding it on a
|
|
1285
|
+
// guess about a threshold nobody knows would be refusing to do the thing
|
|
1286
|
+
// they asked for on no evidence.
|
|
1287
|
+
minPrefixTokens: getModel(usage.model).cacheMinTokens ?? undefined,
|
|
1288
|
+
}) : null;
|
|
1289
|
+
const prompt = reorder?.text ?? original;
|
|
1290
|
+
|
|
1291
|
+
let result = optimize(prompt, {
|
|
1292
|
+
level,
|
|
1293
|
+
usage,
|
|
1294
|
+
locale,
|
|
1295
|
+
disableRules,
|
|
1296
|
+
pricing,
|
|
1297
|
+
});
|
|
1298
|
+
|
|
1299
|
+
// The diff has to show the move. Optimising the reordered text means
|
|
1300
|
+
// `result.original` is the rearrangement, so a diff against it would show only
|
|
1301
|
+
// the deletions — and hide the one change the report just told you to review.
|
|
1302
|
+
if (reorder !== null && reorder.moved.length > 0) {
|
|
1303
|
+
result = { ...result, original };
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
let examplesReview: ExampleReview | null = null;
|
|
1307
|
+
let suggestions: SuggestResult | null = null;
|
|
1308
|
+
|
|
1309
|
+
// A flag that quietly does nothing is the same failure as a typo'd flag being
|
|
1310
|
+
// accepted, which this CLI already refuses.
|
|
1311
|
+
if (boolFlag(args, 'apply-suggestions') && !boolFlag(args, 'suggest')) {
|
|
1312
|
+
throw new Error(t.errors.applyNeedsSuggest());
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1315
|
+
if (boolFlag(args, 'suggest')) {
|
|
1316
|
+
const base = providerFromEnv();
|
|
1317
|
+
if (!base) throw new Error(t.errors.llmNotConfigured());
|
|
1318
|
+
|
|
1319
|
+
/**
|
|
1320
|
+
* Opt-in, like everything else here that touches a model.
|
|
1321
|
+
*
|
|
1322
|
+
* A cache hit returns what the model said last time, and a model is not a
|
|
1323
|
+
* pure function — answering from a week-old response without being asked
|
|
1324
|
+
* would be a surprise in a tool that already makes you opt in twice to let
|
|
1325
|
+
* one edit your prompt.
|
|
1326
|
+
*
|
|
1327
|
+
* The saving is not the API's prompt-caching discount, which cannot apply:
|
|
1328
|
+
* the only stable prefix is a 291-token system prompt, below every model's
|
|
1329
|
+
* minimum cacheable prefix, so marking it would silently cache nothing.
|
|
1330
|
+
* See `suggest-cache.ts`.
|
|
1331
|
+
*/
|
|
1332
|
+
const cached = boolFlag(args, 'cache-suggestions')
|
|
1333
|
+
? cachingProvider(base, { dir: cacheDir() })
|
|
1334
|
+
: null;
|
|
1335
|
+
const provider = cached ?? base;
|
|
1336
|
+
|
|
1337
|
+
// On the deterministic result rather than the text as written: the rules
|
|
1338
|
+
// have already taken the easy wins, and asking the model to find them again
|
|
1339
|
+
// spends a call to be told what Trazum knew for free.
|
|
1340
|
+
suggestions = await suggestRewrites(result.optimized, provider, { locale });
|
|
1341
|
+
|
|
1342
|
+
// Said out loud, on stderr so it never lands in `--json`. A cache hit
|
|
1343
|
+
// returns last week's answer, and a reader who does not know that will
|
|
1344
|
+
// wonder why the model stopped noticing a phrase they just added.
|
|
1345
|
+
if (cached) console.error(t.cache.used(cached.hits, cached.misses));
|
|
1346
|
+
|
|
1347
|
+
// Opt in twice, deliberately. Listing is safe — nothing changes and the
|
|
1348
|
+
// author reads eight one-line proposals. Applying is a model editing their
|
|
1349
|
+
// prompt, which is the same class of act as `--reorder` and gets the same
|
|
1350
|
+
// treatment: it does not happen because you asked to look.
|
|
1351
|
+
if (boolFlag(args, 'apply-suggestions') && suggestions.suggestions.length > 0) {
|
|
1352
|
+
const rewritten = applyRewrites(result.optimized, suggestions.suggestions);
|
|
1353
|
+
result = {
|
|
1354
|
+
...result,
|
|
1355
|
+
optimized: rewritten,
|
|
1356
|
+
tokensAfter: estimateTokens(rewritten),
|
|
1357
|
+
};
|
|
1358
|
+
result = {
|
|
1359
|
+
...result,
|
|
1360
|
+
tokensSaved: result.tokensBefore - result.tokensAfter,
|
|
1361
|
+
reductionPct:
|
|
1362
|
+
result.tokensBefore > 0
|
|
1363
|
+
? ((result.tokensBefore - result.tokensAfter) / result.tokensBefore) * 100
|
|
1364
|
+
: 0,
|
|
1365
|
+
savings: computeSavings(result.tokensBefore, result.tokensAfter, result.usage, new Date(), pricing),
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
if (boolFlag(args, 'llm')) {
|
|
1371
|
+
const provider = providerFromEnv();
|
|
1372
|
+
if (!provider) {
|
|
1373
|
+
throw new Error(t.errors.llmNotConfigured());
|
|
1374
|
+
}
|
|
1375
|
+
result = await refineWithLlm(result, provider, { locale });
|
|
1376
|
+
|
|
1377
|
+
// A second call, and only when there is something for it to judge:
|
|
1378
|
+
// `reviewExamples` returns null below two examples rather than paying for
|
|
1379
|
+
// a foregone answer. This is the paraphrase case the deterministic
|
|
1380
|
+
// detector refuses to guess at.
|
|
1381
|
+
examplesReview = await reviewExamples(result.optimized, provider);
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
if (boolFlag(args, 'exact-tokens')) {
|
|
1385
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
1386
|
+
if (!apiKey) {
|
|
1387
|
+
throw new Error(t.errors.exactTokensNeedsKey());
|
|
1388
|
+
}
|
|
1389
|
+
result = await withExactTokenCounts(
|
|
1390
|
+
result,
|
|
1391
|
+
countTokensAnthropic({ apiKey, model: result.usage.model }),
|
|
1392
|
+
pricing,
|
|
1393
|
+
);
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
const outPath = stringFlag(args, 'out');
|
|
1397
|
+
if (outPath) {
|
|
1398
|
+
await writeFile(outPath, result.optimized, 'utf8');
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
if (boolFlag(args, 'json')) {
|
|
1402
|
+
// `reorder` goes in whenever the flag was passed, including when nothing
|
|
1403
|
+
// moved. A consumer reading `optimized` is reading text the author did not
|
|
1404
|
+
// write in that order, and it must not have to infer that from the diff.
|
|
1405
|
+
console.log(
|
|
1406
|
+
JSON.stringify(
|
|
1407
|
+
{
|
|
1408
|
+
...result,
|
|
1409
|
+
...(examplesReview ? { examplesReview } : {}),
|
|
1410
|
+
...(reorder ? { reorder } : {}),
|
|
1411
|
+
// Present whenever --suggest was passed, applied or not: a consumer
|
|
1412
|
+
// needs to tell "nothing was proposed" from "proposals are waiting".
|
|
1413
|
+
...(suggestions
|
|
1414
|
+
? { suggestions: { ...suggestions, applied: boolFlag(args, 'apply-suggestions') } }
|
|
1415
|
+
: {}),
|
|
1416
|
+
},
|
|
1417
|
+
null,
|
|
1418
|
+
2,
|
|
1419
|
+
),
|
|
1420
|
+
);
|
|
1421
|
+
return;
|
|
1422
|
+
}
|
|
1423
|
+
|
|
1424
|
+
if (!process.stdout.isTTY && !outPath) {
|
|
1425
|
+
// Redirected to a file or another process: the prompt alone, no chrome.
|
|
1426
|
+
process.stdout.write(result.optimized);
|
|
1427
|
+
// Except that a rearrangement is not chrome. Everything else this command
|
|
1428
|
+
// does is a deletion the diff will show; `--reorder` moves text, and piping
|
|
1429
|
+
// it made both the move and the refusals invisible — which is the one thing
|
|
1430
|
+
// this module promises not to do. One line, on stderr, so the pipe carries
|
|
1431
|
+
// the prompt and nothing else.
|
|
1432
|
+
if (reorder !== null) {
|
|
1433
|
+
console.error(
|
|
1434
|
+
t.report.reorderPiped(
|
|
1435
|
+
reorder.moved.length,
|
|
1436
|
+
reorder.tokensMoved.toLocaleString(t.numberLocale),
|
|
1437
|
+
reorder.declined.length,
|
|
1438
|
+
),
|
|
1439
|
+
);
|
|
1440
|
+
}
|
|
1441
|
+
return;
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
// Tokens-only when the host bills by subscription: there is no bill to
|
|
1445
|
+
// reduce, so a monthly figure would be arithmetic about tokens dressed as
|
|
1446
|
+
// money. Either flag overrides it, because the host says where *Trazum* runs
|
|
1447
|
+
// and not where the prompt goes — somebody editing a production prompt inside
|
|
1448
|
+
// Cursor wants the dollars, and they should not have to leave the editor to
|
|
1449
|
+
// see them.
|
|
1450
|
+
const host = detectHost();
|
|
1451
|
+
const tokensOnly = boolFlag(args, 'cost')
|
|
1452
|
+
? false
|
|
1453
|
+
: boolFlag(args, 'tokens-only') || host.billing === 'subscription';
|
|
1454
|
+
|
|
1455
|
+
printReport(result, boolFlag(args, 'diff'), t, examplesReview, reorder, tokensOnly, host,
|
|
1456
|
+
suggestions
|
|
1457
|
+
? { result: suggestions, applied: boolFlag(args, 'apply-suggestions'), locale }
|
|
1458
|
+
: null,
|
|
1459
|
+
);
|
|
1460
|
+
if (outPath) {
|
|
1461
|
+
console.log(c.dim(t.report.wroteTo(outPath)));
|
|
1462
|
+
console.log();
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
/** A token counter, plus where its numbers came from. */
|
|
1467
|
+
interface Counter {
|
|
1468
|
+
count: (text: string) => Promise<number>;
|
|
1469
|
+
source: 'heuristic' | 'external';
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
function counterFor(args: Args, t: CliMessages): Counter {
|
|
1473
|
+
if (!boolFlag(args, 'exact-tokens')) {
|
|
1474
|
+
return { count: (text) => Promise.resolve(estimateTokens(text)), source: 'heuristic' };
|
|
1475
|
+
}
|
|
1476
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
1477
|
+
if (!apiKey) throw new Error(t.errors.exactTokensNeedsKey());
|
|
1478
|
+
const exact = countTokensAnthropic({ apiKey });
|
|
1479
|
+
return { count: (text) => exact(text), source: 'external' };
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
interface FileVerdict {
|
|
1483
|
+
path: string;
|
|
1484
|
+
tokens: number;
|
|
1485
|
+
/** null when no budget covers this file. */
|
|
1486
|
+
maxTokens: number | null;
|
|
1487
|
+
/** The config pattern the budget came from, so a surprise can be traced. */
|
|
1488
|
+
pattern: string | null;
|
|
1489
|
+
/** null unless the file is over budget and we worked out the alternative. */
|
|
1490
|
+
optimizedTokens: number | null;
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
async function judgeFile(
|
|
1494
|
+
path: string,
|
|
1495
|
+
text: string,
|
|
1496
|
+
budget: { maxTokens: number; pattern: string | null } | null,
|
|
1497
|
+
counter: Counter,
|
|
1498
|
+
level: RuleLevel,
|
|
1499
|
+
locale: Locale,
|
|
1500
|
+
pricing: PricingCatalogue,
|
|
1501
|
+
): Promise<FileVerdict> {
|
|
1502
|
+
const tokens = await counter.count(text);
|
|
1503
|
+
const maxTokens = budget?.maxTokens ?? null;
|
|
1504
|
+
|
|
1505
|
+
// Over budget: work out whether optimising would be enough, so the CI failure
|
|
1506
|
+
// carries a concrete next step instead of just a red number. Only for the
|
|
1507
|
+
// files that failed — optimising all of them would triple the work of a
|
|
1508
|
+
// directory run that is fine.
|
|
1509
|
+
let optimizedTokens: number | null = null;
|
|
1510
|
+
if (maxTokens !== null && tokens > maxTokens) {
|
|
1511
|
+
optimizedTokens = await counter.count(optimize(text, { level, locale, pricing }).optimized);
|
|
1512
|
+
}
|
|
1513
|
+
|
|
1514
|
+
return { path, tokens, maxTokens, pattern: budget?.pattern ?? null, optimizedTokens };
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
const isOverBudget = (v: FileVerdict): boolean => v.maxTokens !== null && v.tokens > v.maxTokens;
|
|
1518
|
+
|
|
1519
|
+
/**
|
|
1520
|
+
* Token budget for CI: fails (exit code 1) when a prompt busts its budget, so a
|
|
1521
|
+
* template that grows unchecked breaks the build, not the bill.
|
|
1522
|
+
*
|
|
1523
|
+
* Given a directory it checks every prompt inside it against the budgets in
|
|
1524
|
+
* `trazum.config.json`, which is what makes a repository of prompts governable
|
|
1525
|
+
* as a whole rather than one CI step per file.
|
|
1526
|
+
*/
|
|
1527
|
+
async function commandCheck(
|
|
1528
|
+
args: Args,
|
|
1529
|
+
config: TrazumConfig,
|
|
1530
|
+
pricing: PricingCatalogue,
|
|
1531
|
+
t: CliMessages,
|
|
1532
|
+
locale: Locale,
|
|
1533
|
+
): Promise<void> {
|
|
1534
|
+
const target = args.positional[0];
|
|
1535
|
+
const level = levelFlag(args, config, t);
|
|
1536
|
+
const counter = counterFor(args, t);
|
|
1537
|
+
|
|
1538
|
+
// A flag beats the config, as everywhere else. -1 means "not given".
|
|
1539
|
+
const flagBudget = numberFlag(args, 'max-tokens', -1, t);
|
|
1540
|
+
|
|
1541
|
+
const asDirectory = target !== undefined && target !== '-' ? await isDirectory(target) : false;
|
|
1542
|
+
|
|
1543
|
+
if (asDirectory) {
|
|
1544
|
+
await checkDirectory(target!, args, flagBudget, config, counter, level, t, locale, pricing);
|
|
1545
|
+
return;
|
|
1546
|
+
}
|
|
1547
|
+
|
|
1548
|
+
const prompt = await readInput(target, t);
|
|
1549
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
1550
|
+
|
|
1551
|
+
// A single file falls back to the config budget for its own path, so
|
|
1552
|
+
// `trazum check prompts/system.txt` works with no flag once budgets exist.
|
|
1553
|
+
const configBudget = target ? budgetFor(target, config.budgets) : null;
|
|
1554
|
+
const maxTokens = flagBudget >= 0 ? flagBudget : (configBudget?.maxTokens ?? -1);
|
|
1555
|
+
if (maxTokens < 0) throw new Error(t.errors.checkNeedsMaxTokens());
|
|
1556
|
+
|
|
1557
|
+
// A source file carrying markers is not one prompt, it is several. Budgeting
|
|
1558
|
+
// the whole file would measure the code around them, which is not what the
|
|
1559
|
+
// author asked to govern.
|
|
1560
|
+
const embedded = target && target !== '-' && hasMarker(prompt) ? extractPrompts(prompt) : null;
|
|
1561
|
+
if (embedded !== null && (embedded.prompts.length > 0 || embedded.declined.length > 0)) {
|
|
1562
|
+
await checkEmbedded(
|
|
1563
|
+
target!,
|
|
1564
|
+
embedded,
|
|
1565
|
+
{ maxTokens, pattern: flagBudget >= 0 ? null : (configBudget?.pattern ?? null) },
|
|
1566
|
+
args,
|
|
1567
|
+
counter,
|
|
1568
|
+
level,
|
|
1569
|
+
t,
|
|
1570
|
+
locale,
|
|
1571
|
+
pricing,
|
|
1572
|
+
);
|
|
1573
|
+
return;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
const verdict = await judgeFile(
|
|
1577
|
+
target ?? '-',
|
|
1578
|
+
prompt,
|
|
1579
|
+
{ maxTokens, pattern: flagBudget >= 0 ? null : (configBudget?.pattern ?? null) },
|
|
1580
|
+
counter,
|
|
1581
|
+
level,
|
|
1582
|
+
locale,
|
|
1583
|
+
pricing,
|
|
1584
|
+
);
|
|
1585
|
+
const ok = !isOverBudget(verdict);
|
|
1586
|
+
|
|
1587
|
+
// Written before anything can exit, and independently of --json, because the
|
|
1588
|
+
// whole point of the file is to survive a run that failed.
|
|
1589
|
+
await writeMarkdown(args, () =>
|
|
1590
|
+
renderCheckMarkdown({
|
|
1591
|
+
target: target ?? '-',
|
|
1592
|
+
verdicts: [verdict],
|
|
1593
|
+
level,
|
|
1594
|
+
tokenSource: counter.source,
|
|
1595
|
+
truncated: false,
|
|
1596
|
+
t,
|
|
1597
|
+
}),
|
|
1598
|
+
);
|
|
1599
|
+
|
|
1600
|
+
if (boolFlag(args, 'json')) {
|
|
1601
|
+
console.log(
|
|
1602
|
+
JSON.stringify({
|
|
1603
|
+
ok,
|
|
1604
|
+
tokens: verdict.tokens,
|
|
1605
|
+
maxTokens,
|
|
1606
|
+
budgetPattern: verdict.pattern,
|
|
1607
|
+
tokenSource: counter.source,
|
|
1608
|
+
optimizedTokens: verdict.optimizedTokens,
|
|
1609
|
+
wouldFitOptimized:
|
|
1610
|
+
verdict.optimizedTokens !== null ? verdict.optimizedTokens <= maxTokens : null,
|
|
1611
|
+
}),
|
|
1612
|
+
);
|
|
1613
|
+
} else if (ok) {
|
|
1614
|
+
console.log(`${c.green(t.check.okLabel())} ${t.check.ok(n(verdict.tokens), n(maxTokens))}`);
|
|
1615
|
+
} else {
|
|
1616
|
+
console.error(
|
|
1617
|
+
`${c.red(t.check.failedLabel())} ${t.check.failed(n(verdict.tokens), n(maxTokens))}`,
|
|
1618
|
+
);
|
|
1619
|
+
if (verdict.optimizedTokens !== null) {
|
|
1620
|
+
console.error(
|
|
1621
|
+
verdict.optimizedTokens <= maxTokens
|
|
1622
|
+
? t.check.wouldFit(level, n(verdict.optimizedTokens))
|
|
1623
|
+
: t.check.stillTooBig(n(verdict.optimizedTokens)),
|
|
1624
|
+
);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
if (!ok) process.exitCode = 1;
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
async function isDirectory(path: string): Promise<boolean> {
|
|
1632
|
+
return stat(path)
|
|
1633
|
+
.then((info) => info.isDirectory())
|
|
1634
|
+
.catch(() => false);
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
/**
|
|
1638
|
+
* Writes the markdown report, if one was asked for.
|
|
1639
|
+
*
|
|
1640
|
+
* Takes a thunk so a run without `--markdown-out` never pays to render it, and
|
|
1641
|
+
* is called before any `process.exitCode` is set: a report that only appears
|
|
1642
|
+
* when the check passed is a report nobody needs.
|
|
1643
|
+
*
|
|
1644
|
+
* A failure to write is reported and swallowed. The exit code belongs to the
|
|
1645
|
+
* budget, not to the reporting — a full disk on a CI runner must not turn a
|
|
1646
|
+
* passing check into a failing build, and it must certainly not turn a failing
|
|
1647
|
+
* one into a confusing one.
|
|
1648
|
+
*/
|
|
1649
|
+
async function writeMarkdown(args: Args, render: () => string): Promise<void> {
|
|
1650
|
+
const path = stringFlag(args, 'markdown-out');
|
|
1651
|
+
if (!path) return;
|
|
1652
|
+
|
|
1653
|
+
try {
|
|
1654
|
+
const body = fitWithin(
|
|
1655
|
+
render(),
|
|
1656
|
+
MAX_SUMMARY_CHARS,
|
|
1657
|
+
'\n_Trimmed: the report is larger than a step summary can hold._',
|
|
1658
|
+
);
|
|
1659
|
+
await writeFile(path, `${body}\n`, 'utf8');
|
|
1660
|
+
} catch (error) {
|
|
1661
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1662
|
+
console.error(c.yellow(`Could not write ${path}: ${message}`));
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
/**
|
|
1667
|
+
* Writes the OTLP payload, if one was asked for.
|
|
1668
|
+
*
|
|
1669
|
+
* Same shape and same posture as `writeMarkdown`: a thunk so a run without the
|
|
1670
|
+
* flag never pays to build it, and a write failure is reported and swallowed. A
|
|
1671
|
+
* full disk on a metrics runner must not turn a survey into a failure — the
|
|
1672
|
+
* survey is the thing somebody asked for, and the metrics are a copy of it.
|
|
1673
|
+
*/
|
|
1674
|
+
async function writeOtlp(args: Args, build: () => unknown): Promise<void> {
|
|
1675
|
+
const path = stringFlag(args, 'otlp-out');
|
|
1676
|
+
if (!path) return;
|
|
1677
|
+
|
|
1678
|
+
try {
|
|
1679
|
+
await writeFile(path, `${JSON.stringify(build(), null, 2)}\n`, 'utf8');
|
|
1680
|
+
} catch (error) {
|
|
1681
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1682
|
+
console.error(c.yellow(`Could not write ${path}: ${message}`));
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
/**
|
|
1687
|
+
* Checks every prompt under a directory.
|
|
1688
|
+
*
|
|
1689
|
+
* Two decisions worth naming. **A file with no budget is listed, not hidden**:
|
|
1690
|
+
* silently skipping it would let a prompt sit outside every pattern for months
|
|
1691
|
+
* while the report says everything is fine. And **finding no budget at all is
|
|
1692
|
+
* an error**, because "checked 40 files, 0 failures" from a run that measured
|
|
1693
|
+
* nothing is the most misleading output this tool could produce.
|
|
1694
|
+
*/
|
|
1695
|
+
/**
|
|
1696
|
+
* Budgets each prompt marked inside a source file.
|
|
1697
|
+
*
|
|
1698
|
+
* The budget applies per prompt, not to the file: a file holding four prompts is
|
|
1699
|
+
* four things to govern, and summing them would fail a build because somebody
|
|
1700
|
+
* added a fifth short one.
|
|
1701
|
+
*
|
|
1702
|
+
* Declined markers are reported before the verdicts and are **a failure**, not a
|
|
1703
|
+
* note. The author marked a prompt to have it governed; if Trazum cannot read it
|
|
1704
|
+
* then it is not governed, and a green build saying otherwise is the same lie as
|
|
1705
|
+
* "0 failures" from a run that measured nothing.
|
|
1706
|
+
*/
|
|
1707
|
+
async function checkEmbedded(
|
|
1708
|
+
path: string,
|
|
1709
|
+
extraction: { prompts: ExtractedPrompt[]; declined: DeclinedPrompt[] },
|
|
1710
|
+
budget: { maxTokens: number; pattern: string | null },
|
|
1711
|
+
args: Args,
|
|
1712
|
+
counter: Counter,
|
|
1713
|
+
level: RuleLevel,
|
|
1714
|
+
t: CliMessages,
|
|
1715
|
+
locale: Locale,
|
|
1716
|
+
pricing: PricingCatalogue,
|
|
1717
|
+
): Promise<void> {
|
|
1718
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
1719
|
+
|
|
1720
|
+
const verdicts: FileVerdict[] = [];
|
|
1721
|
+
for (const prompt of extraction.prompts) {
|
|
1722
|
+
verdicts.push(
|
|
1723
|
+
await judgeFile(promptId(path, prompt), prompt.text, budget, counter, level, locale, pricing),
|
|
1724
|
+
);
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
const failures = verdicts.filter(isOverBudget);
|
|
1728
|
+
const ok = failures.length === 0 && extraction.declined.length === 0;
|
|
1729
|
+
|
|
1730
|
+
await writeMarkdown(args, () =>
|
|
1731
|
+
renderCheckMarkdown({
|
|
1732
|
+
target: path,
|
|
1733
|
+
verdicts,
|
|
1734
|
+
level,
|
|
1735
|
+
tokenSource: counter.source,
|
|
1736
|
+
truncated: false,
|
|
1737
|
+
t,
|
|
1738
|
+
}),
|
|
1739
|
+
);
|
|
1740
|
+
|
|
1741
|
+
if (boolFlag(args, 'json')) {
|
|
1742
|
+
console.log(
|
|
1743
|
+
JSON.stringify(
|
|
1744
|
+
{
|
|
1745
|
+
ok,
|
|
1746
|
+
target: path,
|
|
1747
|
+
embedded: true,
|
|
1748
|
+
prompts: verdicts.map((v) => ({
|
|
1749
|
+
id: v.path,
|
|
1750
|
+
tokens: v.tokens,
|
|
1751
|
+
maxTokens: v.maxTokens,
|
|
1752
|
+
ok: !isOverBudget(v),
|
|
1753
|
+
})),
|
|
1754
|
+
declined: extraction.declined,
|
|
1755
|
+
},
|
|
1756
|
+
null,
|
|
1757
|
+
2,
|
|
1758
|
+
),
|
|
1759
|
+
);
|
|
1760
|
+
if (!ok) process.exitCode = 1;
|
|
1761
|
+
return;
|
|
1762
|
+
}
|
|
1763
|
+
|
|
1764
|
+
console.log();
|
|
1765
|
+
console.log(c.bold(t.check.embeddedHeading(path, extraction.prompts.length)));
|
|
1766
|
+
|
|
1767
|
+
for (const verdict of verdicts) {
|
|
1768
|
+
const over = isOverBudget(verdict);
|
|
1769
|
+
const label = over ? c.red(t.check.failedLabel()) : c.green(t.check.okLabel());
|
|
1770
|
+
console.log(
|
|
1771
|
+
` ${label} ${verdict.path} — ${n(verdict.tokens)}` +
|
|
1772
|
+
(verdict.maxTokens === null ? '' : ` / ${n(verdict.maxTokens)}`),
|
|
1773
|
+
);
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
if (extraction.declined.length > 0) {
|
|
1777
|
+
console.log();
|
|
1778
|
+
console.log(c.red(t.check.declinedHeading(extraction.declined.length)));
|
|
1779
|
+
for (const declined of extraction.declined) {
|
|
1780
|
+
console.log(` ${c.dim(t.check.declinedAt(declined.line, declined.detail))}`);
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
console.log();
|
|
1785
|
+
if (!ok) process.exitCode = 1;
|
|
1786
|
+
}
|
|
1787
|
+
|
|
1788
|
+
/**
|
|
1789
|
+
* The scenario a baseline is recorded under, and the money it implies.
|
|
1790
|
+
*
|
|
1791
|
+
* Shared by `baseline` and the gate so both compute the monthly figure the same
|
|
1792
|
+
* way. `computeSavings` is asked for a before/after where both sides are the
|
|
1793
|
+
* same token count, because what is wanted here is the cost of a total, not a
|
|
1794
|
+
* saving — `perMonth.before.totalUsd` is that number.
|
|
1795
|
+
*/
|
|
1796
|
+
function monthlyCostOf(tokens: number, usage: UsageProfile, pricing: PricingCatalogue): number {
|
|
1797
|
+
return computeSavings(tokens, tokens, usage, new Date(), pricing).perMonth.before.totalUsd;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
/** Today, as the ISO date a baseline records. */
|
|
1801
|
+
function isoDate(): string {
|
|
1802
|
+
return new Date().toISOString().slice(0, 10);
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
/**
|
|
1806
|
+
* `trazum baseline <dir>` — record what the estate costs now.
|
|
1807
|
+
*
|
|
1808
|
+
* Writes the file and says what to do with it. It never gates: recording is not
|
|
1809
|
+
* a verdict, and a command that could fail while writing the thing you would fix
|
|
1810
|
+
* the failure with is a loop.
|
|
1811
|
+
*/
|
|
1812
|
+
async function commandBaseline(
|
|
1813
|
+
args: Args,
|
|
1814
|
+
config: TrazumConfig,
|
|
1815
|
+
pricing: PricingCatalogue,
|
|
1816
|
+
t: CliMessages,
|
|
1817
|
+
locale: Locale,
|
|
1818
|
+
): Promise<void> {
|
|
1819
|
+
const root = args.positional[0] ?? '.';
|
|
1820
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
1821
|
+
const counter = counterFor(args, t);
|
|
1822
|
+
const usage = usageFrom(args, config, t);
|
|
1823
|
+
|
|
1824
|
+
// `level` is irrelevant to a baseline — it records what the prompts cost as
|
|
1825
|
+
// written, not what they would cost optimised — but `scanPrompts` wants one
|
|
1826
|
+
// for the advisory second pass that only runs on an over-budget file. Nothing
|
|
1827
|
+
// here is over budget, because nothing here has a budget.
|
|
1828
|
+
const { verdicts } = await scanPrompts(
|
|
1829
|
+
root,
|
|
1830
|
+
args,
|
|
1831
|
+
-1,
|
|
1832
|
+
config,
|
|
1833
|
+
counter,
|
|
1834
|
+
'safe',
|
|
1835
|
+
t,
|
|
1836
|
+
locale,
|
|
1837
|
+
pricing,
|
|
1838
|
+
);
|
|
1839
|
+
|
|
1840
|
+
const files: Record<string, number> = {};
|
|
1841
|
+
for (const verdict of verdicts) files[verdict.path] = verdict.tokens;
|
|
1842
|
+
const tokens = Object.values(files).reduce((a, b) => a + b, 0);
|
|
1843
|
+
|
|
1844
|
+
const document: BaselineDocument = {
|
|
1845
|
+
version: BASELINE_VERSION,
|
|
1846
|
+
recorded: isoDate(),
|
|
1847
|
+
scenario: usage,
|
|
1848
|
+
pricingReviewed: pricing.lastReviewed,
|
|
1849
|
+
totals: { tokens, monthlyUsd: monthlyCostOf(tokens, usage, pricing) },
|
|
1850
|
+
files,
|
|
1851
|
+
};
|
|
1852
|
+
|
|
1853
|
+
const out = stringFlag(args, 'out') ?? stringFlag(args, 'o') ?? config.baseline?.path ?? BASELINE_FILENAME;
|
|
1854
|
+
await writeFile(out, formatBaseline(document), 'utf8');
|
|
1855
|
+
|
|
1856
|
+
if (boolFlag(args, 'json')) {
|
|
1857
|
+
console.log(JSON.stringify({ path: out, files: verdicts.length, ...document.totals }));
|
|
1858
|
+
return;
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
console.log(`\n${c.green(t.baseline.recorded(out, n(verdicts.length), n(tokens)))}`);
|
|
1862
|
+
console.log(
|
|
1863
|
+
c.dim(
|
|
1864
|
+
t.baseline.recordedMoney(
|
|
1865
|
+
formatUsd(document.totals.monthlyUsd),
|
|
1866
|
+
usage.model,
|
|
1867
|
+
n(usage.callsPerMonth),
|
|
1868
|
+
),
|
|
1869
|
+
),
|
|
1870
|
+
);
|
|
1871
|
+
console.log();
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
/**
|
|
1875
|
+
* Reports a directory against its baseline, and returns whether it passed.
|
|
1876
|
+
*
|
|
1877
|
+
* Returns rather than exiting, so the caller decides how a breach combines with
|
|
1878
|
+
* a busted budget — they are two independent verdicts about the same run and
|
|
1879
|
+
* either one failing has to fail the build.
|
|
1880
|
+
*/
|
|
1881
|
+
function reportBaseline(
|
|
1882
|
+
comparison: BaselineComparison,
|
|
1883
|
+
breached: BaselineBreach[],
|
|
1884
|
+
baseline: BaselineDocument,
|
|
1885
|
+
path: string,
|
|
1886
|
+
usage: UsageProfile,
|
|
1887
|
+
pricing: PricingCatalogue,
|
|
1888
|
+
t: CliMessages,
|
|
1889
|
+
): void {
|
|
1890
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
1891
|
+
const pct = (value: number): string => `${value > 0 ? '+' : ''}${value.toFixed(1)}%`;
|
|
1892
|
+
const signed = (value: number): string => `${value > 0 ? '+' : ''}${n(value)}`;
|
|
1893
|
+
|
|
1894
|
+
console.log(` ${c.bold(t.baseline.heading())}`);
|
|
1895
|
+
|
|
1896
|
+
const headline =
|
|
1897
|
+
comparison.delta === 0
|
|
1898
|
+
? c.dim(t.baseline.unchanged(n(comparison.tokensAfter)))
|
|
1899
|
+
: comparison.delta > 0
|
|
1900
|
+
? t.baseline.grew(n(comparison.delta), pct(comparison.deltaPct), n(comparison.tokensAfter))
|
|
1901
|
+
: t.baseline.shrank(
|
|
1902
|
+
n(-comparison.delta),
|
|
1903
|
+
pct(comparison.deltaPct),
|
|
1904
|
+
n(comparison.tokensAfter),
|
|
1905
|
+
);
|
|
1906
|
+
console.log(
|
|
1907
|
+
` ${breached.length > 0 ? c.red(headline) : comparison.delta < 0 ? c.green(headline) : headline}`,
|
|
1908
|
+
);
|
|
1909
|
+
|
|
1910
|
+
// Only the directions that cost money are itemised. A list of everything that
|
|
1911
|
+
// shrank is a list nobody acts on, and it buries the two lines that matter.
|
|
1912
|
+
for (const [heading, changes] of [
|
|
1913
|
+
[t.baseline.grownHeading(comparison.grown.length), comparison.grown],
|
|
1914
|
+
[t.baseline.addedHeading(comparison.added.length), comparison.added],
|
|
1915
|
+
[t.baseline.removedHeading(comparison.removed.length), comparison.removed],
|
|
1916
|
+
] as Array<[string, BaselineChange[]]>) {
|
|
1917
|
+
if (changes.length === 0) continue;
|
|
1918
|
+
console.log(` ${c.dim(heading)}`);
|
|
1919
|
+
for (const change of changes) {
|
|
1920
|
+
console.log(
|
|
1921
|
+
` ${t.baseline.entry(change.path, n(change.before), n(change.after), signed(change.delta))}`,
|
|
1922
|
+
);
|
|
1923
|
+
}
|
|
1924
|
+
}
|
|
1925
|
+
|
|
1926
|
+
const money = moneyIsComparable(baseline, usage, pricing.lastReviewed);
|
|
1927
|
+
const now = monthlyCostOf(comparison.tokensAfter, usage, pricing);
|
|
1928
|
+
if (money.comparable) {
|
|
1929
|
+
console.log(
|
|
1930
|
+
` ${t.baseline.money(
|
|
1931
|
+
formatUsd(baseline.totals.monthlyUsd),
|
|
1932
|
+
formatUsd(now),
|
|
1933
|
+
formatSignedUsd(now - baseline.totals.monthlyUsd),
|
|
1934
|
+
)}`,
|
|
1935
|
+
);
|
|
1936
|
+
} else {
|
|
1937
|
+
// Two different measurements are not subtracted. Saying which one moved is
|
|
1938
|
+
// more use than a delta that means nothing.
|
|
1939
|
+
console.log(
|
|
1940
|
+
` ${c.yellow(
|
|
1941
|
+
money.pricingChanged
|
|
1942
|
+
? t.baseline.moneyIncomparablePricing(baseline.pricingReviewed, pricing.lastReviewed)
|
|
1943
|
+
: t.baseline.moneyIncomparableScenario(),
|
|
1944
|
+
)}`,
|
|
1945
|
+
);
|
|
1946
|
+
}
|
|
1947
|
+
|
|
1948
|
+
for (const breach of breached) {
|
|
1949
|
+
console.log(
|
|
1950
|
+
` ${c.red(
|
|
1951
|
+
breach.kind === 'tokens'
|
|
1952
|
+
? t.baseline.breachTokens(n(breach.actual), n(breach.limit))
|
|
1953
|
+
: t.baseline.breachPct(pct(breach.actual), `${breach.limit}%`),
|
|
1954
|
+
)}`,
|
|
1955
|
+
);
|
|
1956
|
+
}
|
|
1957
|
+
if (breached.length > 0) console.log(` ${c.dim(t.baseline.reRecord(path))}`);
|
|
1958
|
+
}
|
|
1959
|
+
|
|
1960
|
+
interface PromptScan {
|
|
1961
|
+
verdicts: FileVerdict[];
|
|
1962
|
+
declined: Array<{ path: string; line: number; detail: string }>;
|
|
1963
|
+
truncated: boolean;
|
|
1964
|
+
/** The extensions actually walked, so an error can name them. */
|
|
1965
|
+
extensions: string[];
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
/**
|
|
1969
|
+
* Walks a directory and counts every prompt in it.
|
|
1970
|
+
*
|
|
1971
|
+
* Extracted from `checkDirectory` when `baseline` arrived, because the two
|
|
1972
|
+
* commands have to agree about what a prompt is down to the last token. Two
|
|
1973
|
+
* walks would be two definitions of the estate — a marker convention read one
|
|
1974
|
+
* way here and another way there — and the baseline would then be a record of
|
|
1975
|
+
* files the gate does not check. One walk, one answer, and the budget resolution
|
|
1976
|
+
* comes along for free so `check` still sees exactly what it always did.
|
|
1977
|
+
*/
|
|
1978
|
+
async function scanPrompts(
|
|
1979
|
+
root: string,
|
|
1980
|
+
args: Args,
|
|
1981
|
+
flagBudget: number,
|
|
1982
|
+
config: TrazumConfig,
|
|
1983
|
+
counter: Counter,
|
|
1984
|
+
level: RuleLevel,
|
|
1985
|
+
t: CliMessages,
|
|
1986
|
+
locale: Locale,
|
|
1987
|
+
pricing: PricingCatalogue,
|
|
1988
|
+
): Promise<PromptScan> {
|
|
1989
|
+
// Source files are walked alongside prompt files rather than opted into.
|
|
1990
|
+
// Requiring config to discover a marker somebody just wrote is how `eval` came
|
|
1991
|
+
// to be fully implemented and completely undiscoverable; an unmarked source
|
|
1992
|
+
// file costs one `includes()` and is dropped.
|
|
1993
|
+
const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
|
|
1994
|
+
const { files, truncated } = await walkPrompts(root, { extensions });
|
|
1995
|
+
|
|
1996
|
+
if (files.length === 0) {
|
|
1997
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// --exact-tokens over a directory is one API round trip per file, and another
|
|
2001
|
+
// for each file that fails. `eval` prints its call count before spending
|
|
2002
|
+
// anything for the same reason: a command that looks hung gets killed, and
|
|
2003
|
+
// then nobody trusts it again.
|
|
2004
|
+
if (counter.source === 'external' && !boolFlag(args, 'json')) {
|
|
2005
|
+
console.log(c.dim(t.check.exactCountsCost(files.length)));
|
|
2006
|
+
}
|
|
2007
|
+
|
|
2008
|
+
const verdicts: FileVerdict[] = [];
|
|
2009
|
+
const declined: Array<{ path: string; line: number; detail: string }> = [];
|
|
2010
|
+
|
|
2011
|
+
for (const relativePath of files) {
|
|
2012
|
+
const text = await readFile(join(root, relativePath), 'utf8');
|
|
2013
|
+
// Budgets are keyed on paths as written in the repository, so a pattern like
|
|
2014
|
+
// `prompts/**` has to be matched against the path including the root the
|
|
2015
|
+
// user passed — not against the name relative to it.
|
|
2016
|
+
const keyed = joinPosix(root, relativePath);
|
|
2017
|
+
const fromConfig = budgetFor(keyed, config.budgets);
|
|
2018
|
+
const budget =
|
|
2019
|
+
fromConfig ?? (flagBudget >= 0 ? { maxTokens: flagBudget, pattern: null } : null);
|
|
2020
|
+
|
|
2021
|
+
const isSource = SOURCE_EXTENSIONS.some((ext) => relativePath.toLowerCase().endsWith(ext));
|
|
2022
|
+
if (isSource) {
|
|
2023
|
+
// A source file is only a prompt file if it says so. One that does not is
|
|
2024
|
+
// dropped silently — it was never something the author asked to govern,
|
|
2025
|
+
// and listing it as unbudgeted would bury the files that are.
|
|
2026
|
+
if (!hasMarker(text)) continue;
|
|
2027
|
+
const extraction = extractPrompts(text);
|
|
2028
|
+
for (const prompt of extraction.prompts) {
|
|
2029
|
+
const id = promptId(keyed, prompt);
|
|
2030
|
+
const own = budgetFor(id, config.budgets) ?? budget;
|
|
2031
|
+
verdicts.push(await judgeFile(id, prompt.text, own, counter, level, locale, pricing));
|
|
2032
|
+
}
|
|
2033
|
+
for (const entry of extraction.declined) {
|
|
2034
|
+
declined.push({ path: keyed, line: entry.line, detail: entry.detail });
|
|
2035
|
+
}
|
|
2036
|
+
continue;
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
verdicts.push(await judgeFile(keyed, text, budget, counter, level, locale, pricing));
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
if (verdicts.length === 0 && declined.length === 0) {
|
|
2043
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
return { verdicts, declined, truncated, extensions };
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
async function checkDirectory(
|
|
2050
|
+
root: string,
|
|
2051
|
+
args: Args,
|
|
2052
|
+
flagBudget: number,
|
|
2053
|
+
config: TrazumConfig,
|
|
2054
|
+
counter: Counter,
|
|
2055
|
+
level: RuleLevel,
|
|
2056
|
+
t: CliMessages,
|
|
2057
|
+
locale: Locale,
|
|
2058
|
+
pricing: PricingCatalogue,
|
|
2059
|
+
): Promise<void> {
|
|
2060
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
2061
|
+
const { verdicts, declined, truncated, extensions } = await scanPrompts(
|
|
2062
|
+
root,
|
|
2063
|
+
args,
|
|
2064
|
+
flagBudget,
|
|
2065
|
+
config,
|
|
2066
|
+
counter,
|
|
2067
|
+
level,
|
|
2068
|
+
t,
|
|
2069
|
+
locale,
|
|
2070
|
+
pricing,
|
|
2071
|
+
);
|
|
2072
|
+
|
|
2073
|
+
/**
|
|
2074
|
+
* The baseline gate, when the config declares one and `--no-baseline` did not
|
|
2075
|
+
* switch it off for this run.
|
|
2076
|
+
*
|
|
2077
|
+
* Read before the budget verdict is reported so a missing or malformed file
|
|
2078
|
+
* fails the run loudly rather than after a green summary. A gate the config
|
|
2079
|
+
* asked for and could not run is not a pass: that is the whole reason
|
|
2080
|
+
* `parseBaseline` throws on everything.
|
|
2081
|
+
*/
|
|
2082
|
+
const wantsBaseline = config.baseline !== undefined && boolFlag(args, 'baseline', true);
|
|
2083
|
+
let baselineOutcome: {
|
|
2084
|
+
comparison: BaselineComparison;
|
|
2085
|
+
breached: BaselineBreach[];
|
|
2086
|
+
document: BaselineDocument;
|
|
2087
|
+
path: string;
|
|
2088
|
+
usage: UsageProfile;
|
|
2089
|
+
} | null = null;
|
|
2090
|
+
|
|
2091
|
+
if (wantsBaseline) {
|
|
2092
|
+
const path = config.baseline!.path;
|
|
2093
|
+
let raw: string;
|
|
2094
|
+
try {
|
|
2095
|
+
raw = await readFile(path, 'utf8');
|
|
2096
|
+
} catch {
|
|
2097
|
+
throw new Error(t.errors.baselineMissing(path));
|
|
2098
|
+
}
|
|
2099
|
+
if (Buffer.byteLength(raw) > MAX_BASELINE_BYTES) {
|
|
2100
|
+
throw new Error(t.errors.baselineTooBig(path, MAX_BASELINE_BYTES));
|
|
2101
|
+
}
|
|
2102
|
+
const document = parseBaseline(raw, path);
|
|
2103
|
+
const current: Record<string, number> = {};
|
|
2104
|
+
for (const verdict of verdicts) current[verdict.path] = verdict.tokens;
|
|
2105
|
+
const comparison = compareToBaseline(document, current);
|
|
2106
|
+
baselineOutcome = {
|
|
2107
|
+
comparison,
|
|
2108
|
+
breached: breaches(comparison, config.baseline!),
|
|
2109
|
+
document,
|
|
2110
|
+
path,
|
|
2111
|
+
usage: usageFrom(args, config, t),
|
|
2112
|
+
};
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
// A budget ceiling is no longer the only thing that can govern a directory: a
|
|
2116
|
+
// baseline governs it too, and a repository using only a baseline is not an
|
|
2117
|
+
// unmeasured one. Without this, adopting `baseline` alone would fail every run
|
|
2118
|
+
// with "no budget covers anything here".
|
|
2119
|
+
if (!wantsBaseline && verdicts.every((v) => v.maxTokens === null)) {
|
|
2120
|
+
throw new Error(t.errors.noBudgetsApply(root, CONFIG_FILENAME));
|
|
2121
|
+
}
|
|
2122
|
+
|
|
2123
|
+
const failures = verdicts.filter(isOverBudget);
|
|
2124
|
+
|
|
2125
|
+
await writeMarkdown(args, () =>
|
|
2126
|
+
renderCheckMarkdown({
|
|
2127
|
+
target: root,
|
|
2128
|
+
verdicts,
|
|
2129
|
+
level,
|
|
2130
|
+
tokenSource: counter.source,
|
|
2131
|
+
truncated,
|
|
2132
|
+
// The same outcome the exit code was computed from, so a pull-request
|
|
2133
|
+
// comment and a red build can never disagree about whether the branch got
|
|
2134
|
+
// more expensive.
|
|
2135
|
+
baseline: baselineOutcome
|
|
2136
|
+
? {
|
|
2137
|
+
comparison: baselineOutcome.comparison,
|
|
2138
|
+
breached: baselineOutcome.breached,
|
|
2139
|
+
money: {
|
|
2140
|
+
before: baselineOutcome.document.totals.monthlyUsd,
|
|
2141
|
+
after: monthlyCostOf(baselineOutcome.comparison.tokensAfter, baselineOutcome.usage, pricing),
|
|
2142
|
+
comparable: moneyIsComparable(
|
|
2143
|
+
baselineOutcome.document,
|
|
2144
|
+
baselineOutcome.usage,
|
|
2145
|
+
pricing.lastReviewed,
|
|
2146
|
+
).comparable,
|
|
2147
|
+
},
|
|
2148
|
+
path: baselineOutcome.path,
|
|
2149
|
+
}
|
|
2150
|
+
: undefined,
|
|
2151
|
+
t,
|
|
2152
|
+
}),
|
|
2153
|
+
);
|
|
2154
|
+
|
|
2155
|
+
if (boolFlag(args, 'json')) {
|
|
2156
|
+
console.log(
|
|
2157
|
+
JSON.stringify(
|
|
2158
|
+
{
|
|
2159
|
+
ok: failures.length === 0 && declined.length === 0,
|
|
2160
|
+
root,
|
|
2161
|
+
tokenSource: counter.source,
|
|
2162
|
+
truncated,
|
|
2163
|
+
declined,
|
|
2164
|
+
files: verdicts.map((v) => ({
|
|
2165
|
+
path: v.path,
|
|
2166
|
+
tokens: v.tokens,
|
|
2167
|
+
maxTokens: v.maxTokens,
|
|
2168
|
+
budgetPattern: v.pattern,
|
|
2169
|
+
ok: !isOverBudget(v),
|
|
2170
|
+
optimizedTokens: v.optimizedTokens,
|
|
2171
|
+
wouldFitOptimized:
|
|
2172
|
+
v.optimizedTokens !== null && v.maxTokens !== null
|
|
2173
|
+
? v.optimizedTokens <= v.maxTokens
|
|
2174
|
+
: null,
|
|
2175
|
+
})),
|
|
2176
|
+
},
|
|
2177
|
+
null,
|
|
2178
|
+
2,
|
|
2179
|
+
),
|
|
2180
|
+
);
|
|
2181
|
+
if (failures.length > 0 || declined.length > 0) process.exitCode = 1;
|
|
2182
|
+
return;
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
// Every column is sized from the whole set before anything is printed, so the
|
|
2186
|
+
// paths line up down the page whatever the locale calls OK and FAILED, and
|
|
2187
|
+
// whether or not a row has a budget. A ragged table is one nobody scans.
|
|
2188
|
+
const labelWidth = Math.max(t.check.okLabel().length, t.check.failedLabel().length) + 2;
|
|
2189
|
+
const tokenWidth = Math.max(...verdicts.map((v) => n(v.tokens).length));
|
|
2190
|
+
const budgetWidth = Math.max(
|
|
2191
|
+
0,
|
|
2192
|
+
...verdicts.map((v) => (v.maxTokens === null ? 0 : n(v.maxTokens).length)),
|
|
2193
|
+
);
|
|
2194
|
+
|
|
2195
|
+
console.log();
|
|
2196
|
+
console.log(c.bold(t.check.directoryHeading(root, verdicts.length)));
|
|
2197
|
+
console.log();
|
|
2198
|
+
|
|
2199
|
+
for (const verdict of verdicts) {
|
|
2200
|
+
const tokens = n(verdict.tokens).padStart(tokenWidth);
|
|
2201
|
+
|
|
2202
|
+
if (verdict.maxTokens === null) {
|
|
2203
|
+
// Blanks where " / <budget>" would be, so the path column does not shift.
|
|
2204
|
+
console.log(
|
|
2205
|
+
` ${c.dim('—'.padEnd(labelWidth))}${tokens}${' '.repeat(3 + budgetWidth)} ` +
|
|
2206
|
+
`${verdict.path} ${c.dim(t.check.noBudget())}`,
|
|
2207
|
+
);
|
|
2208
|
+
continue;
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
const over = isOverBudget(verdict);
|
|
2212
|
+
const plain = over ? t.check.failedLabel() : t.check.okLabel();
|
|
2213
|
+
const label = (over ? c.red(plain) : c.green(plain)) + ' '.repeat(labelWidth - plain.length);
|
|
2214
|
+
const budget = n(verdict.maxTokens).padEnd(budgetWidth);
|
|
2215
|
+
console.log(` ${label}${tokens} / ${budget} ${verdict.path}`);
|
|
2216
|
+
|
|
2217
|
+
if (over && verdict.optimizedTokens !== null) {
|
|
2218
|
+
console.log(
|
|
2219
|
+
` ${' '.repeat(labelWidth)}${c.dim(
|
|
2220
|
+
verdict.optimizedTokens <= verdict.maxTokens
|
|
2221
|
+
? t.check.wouldFit(level, n(verdict.optimizedTokens))
|
|
2222
|
+
: t.check.stillTooBig(n(verdict.optimizedTokens)),
|
|
2223
|
+
)}`,
|
|
2224
|
+
);
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2228
|
+
// A marker Trazum could not read is a failure, not a footnote. The author
|
|
2229
|
+
// marked that prompt to have it governed; it is not being governed, and a
|
|
2230
|
+
// green summary alongside would be the same lie as "0 failures" from a run
|
|
2231
|
+
// that measured nothing.
|
|
2232
|
+
if (declined.length > 0) {
|
|
2233
|
+
console.log();
|
|
2234
|
+
console.log(c.red(t.check.declinedHeading(declined.length)));
|
|
2235
|
+
for (const entry of declined) {
|
|
2236
|
+
console.log(` ${c.dim(`${entry.path} ${t.check.declinedAt(entry.line, entry.detail)}`)}`);
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
console.log();
|
|
2241
|
+
const summary = t.check.directorySummary(failures.length, verdicts.length);
|
|
2242
|
+
// Three independent verdicts about one run: a busted budget, an unreadable
|
|
2243
|
+
// marker, and drift past the baseline. Any of them failing fails the build —
|
|
2244
|
+
// an && here would let a breach ride out on a green budget.
|
|
2245
|
+
// The summary sentence counts budgets, so its colour follows budgets. `bad`
|
|
2246
|
+
// is the run's verdict and folds in the baseline: three independent findings
|
|
2247
|
+
// about one run, and any of them failing fails the build. An && here would let
|
|
2248
|
+
// a breach ride out on a green budget.
|
|
2249
|
+
const budgetBad = failures.length > 0 || declined.length > 0;
|
|
2250
|
+
const bad = budgetBad || (baselineOutcome?.breached.length ?? 0) > 0;
|
|
2251
|
+
console.log(` ${budgetBad ? c.red(summary) : c.green(summary)}`);
|
|
2252
|
+
if (truncated) console.log(` ${c.yellow(t.check.walkTruncated())}`);
|
|
2253
|
+
|
|
2254
|
+
// After the per-file summary, because the two answer different questions and
|
|
2255
|
+
// the wider one reads last: budgets are about files, the baseline is about the
|
|
2256
|
+
// repository. Printing it first put "All 2 within budget" underneath a failed
|
|
2257
|
+
// gate, which reads as a contradiction.
|
|
2258
|
+
if (baselineOutcome && !boolFlag(args, 'json')) {
|
|
2259
|
+
console.log();
|
|
2260
|
+
reportBaseline(
|
|
2261
|
+
baselineOutcome.comparison,
|
|
2262
|
+
baselineOutcome.breached,
|
|
2263
|
+
baselineOutcome.document,
|
|
2264
|
+
baselineOutcome.path,
|
|
2265
|
+
baselineOutcome.usage,
|
|
2266
|
+
pricing,
|
|
2267
|
+
t,
|
|
2268
|
+
);
|
|
2269
|
+
}
|
|
2270
|
+
console.log();
|
|
2271
|
+
|
|
2272
|
+
if (bad) process.exitCode = 1;
|
|
2273
|
+
}
|
|
2274
|
+
|
|
2275
|
+
/** Joins two path fragments for display and glob matching, always with `/`. */
|
|
2276
|
+
function joinPosix(root: string, relativePath: string): string {
|
|
2277
|
+
const trimmed = root.replace(/[\\/]+$/, '').replace(/\\/g, '/');
|
|
2278
|
+
if (trimmed === '' || trimmed === '.') return relativePath;
|
|
2279
|
+
return `${trimmed}/${relativePath}`;
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
|
|
2283
|
+
/**
|
|
2284
|
+
* Runs both prompt versions over a set of inputs and reports whether the
|
|
2285
|
+
* optimisation changed the answers.
|
|
2286
|
+
*
|
|
2287
|
+
* This is the only command that spends real money, and it spends it three
|
|
2288
|
+
* times per case: the original twice to measure the model's own variance, the
|
|
2289
|
+
* optimised once. The doubled original is what makes the answer mean anything
|
|
2290
|
+
* — without it, "diverged on 3 of 10" could be better than the original
|
|
2291
|
+
* manages against itself. The cost is printed before any call goes out.
|
|
2292
|
+
*/
|
|
2293
|
+
/**
|
|
2294
|
+
* `trazum prune <file> --cases <file>` — which few-shot examples earn their tokens.
|
|
2295
|
+
*
|
|
2296
|
+
* The most expensive command here, and the only one that says what it will cost
|
|
2297
|
+
* and then stops. `eval` spends `3 × cases`, which is predictable enough to just
|
|
2298
|
+
* do. This spends `(2 + examples) × cases`, which for a nine-example prompt over
|
|
2299
|
+
* twenty cases is 220 calls — the sort of number somebody should agree to rather
|
|
2300
|
+
* than discover in a bill. So it prints the figure and requires `--yes`.
|
|
2301
|
+
*
|
|
2302
|
+
* The wording of the output matters as much as the measurement. An example whose
|
|
2303
|
+
* removal changes nothing **on these inputs** is not an example to delete: it may
|
|
2304
|
+
* exist for the boundary case somebody hit in production last March, which these
|
|
2305
|
+
* twenty cases do not contain. The report says "no effect on these inputs" and
|
|
2306
|
+
* never "delete this", and nothing here edits the prompt.
|
|
2307
|
+
*/
|
|
2308
|
+
async function commandPrune(args: Args, t: CliMessages): Promise<void> {
|
|
2309
|
+
const prompt = await readInput(args.positional[0], t);
|
|
2310
|
+
|
|
2311
|
+
const casesPath = stringFlag(args, 'cases');
|
|
2312
|
+
if (!casesPath) throw new Error(t.errors.evalNeedsCases());
|
|
2313
|
+
const inputs = parseCases(await readFile(casesPath, 'utf8'));
|
|
2314
|
+
if (inputs.length === 0) throw new Error(t.errors.evalNoCases(casesPath));
|
|
2315
|
+
|
|
2316
|
+
const examples = findExamples(prompt, estimateTokens);
|
|
2317
|
+
if (examples.length < 2) throw new Error(t.prune.needsExamples());
|
|
2318
|
+
|
|
2319
|
+
const calls = plannedCalls(examples.length, inputs.length);
|
|
2320
|
+
|
|
2321
|
+
// Printed before the key is even looked up, so somebody weighing it up does not
|
|
2322
|
+
// need a configured provider to see the number.
|
|
2323
|
+
console.log();
|
|
2324
|
+
console.log(c.bold(t.prune.estimate(examples.length, inputs.length, calls)));
|
|
2325
|
+
|
|
2326
|
+
if (!boolFlag(args, 'yes')) {
|
|
2327
|
+
console.log(c.yellow(` ${t.prune.needsConsent()}`));
|
|
2328
|
+
return;
|
|
2329
|
+
}
|
|
2330
|
+
|
|
2331
|
+
const provider = providerFromEnv();
|
|
2332
|
+
if (!provider) throw new Error(t.errors.llmNotConfigured());
|
|
2333
|
+
|
|
2334
|
+
const report = await pruneExamples(prompt, inputs, provider, {
|
|
2335
|
+
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
2336
|
+
});
|
|
2337
|
+
|
|
2338
|
+
if (boolFlag(args, 'json')) {
|
|
2339
|
+
console.log(JSON.stringify(report, null, 2));
|
|
2340
|
+
return;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
const pct = (value: number): string => `${(value * 100).toFixed(0)}%`;
|
|
2344
|
+
console.log();
|
|
2345
|
+
console.log(c.bold(t.prune.heading(provider.model)));
|
|
2346
|
+
console.log(` ${c.dim(t.prune.selfAgreement(pct(report.selfAgreement)))}`);
|
|
2347
|
+
console.log();
|
|
2348
|
+
|
|
2349
|
+
for (const contribution of report.contributions) {
|
|
2350
|
+
/**
|
|
2351
|
+
* The mark points at what the reader can act on, which is the *recoverable*
|
|
2352
|
+
* ones — and the first draft had it backwards, putting a green tick beside
|
|
2353
|
+
* "0% agreement without it". That reads as approval next to the one line
|
|
2354
|
+
* meaning "this example is load-bearing, leave it alone". Only visible by
|
|
2355
|
+
* running it.
|
|
2356
|
+
*/
|
|
2357
|
+
const needed = contribution.verdict === 'diverges';
|
|
2358
|
+
const unknown = contribution.verdict === 'inconclusive';
|
|
2359
|
+
const mark = unknown ? c.yellow('?') : needed ? c.dim('·') : c.green('→');
|
|
2360
|
+
const label = unknown
|
|
2361
|
+
? t.prune.verdictUnknown()
|
|
2362
|
+
: needed
|
|
2363
|
+
? t.prune.verdictNeeded()
|
|
2364
|
+
: t.prune.verdictRecoverable();
|
|
2365
|
+
|
|
2366
|
+
console.log(
|
|
2367
|
+
` ${mark} ${t.prune.line(contribution.index + 1, contribution.tokens, pct(contribution.agreementWithout))}`
|
|
2368
|
+
+ ` ${unknown ? c.yellow(label) : needed ? c.dim(label) : c.green(label)}`,
|
|
2369
|
+
);
|
|
2370
|
+
|
|
2371
|
+
/**
|
|
2372
|
+
* The first line that is not the header, because the header is the same on
|
|
2373
|
+
* every block. Printing `contribution.text`'s first non-empty line showed
|
|
2374
|
+
* "Example:" four times over, which identifies nothing — again, only visible
|
|
2375
|
+
* by running it.
|
|
2376
|
+
*/
|
|
2377
|
+
const lines = contribution.text.split('\n').filter((line) => line.trim() !== '');
|
|
2378
|
+
const body = lines.find((line) => !/^\s*(?:#+\s*)?(?:example|ejemplo)\b[\s:.-]*$/i.test(line));
|
|
2379
|
+
console.log(` ${c.dim(truncate((body ?? lines[0] ?? '').trim(), 60))}`);
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
console.log();
|
|
2383
|
+
if (report.recoverableTokens > 0) {
|
|
2384
|
+
console.log(` ${t.prune.recoverable(report.recoverableTokens)}`);
|
|
2385
|
+
}
|
|
2386
|
+
console.log(` ${c.dim(wrap(t.prune.caveat(), 74, ' '))}`);
|
|
2387
|
+
console.log();
|
|
2388
|
+
console.log(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
2389
|
+
}
|
|
2390
|
+
|
|
2391
|
+
async function commandEval(
|
|
2392
|
+
args: Args,
|
|
2393
|
+
config: TrazumConfig,
|
|
2394
|
+
t: CliMessages,
|
|
2395
|
+
locale: Locale,
|
|
2396
|
+
): Promise<void> {
|
|
2397
|
+
const prompt = await readInput(args.positional[0], t);
|
|
2398
|
+
const level = levelFlag(args, config, t);
|
|
2399
|
+
|
|
2400
|
+
const casesPath = stringFlag(args, 'cases');
|
|
2401
|
+
if (!casesPath) throw new Error(t.errors.evalNeedsCases());
|
|
2402
|
+
|
|
2403
|
+
const inputs = parseCases(await readFile(casesPath, 'utf8'));
|
|
2404
|
+
if (inputs.length === 0) throw new Error(t.errors.evalNoCases(casesPath));
|
|
2405
|
+
|
|
2406
|
+
// Export short-circuits before the provider is even looked up. Writing a
|
|
2407
|
+
// suite for somebody else's harness must not require a key or spend a call:
|
|
2408
|
+
// the whole point is to hand the run over.
|
|
2409
|
+
const exportTo = stringFlag(args, 'export');
|
|
2410
|
+
if (exportTo !== undefined) {
|
|
2411
|
+
await exportEvalSuite(exportTo, prompt, inputs, { args, config, level, locale, t });
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
const provider = providerFromEnv();
|
|
2416
|
+
if (!provider) throw new Error(t.errors.llmNotConfigured());
|
|
2417
|
+
|
|
2418
|
+
const optimized = optimize(prompt, { level, locale }).optimized;
|
|
2419
|
+
if (optimized === prompt) {
|
|
2420
|
+
console.log(c.yellow(t.eval.nothingToCompare()));
|
|
2421
|
+
return;
|
|
2422
|
+
}
|
|
2423
|
+
|
|
2424
|
+
console.log();
|
|
2425
|
+
console.log(c.dim(t.eval.starting(inputs.length, inputs.length * 3, provider.model)));
|
|
2426
|
+
|
|
2427
|
+
const report = await evaluate(prompt, optimized, inputs, provider, {
|
|
2428
|
+
concurrency: numberFlag(args, 'concurrency', 3, t),
|
|
2429
|
+
});
|
|
2430
|
+
|
|
2431
|
+
if (boolFlag(args, 'json')) {
|
|
2432
|
+
console.log(JSON.stringify(report, null, 2));
|
|
2433
|
+
return;
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
const pct = (value: number): string => `${(value * 100).toFixed(0)}%`;
|
|
2437
|
+
console.log();
|
|
2438
|
+
console.log(c.bold(t.eval.heading()));
|
|
2439
|
+
console.log(` ${t.eval.selfAgreement(pct(report.selfAgreement))}`);
|
|
2440
|
+
console.log(` ${t.eval.crossAgreement(pct(report.crossAgreement))}`);
|
|
2441
|
+
console.log();
|
|
2442
|
+
|
|
2443
|
+
const verdict = t.eval.verdict(report.verdict);
|
|
2444
|
+
const paint =
|
|
2445
|
+
report.verdict === 'diverges'
|
|
2446
|
+
? c.red
|
|
2447
|
+
: report.verdict === 'inconclusive'
|
|
2448
|
+
? c.yellow
|
|
2449
|
+
: c.green;
|
|
2450
|
+
console.log(` ${paint(verdict.label)}`);
|
|
2451
|
+
console.log(` ${c.dim(wrap(verdict.detail, 74, ' '))}`);
|
|
2452
|
+
|
|
2453
|
+
// The worst cases first: if anything broke, it is what the reader came for.
|
|
2454
|
+
const worst = [...report.cases]
|
|
2455
|
+
.sort((a, b) => a.crossSimilarity - b.crossSimilarity)
|
|
2456
|
+
.slice(0, 3)
|
|
2457
|
+
.filter((entry) => entry.crossSimilarity < 0.999);
|
|
2458
|
+
|
|
2459
|
+
if (worst.length > 0) {
|
|
2460
|
+
console.log();
|
|
2461
|
+
console.log(c.bold(t.eval.mostChanged()));
|
|
2462
|
+
for (const entry of worst) {
|
|
2463
|
+
console.log(` ${c.dim(truncate(entry.input, 62))}`);
|
|
2464
|
+
console.log(
|
|
2465
|
+
` ${t.eval.caseAgreement(pct(entry.crossSimilarity), pct(entry.selfSimilarity))}`,
|
|
2466
|
+
);
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2469
|
+
|
|
2470
|
+
console.log();
|
|
2471
|
+
console.log(c.dim(` ${t.eval.callsMade(report.callsMade)}`));
|
|
2472
|
+
console.log();
|
|
2473
|
+
|
|
2474
|
+
if (report.verdict === 'diverges') process.exitCode = 1;
|
|
2475
|
+
}
|
|
2476
|
+
|
|
2477
|
+
/** One case per line, or a JSON array of strings. Blank lines and # comments ignored. */
|
|
2478
|
+
/**
|
|
2479
|
+
* Writes a before/after suite for an external harness.
|
|
2480
|
+
*
|
|
2481
|
+
* `trazum eval` measures semantic agreement, which is the question Trazum is
|
|
2482
|
+
* qualified to ask and not the one a team needs answered before shipping.
|
|
2483
|
+
* Theirs is whether the classifier still hits 94% — an assertion about their
|
|
2484
|
+
* task, which this tool has no business inventing. So the suite is handed over
|
|
2485
|
+
* with both prompts and every case wired up, and the assertions left blank on
|
|
2486
|
+
* purpose.
|
|
2487
|
+
*/
|
|
2488
|
+
async function exportEvalSuite(
|
|
2489
|
+
format: string,
|
|
2490
|
+
prompt: string,
|
|
2491
|
+
inputs: string[],
|
|
2492
|
+
context: {
|
|
2493
|
+
args: Args;
|
|
2494
|
+
config: TrazumConfig;
|
|
2495
|
+
level: RuleLevel;
|
|
2496
|
+
locale: Locale;
|
|
2497
|
+
t: CliMessages;
|
|
2498
|
+
},
|
|
2499
|
+
): Promise<void> {
|
|
2500
|
+
const { args, config, level, locale, t } = context;
|
|
2501
|
+
|
|
2502
|
+
if (format !== 'promptfoo') {
|
|
2503
|
+
throw new Error(t.errors.unknownExportFormat(format, 'promptfoo'));
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
const optimized = optimize(prompt, { level, locale }).optimized;
|
|
2507
|
+
if (optimized === prompt) {
|
|
2508
|
+
// Two identical prompts is a suite that can only ever report "no change",
|
|
2509
|
+
// and an hour of somebody's API budget to find that out.
|
|
2510
|
+
console.log(c.yellow(t.eval.nothingToCompare()));
|
|
2511
|
+
return;
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
const usage = usageFrom(args, config, t);
|
|
2515
|
+
const { config: suite, warnings } = toPromptfoo(prompt, optimized, inputs, {
|
|
2516
|
+
model: usage.model,
|
|
2517
|
+
level,
|
|
2518
|
+
});
|
|
2519
|
+
const body = `${JSON.stringify(suite, null, 2)}\n`;
|
|
2520
|
+
|
|
2521
|
+
const outPath = stringFlag(args, 'out');
|
|
2522
|
+
if (outPath) {
|
|
2523
|
+
await writeFile(outPath, body, 'utf8');
|
|
2524
|
+
} else {
|
|
2525
|
+
process.stdout.write(body);
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
// Warnings to stderr, so a redirected suite is the suite alone — and so they
|
|
2529
|
+
// are still seen when it is.
|
|
2530
|
+
if (warnings.length > 0) {
|
|
2531
|
+
console.error();
|
|
2532
|
+
console.error(t.eval.exportWarnings(warnings.length));
|
|
2533
|
+
for (const warning of warnings) console.error(` ${warning.detail}`);
|
|
2534
|
+
}
|
|
2535
|
+
if (outPath) {
|
|
2536
|
+
console.error();
|
|
2537
|
+
const seeded = (suite as { defaultTest?: { assert?: unknown[] } }).defaultTest?.assert?.length ?? 0;
|
|
2538
|
+
console.error(t.eval.exportWrote(outPath, inputs.length, seeded));
|
|
2539
|
+
}
|
|
2540
|
+
}
|
|
2541
|
+
|
|
2542
|
+
function parseCases(raw: string): string[] {
|
|
2543
|
+
const trimmed = raw.trim();
|
|
2544
|
+
if (trimmed.startsWith('[')) {
|
|
2545
|
+
try {
|
|
2546
|
+
const parsed: unknown = JSON.parse(trimmed);
|
|
2547
|
+
if (Array.isArray(parsed)) {
|
|
2548
|
+
return parsed.filter((value): value is string => typeof value === 'string');
|
|
2549
|
+
}
|
|
2550
|
+
} catch {
|
|
2551
|
+
// Fall through to line mode: a file that merely starts with "[" is more
|
|
2552
|
+
// likely a prompt than a broken JSON document.
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
return trimmed
|
|
2556
|
+
.split('\n')
|
|
2557
|
+
.map((line) => line.trim())
|
|
2558
|
+
.filter((line) => line.length > 0 && !line.startsWith('#'));
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
|
|
2562
|
+
/**
|
|
2563
|
+
* Compares two versions of a prompt. Built for a pull request: it reports by
|
|
2564
|
+
* default and only fails the build when a growth limit was explicitly asked
|
|
2565
|
+
* for, because a tool that fails a build nobody armed gets removed from the
|
|
2566
|
+
* pipeline rather than fixed.
|
|
2567
|
+
*/
|
|
2568
|
+
async function commandDiff(
|
|
2569
|
+
args: Args,
|
|
2570
|
+
config: TrazumConfig,
|
|
2571
|
+
pricing: PricingCatalogue,
|
|
2572
|
+
t: CliMessages,
|
|
2573
|
+
locale: Locale,
|
|
2574
|
+
): Promise<void> {
|
|
2575
|
+
const [beforePath, afterPath] = args.positional;
|
|
2576
|
+
if (!beforePath || !afterPath) throw new Error(t.errors.diffNeedsTwoFiles());
|
|
2577
|
+
|
|
2578
|
+
if (boolFlag(args, 'all')) {
|
|
2579
|
+
await diffDirectories(beforePath, afterPath, args, config, pricing, t, locale);
|
|
2580
|
+
return;
|
|
2581
|
+
}
|
|
2582
|
+
|
|
2583
|
+
const [before, after] = await Promise.all([
|
|
2584
|
+
readInput(beforePath, t),
|
|
2585
|
+
readInput(afterPath, t),
|
|
2586
|
+
]);
|
|
2587
|
+
|
|
2588
|
+
const comparison = comparePrompts(before, after, {
|
|
2589
|
+
level: levelFlag(args, config, t),
|
|
2590
|
+
locale,
|
|
2591
|
+
optimizeBoth: boolFlag(args, 'optimized'),
|
|
2592
|
+
usage: usageFrom(args, config, t),
|
|
2593
|
+
pricing,
|
|
2594
|
+
});
|
|
2595
|
+
|
|
2596
|
+
await writeMarkdown(args, () =>
|
|
2597
|
+
renderDiffMarkdown({
|
|
2598
|
+
comparison,
|
|
2599
|
+
beforePath,
|
|
2600
|
+
afterPath,
|
|
2601
|
+
optimized: boolFlag(args, 'optimized'),
|
|
2602
|
+
locale,
|
|
2603
|
+
t,
|
|
2604
|
+
}),
|
|
2605
|
+
);
|
|
2606
|
+
|
|
2607
|
+
if (boolFlag(args, 'json')) {
|
|
2608
|
+
console.log(JSON.stringify(comparison, null, 2));
|
|
2609
|
+
} else {
|
|
2610
|
+
printComparison(comparison, beforePath, afterPath, boolFlag(args, 'optimized'), t);
|
|
2611
|
+
}
|
|
2612
|
+
|
|
2613
|
+
// The gate stays opt-in, and a config file counts as opting in: a repository
|
|
2614
|
+
// that wrote down `"maxGrowth": 25` has armed it as deliberately as a flag
|
|
2615
|
+
// would. What has not changed is that *absent* both, growth alone exits 0.
|
|
2616
|
+
const limit =
|
|
2617
|
+
typeof args.flags.get('max-growth') === 'string'
|
|
2618
|
+
? numberFlag(args, 'max-growth', 0, t)
|
|
2619
|
+
: config.maxGrowth;
|
|
2620
|
+
|
|
2621
|
+
if (limit !== undefined && comparison.tokenDelta > limit) {
|
|
2622
|
+
console.error(`\n${c.red(t.diff.overLimit(comparison.tokenDelta, limit))}`);
|
|
2623
|
+
process.exitCode = 1;
|
|
2624
|
+
}
|
|
2625
|
+
}
|
|
2626
|
+
|
|
2627
|
+
function printComparison(
|
|
2628
|
+
comparison: PromptComparison,
|
|
2629
|
+
beforePath: string,
|
|
2630
|
+
afterPath: string,
|
|
2631
|
+
optimized: boolean,
|
|
2632
|
+
t: CliMessages,
|
|
2633
|
+
): void {
|
|
2634
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
2635
|
+
const grew = comparison.tokenDelta > 0;
|
|
2636
|
+
const paint = grew ? c.red : comparison.tokenDelta < 0 ? c.green : c.dim;
|
|
2637
|
+
const signed = (value: number): string => `${value > 0 ? '+' : ''}${n(value)}`;
|
|
2638
|
+
|
|
2639
|
+
console.log();
|
|
2640
|
+
console.log(c.bold(t.diff.heading(beforePath, afterPath)));
|
|
2641
|
+
if (optimized) console.log(c.dim(` ${t.diff.measuringOptimised()}`));
|
|
2642
|
+
console.log();
|
|
2643
|
+
console.log(
|
|
2644
|
+
` ${n(comparison.tokensBefore)} → ${n(comparison.tokensAfter)} tokens ` +
|
|
2645
|
+
paint(`${signed(comparison.tokenDelta)} (${signed(Math.round(comparison.deltaPct))}%)`),
|
|
2646
|
+
);
|
|
2647
|
+
console.log(
|
|
2648
|
+
` ${t.diff.monthly(
|
|
2649
|
+
formatSignedUsd(comparison.monthlyDeltaUsd),
|
|
2650
|
+
n(comparison.usage.callsPerMonth),
|
|
2651
|
+
getModel(comparison.usage.model).displayName,
|
|
2652
|
+
)}`,
|
|
2653
|
+
);
|
|
2654
|
+
|
|
2655
|
+
const { rules, advisories } = comparison;
|
|
2656
|
+
const copy = getMessages(t.locale).rules;
|
|
2657
|
+
|
|
2658
|
+
if (advisories.appeared.length > 0) {
|
|
2659
|
+
console.log();
|
|
2660
|
+
console.log(c.yellow(` ${t.diff.advisoriesAppeared()}`));
|
|
2661
|
+
for (const id of advisories.appeared) console.log(` ! ${id}`);
|
|
2662
|
+
}
|
|
2663
|
+
if (advisories.resolved.length > 0) {
|
|
2664
|
+
console.log();
|
|
2665
|
+
console.log(c.green(` ${t.diff.advisoriesResolved()}`));
|
|
2666
|
+
for (const id of advisories.resolved) console.log(` ✓ ${id}`);
|
|
2667
|
+
}
|
|
2668
|
+
if (rules.newlyFiring.length > 0) {
|
|
2669
|
+
console.log();
|
|
2670
|
+
console.log(` ${t.diff.rulesNewlyFiring()}`);
|
|
2671
|
+
for (const id of rules.newlyFiring) console.log(` ${c.dim(copy[id].title)}`);
|
|
2672
|
+
}
|
|
2673
|
+
if (rules.noLongerFiring.length > 0) {
|
|
2674
|
+
console.log();
|
|
2675
|
+
console.log(` ${t.diff.rulesNoLongerFiring()}`);
|
|
2676
|
+
for (const id of rules.noLongerFiring) console.log(` ${c.dim(copy[id].title)}`);
|
|
2677
|
+
}
|
|
2678
|
+
|
|
2679
|
+
console.log();
|
|
2680
|
+
}
|
|
2681
|
+
|
|
2682
|
+
// --------------------------------------------------------------------------
|
|
2683
|
+
|
|
2684
|
+
// --------------------------------------------------------------------------
|
|
2685
|
+
// blame
|
|
2686
|
+
// --------------------------------------------------------------------------
|
|
2687
|
+
|
|
2688
|
+
/**
|
|
2689
|
+
* How many revisions to walk unless told otherwise.
|
|
2690
|
+
*
|
|
2691
|
+
* Each one is a `git show` and a token count, so this is a wall-clock budget as
|
|
2692
|
+
* much as a display choice. Twenty is enough to see a trend and fast enough to
|
|
2693
|
+
* feel instant on a normal file.
|
|
2694
|
+
*/
|
|
2695
|
+
/**
|
|
2696
|
+
* How many unbudgeted paths `doctor` names before summarising the rest.
|
|
2697
|
+
*
|
|
2698
|
+
* Capped and *counted*: a survey that prints forty paths buries the finding it
|
|
2699
|
+
* was meant to deliver, and one that silently shows the first eight claims there
|
|
2700
|
+
* were eight.
|
|
2701
|
+
*/
|
|
2702
|
+
const DOCTOR_LIST_LIMIT = 8;
|
|
2703
|
+
|
|
2704
|
+
const BLAME_DEFAULT_LIMIT = 20;
|
|
2705
|
+
const BLAME_MAX_LIMIT = 500;
|
|
2706
|
+
|
|
2707
|
+
interface BlameRow {
|
|
2708
|
+
revision: Revision;
|
|
2709
|
+
/** `null` when the file did not exist at that commit, or held no marked prompt. */
|
|
2710
|
+
tokens: number | null;
|
|
2711
|
+
/** Tokens added since the previous (older) revision. `null` for the first. */
|
|
2712
|
+
delta: number | null;
|
|
2713
|
+
/** The name the file had at that commit, when it differs from today's. */
|
|
2714
|
+
name: string | null;
|
|
2715
|
+
}
|
|
2716
|
+
|
|
2717
|
+
/**
|
|
2718
|
+
* `trazum blame <file>` — what happened to this prompt's cost, and who did it.
|
|
2719
|
+
*
|
|
2720
|
+
* Git already knows who changed a prompt and when. What it does not know is
|
|
2721
|
+
* that a three-line addition to a system prompt at 50,000 calls a month is a
|
|
2722
|
+
* bill, not a diff. This walks the file's history, counts the tokens at each
|
|
2723
|
+
* commit, and puts the two facts on the same line.
|
|
2724
|
+
*
|
|
2725
|
+
* Reads history and nothing else: no writes, no network, and the one place that
|
|
2726
|
+
* runs git is `git.ts`, which is written as if it were the whole attack surface.
|
|
2727
|
+
*/
|
|
2728
|
+
async function commandBlame(
|
|
2729
|
+
args: Args,
|
|
2730
|
+
config: TrazumConfig,
|
|
2731
|
+
pricing: PricingCatalogue,
|
|
2732
|
+
t: CliMessages,
|
|
2733
|
+
): Promise<void> {
|
|
2734
|
+
const target = args.positional[0];
|
|
2735
|
+
if (!target) throw new Error(t.errors.missingInputFile());
|
|
2736
|
+
|
|
2737
|
+
const cwd = process.cwd();
|
|
2738
|
+
const root = repositoryRoot(cwd);
|
|
2739
|
+
if (root === null) {
|
|
2740
|
+
// Two failures with the same symptom, and the distinction is the whole of
|
|
2741
|
+
// the fix: install git, or run this somewhere else.
|
|
2742
|
+
throw new Error(gitAvailable(cwd) ? t.blame.notARepository() : t.blame.gitMissing());
|
|
2743
|
+
}
|
|
2744
|
+
|
|
2745
|
+
const repoPath = pathInRepository(root, resolvePath(cwd, target));
|
|
2746
|
+
if (repoPath === null) throw new Error(t.blame.outsideRepository(target));
|
|
2747
|
+
|
|
2748
|
+
const limit = Math.min(
|
|
2749
|
+
Math.max(1, Math.floor(numberFlag(args, 'limit', BLAME_DEFAULT_LIMIT, t))),
|
|
2750
|
+
BLAME_MAX_LIMIT,
|
|
2751
|
+
);
|
|
2752
|
+
// One extra, so the oldest shown revision still has something to be a change
|
|
2753
|
+
// *from*. Without it the first row reports "added" for a file that existed.
|
|
2754
|
+
const revisions = revisionsFor(repoPath, { cwd: root, max: limit + 1 });
|
|
2755
|
+
if (revisions.length === 0) throw new Error(t.blame.noHistory(repoPath));
|
|
2756
|
+
|
|
2757
|
+
const wanted = stringFlag(args, 'prompt');
|
|
2758
|
+
|
|
2759
|
+
/**
|
|
2760
|
+
* Tokens in the prompt at a commit.
|
|
2761
|
+
*
|
|
2762
|
+
* A source file is measured through the same marker extraction `optimize`
|
|
2763
|
+
* uses, so `blame src/prompts.ts --prompt support` tracks the prompt rather
|
|
2764
|
+
* than the file around it — otherwise every refactor of the imports would
|
|
2765
|
+
* read as prompt growth.
|
|
2766
|
+
*/
|
|
2767
|
+
const names = namesByRevision(repoPath, root, limit + 1);
|
|
2768
|
+
const tokensAt = (revision: Revision): { tokens: number | null; name: string | null } => {
|
|
2769
|
+
// The name at *that* commit, which is not today's name once a rename is in
|
|
2770
|
+
// the history. Reading with today's name returned "did not exist" for every
|
|
2771
|
+
// revision before the move.
|
|
2772
|
+
const name = names.get(revision.sha) ?? null;
|
|
2773
|
+
const path = name ?? repoPath;
|
|
2774
|
+
const text = contentAt(revision.sha, path, root);
|
|
2775
|
+
if (text === null) return { tokens: null, name: null };
|
|
2776
|
+
|
|
2777
|
+
const source = sourceFileOf(path, text, pricing, wanted);
|
|
2778
|
+
return {
|
|
2779
|
+
tokens: estimateTokens(source ? source.text : text),
|
|
2780
|
+
name: name !== null && name !== repoPath ? name : null,
|
|
2781
|
+
};
|
|
2782
|
+
};
|
|
2783
|
+
|
|
2784
|
+
// Oldest first while computing, so a delta is against the revision before it.
|
|
2785
|
+
const measured = revisions
|
|
2786
|
+
.slice()
|
|
2787
|
+
.reverse()
|
|
2788
|
+
.map((revision) => ({ revision, ...tokensAt(revision) }));
|
|
2789
|
+
|
|
2790
|
+
const rows: BlameRow[] = measured.map((entry, index) => {
|
|
2791
|
+
const previous = index > 0 ? measured[index - 1]!.tokens : null;
|
|
2792
|
+
return {
|
|
2793
|
+
revision: entry.revision,
|
|
2794
|
+
tokens: entry.tokens,
|
|
2795
|
+
delta: entry.tokens !== null && previous !== null ? entry.tokens - previous : null,
|
|
2796
|
+
name: entry.name,
|
|
2797
|
+
};
|
|
2798
|
+
});
|
|
2799
|
+
|
|
2800
|
+
// Drop the extra oldest revision now that it has served as a baseline, and
|
|
2801
|
+
// put the newest first: this is a history, and histories are read backwards.
|
|
2802
|
+
const shown = rows.slice(revisions.length > limit ? 1 : 0).reverse();
|
|
2803
|
+
const truncatedHistory = revisions.length > limit;
|
|
2804
|
+
|
|
2805
|
+
await writeMarkdown(args, () =>
|
|
2806
|
+
renderBlameMarkdown({
|
|
2807
|
+
repoPath,
|
|
2808
|
+
rows: shown,
|
|
2809
|
+
truncated: truncatedHistory,
|
|
2810
|
+
netCost: netCostOf(shown, args, config, pricing, t),
|
|
2811
|
+
t,
|
|
2812
|
+
}),
|
|
2813
|
+
);
|
|
2814
|
+
|
|
2815
|
+
printBlame(shown, { repoPath, args, config, pricing, t, truncated: truncatedHistory });
|
|
2816
|
+
}
|
|
2817
|
+
|
|
2818
|
+
/**
|
|
2819
|
+
* What the movement across a history costs per month, or `null`.
|
|
2820
|
+
*
|
|
2821
|
+
* Extracted so the terminal report and the markdown report read one number
|
|
2822
|
+
* rather than each computing it. The file's own doc comment claims a discrepancy
|
|
2823
|
+
* between a pull-request comment and the job log is "impossible by construction";
|
|
2824
|
+
* two copies of this arithmetic is exactly how that claim stops being true.
|
|
2825
|
+
*
|
|
2826
|
+
* Oldest as "before" and newest as "after", so a prompt that grew reports a
|
|
2827
|
+
* negative saving — which is the honest word for it. The sign becomes a `+`/`−`
|
|
2828
|
+
* on the money here rather than being left for the reader.
|
|
2829
|
+
*/
|
|
2830
|
+
function netCostOf(
|
|
2831
|
+
rows: readonly BlameRow[],
|
|
2832
|
+
args: Args,
|
|
2833
|
+
config: TrazumConfig,
|
|
2834
|
+
pricing: PricingCatalogue,
|
|
2835
|
+
t: CliMessages,
|
|
2836
|
+
): { amount: string; modelDisplayName: string; callsPerMonth: number } | null {
|
|
2837
|
+
const measured = rows.filter((r): r is BlameRow & { tokens: number } => r.tokens !== null);
|
|
2838
|
+
const newest = measured[0];
|
|
2839
|
+
const oldest = measured[measured.length - 1];
|
|
2840
|
+
if (!newest || !oldest || newest === oldest || newest.tokens === oldest.tokens) return null;
|
|
2841
|
+
|
|
2842
|
+
const usage = usageFrom(args, config, t);
|
|
2843
|
+
const model = pricing.models.find((m) => m.id === usage.model);
|
|
2844
|
+
if (!model) return null;
|
|
2845
|
+
|
|
2846
|
+
const savings = computeSavings(oldest.tokens, newest.tokens, usage, new Date(), pricing);
|
|
2847
|
+
const monthly = -savings.monthlySavingsUsd;
|
|
2848
|
+
return {
|
|
2849
|
+
amount: `${monthly >= 0 ? '+' : '\u2212'}${formatUsd(Math.abs(monthly))}`,
|
|
2850
|
+
modelDisplayName: model.displayName,
|
|
2851
|
+
callsPerMonth: usage.callsPerMonth,
|
|
2852
|
+
};
|
|
2853
|
+
}
|
|
2854
|
+
|
|
2855
|
+
/**
|
|
2856
|
+
* The report.
|
|
2857
|
+
*
|
|
2858
|
+
* A table, newest first, and then the two things the table alone does not say:
|
|
2859
|
+
* what the whole history added up to in money, and which single commit did the
|
|
2860
|
+
* most damage. "Tokens grew 40%" is a fact; "+310 tokens, Dana, 'add escalation
|
|
2861
|
+
* rules'" is something somebody can go and look at.
|
|
2862
|
+
*/
|
|
2863
|
+
function printBlame(
|
|
2864
|
+
rows: BlameRow[],
|
|
2865
|
+
context: {
|
|
2866
|
+
repoPath: string;
|
|
2867
|
+
args: Args;
|
|
2868
|
+
config: TrazumConfig;
|
|
2869
|
+
pricing: PricingCatalogue;
|
|
2870
|
+
t: CliMessages;
|
|
2871
|
+
truncated: boolean;
|
|
2872
|
+
},
|
|
2873
|
+
): void {
|
|
2874
|
+
const { repoPath, args, config, pricing, t, truncated } = context;
|
|
2875
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
2876
|
+
|
|
2877
|
+
const measured = rows.filter((r): r is BlameRow & { tokens: number } => r.tokens !== null);
|
|
2878
|
+
const newest = measured[0];
|
|
2879
|
+
const oldest = measured[measured.length - 1];
|
|
2880
|
+
|
|
2881
|
+
if (boolFlag(args, 'json')) {
|
|
2882
|
+
console.log(
|
|
2883
|
+
JSON.stringify(
|
|
2884
|
+
{
|
|
2885
|
+
path: repoPath,
|
|
2886
|
+
truncated,
|
|
2887
|
+
revisions: rows.map((row) => ({
|
|
2888
|
+
sha: row.revision.sha,
|
|
2889
|
+
author: row.revision.author,
|
|
2890
|
+
date: row.revision.date,
|
|
2891
|
+
subject: row.revision.subject,
|
|
2892
|
+
tokens: row.tokens,
|
|
2893
|
+
delta: row.delta,
|
|
2894
|
+
...(row.name ? { path: row.name } : {}),
|
|
2895
|
+
})),
|
|
2896
|
+
},
|
|
2897
|
+
null,
|
|
2898
|
+
2,
|
|
2899
|
+
),
|
|
2900
|
+
);
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
|
|
2904
|
+
console.log(`\n${c.bold(t.blame.heading(repoPath, rows.length))}\n`);
|
|
2905
|
+
|
|
2906
|
+
const cols = t.blame.columns;
|
|
2907
|
+
const widths = {
|
|
2908
|
+
when: Math.max(cols.when.length, 10),
|
|
2909
|
+
tokens: Math.max(cols.tokens.length, ...rows.map((r) => (r.tokens === null ? t.blame.goneAt().length : n(r.tokens).length))),
|
|
2910
|
+
change: Math.max(cols.change.length, 7),
|
|
2911
|
+
who: Math.min(20, Math.max(cols.who.length, ...rows.map((r) => r.revision.author.length))),
|
|
2912
|
+
};
|
|
2913
|
+
|
|
2914
|
+
console.log(
|
|
2915
|
+
c.dim(
|
|
2916
|
+
[
|
|
2917
|
+
cols.when.padEnd(widths.when),
|
|
2918
|
+
cols.tokens.padStart(widths.tokens),
|
|
2919
|
+
cols.change.padStart(widths.change),
|
|
2920
|
+
cols.who.padEnd(widths.who),
|
|
2921
|
+
cols.commit,
|
|
2922
|
+
].join(' '),
|
|
2923
|
+
),
|
|
2924
|
+
);
|
|
2925
|
+
|
|
2926
|
+
for (const row of rows) {
|
|
2927
|
+
const when = row.revision.date.slice(0, 10);
|
|
2928
|
+
const tokens = row.tokens === null ? c.dim(t.blame.goneAt()) : n(row.tokens);
|
|
2929
|
+
// A rise is the thing worth seeing, so it is the thing that gets colour.
|
|
2930
|
+
// A fall is good news and does not need to shout.
|
|
2931
|
+
const change =
|
|
2932
|
+
row.delta === null
|
|
2933
|
+
? c.dim(row.tokens === null ? '' : t.blame.addedAt())
|
|
2934
|
+
: row.delta > 0
|
|
2935
|
+
? c.red(`+${n(row.delta)}`)
|
|
2936
|
+
: row.delta < 0
|
|
2937
|
+
? c.green(n(row.delta))
|
|
2938
|
+
: c.dim('·');
|
|
2939
|
+
|
|
2940
|
+
const rawTokens = row.tokens === null ? t.blame.goneAt() : n(row.tokens);
|
|
2941
|
+
const rawChange =
|
|
2942
|
+
row.delta === null
|
|
2943
|
+
? row.tokens === null
|
|
2944
|
+
? ''
|
|
2945
|
+
: t.blame.addedAt()
|
|
2946
|
+
: row.delta > 0
|
|
2947
|
+
? `+${n(row.delta)}`
|
|
2948
|
+
: row.delta < 0
|
|
2949
|
+
? n(row.delta)
|
|
2950
|
+
: '·';
|
|
2951
|
+
|
|
2952
|
+
console.log(
|
|
2953
|
+
[
|
|
2954
|
+
when.padEnd(widths.when),
|
|
2955
|
+
// Padded on the raw text, coloured after: an ANSI escape has length
|
|
2956
|
+
// and padEnd would count it, so every coloured cell would come out
|
|
2957
|
+
// short by exactly the width of the escape sequence.
|
|
2958
|
+
' '.repeat(Math.max(0, widths.tokens - rawTokens.length)) + tokens,
|
|
2959
|
+
' '.repeat(Math.max(0, widths.change - rawChange.length)) + change,
|
|
2960
|
+
truncate(row.revision.author, widths.who).padEnd(widths.who),
|
|
2961
|
+
c.dim(`${row.revision.shortSha} ${truncate(row.revision.subject, 48)}`),
|
|
2962
|
+
].join(' '),
|
|
2963
|
+
);
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
if (truncated) console.log(`\n${c.dim(t.blame.truncated(rows.length))}`);
|
|
2967
|
+
|
|
2968
|
+
const renamed = rows.find((row) => row.name !== null);
|
|
2969
|
+
if (renamed?.name) console.log(c.dim(t.blame.followedRename(renamed.name)));
|
|
2970
|
+
|
|
2971
|
+
if (newest && oldest && newest !== oldest) {
|
|
2972
|
+
const delta = newest.tokens - oldest.tokens;
|
|
2973
|
+
const pct = oldest.tokens === 0 ? '—' : `${delta >= 0 ? '+' : ''}${((delta / oldest.tokens) * 100).toFixed(0)}%`;
|
|
2974
|
+
console.log(
|
|
2975
|
+
`\n${t.blame.net(n(oldest.tokens), n(newest.tokens), `${delta >= 0 ? '+' : ''}${n(delta)}`, pct)}`,
|
|
2976
|
+
);
|
|
2977
|
+
|
|
2978
|
+
// What the movement costs, priced through the same usage profile every
|
|
2979
|
+
// other command uses — so `--calls` and `--model` mean here what they mean
|
|
2980
|
+
// in `optimize`, and a figure from one is comparable with the other. Shared
|
|
2981
|
+
// with the markdown renderer, so the comment and the log cannot disagree.
|
|
2982
|
+
const cost = netCostOf(rows, args, config, pricing, t);
|
|
2983
|
+
if (cost) {
|
|
2984
|
+
console.log(
|
|
2985
|
+
c.dim(t.blame.netCost(cost.amount, cost.modelDisplayName, n(cost.callsPerMonth))),
|
|
2986
|
+
);
|
|
2987
|
+
}
|
|
2988
|
+
}
|
|
2989
|
+
|
|
2990
|
+
// The single worst commit, which is the question the command is really for.
|
|
2991
|
+
const worst = rows
|
|
2992
|
+
.filter((row): row is BlameRow & { delta: number } => row.delta !== null && row.delta > 0)
|
|
2993
|
+
.sort((a, b) => b.delta - a.delta)[0];
|
|
2994
|
+
if (worst) {
|
|
2995
|
+
console.log(`\n${c.bold(t.blame.biggestRise())}`);
|
|
2996
|
+
console.log(
|
|
2997
|
+
` ${t.blame.biggestRiseDetail(
|
|
2998
|
+
n(worst.delta),
|
|
2999
|
+
worst.revision.author,
|
|
3000
|
+
truncate(worst.revision.subject, 60),
|
|
3001
|
+
worst.revision.shortSha,
|
|
3002
|
+
)}`,
|
|
3003
|
+
);
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
console.log(`\n${c.dim(t.blame.estimateNote())}\n`);
|
|
3007
|
+
}
|
|
3008
|
+
|
|
3009
|
+
// --------------------------------------------------------------------------
|
|
3010
|
+
// rank
|
|
3011
|
+
// --------------------------------------------------------------------------
|
|
3012
|
+
|
|
3013
|
+
interface RankedPrompt {
|
|
3014
|
+
path: string;
|
|
3015
|
+
profile: PromptProfile;
|
|
3016
|
+
/** Tokens the deterministic rules would take, at the level asked for. */
|
|
3017
|
+
recoverable: number;
|
|
3018
|
+
/** What those tokens cost per month under the usage profile. */
|
|
3019
|
+
recoverableUsd: number;
|
|
3020
|
+
/** Set when the file is source and its marked prompt was measured. */
|
|
3021
|
+
promptName: string | null;
|
|
3022
|
+
}
|
|
3023
|
+
|
|
3024
|
+
/**
|
|
3025
|
+
* `trazum rank <dir>` — which of these prompts to fix first.
|
|
3026
|
+
*
|
|
3027
|
+
* The obvious design is a complexity score out of a hundred, and it is the
|
|
3028
|
+
* wrong one. A number nobody can reproduce by hand cannot be argued with, and
|
|
3029
|
+
* the weights that combine four measurements into one get tuned until the
|
|
3030
|
+
* ranking looks right — which is fitting the metric to the answer.
|
|
3031
|
+
*
|
|
3032
|
+
* So this sorts on the one quantity that is not a matter of opinion: **what
|
|
3033
|
+
* optimising each prompt would actually save**, obtained by running the rules
|
|
3034
|
+
* rather than by evaluating a formula. The structural measurements are printed
|
|
3035
|
+
* beside it as the *explanation* — "1,204 tokens across 8 sentences" says why a
|
|
3036
|
+
* prompt is worth looking at, and the recoverable figure says whether it is
|
|
3037
|
+
* worth looking at before the other thirty-nine.
|
|
3038
|
+
*/
|
|
3039
|
+
async function commandRank(
|
|
3040
|
+
args: Args,
|
|
3041
|
+
config: TrazumConfig,
|
|
3042
|
+
pricing: PricingCatalogue,
|
|
3043
|
+
t: CliMessages,
|
|
3044
|
+
locale: Locale,
|
|
3045
|
+
): Promise<void> {
|
|
3046
|
+
const root = args.positional[0] ?? '.';
|
|
3047
|
+
const level = levelFlag(args, config, t);
|
|
3048
|
+
const usage = usageFrom(args, config, t);
|
|
3049
|
+
|
|
3050
|
+
const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
|
|
3051
|
+
const { files, truncated } = await walkPrompts(root, { extensions });
|
|
3052
|
+
if (files.length === 0) {
|
|
3053
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
3054
|
+
}
|
|
3055
|
+
|
|
3056
|
+
const ranked: RankedPrompt[] = [];
|
|
3057
|
+
let skipped = 0;
|
|
3058
|
+
|
|
3059
|
+
for (const file of files) {
|
|
3060
|
+
const raw = await readFile(join(root, file), 'utf8');
|
|
3061
|
+
|
|
3062
|
+
// A source file contributes its marked prompt, or nothing. Ranking
|
|
3063
|
+
// `src/prompts.ts` by the size of its imports would put the wrong file at
|
|
3064
|
+
// the top of a list whose whole job is to point somewhere.
|
|
3065
|
+
//
|
|
3066
|
+
// `sourceFileOf` *throws* for a source file with no marker, which is the
|
|
3067
|
+
// right answer for `optimize` — you named that file, and optimising it
|
|
3068
|
+
// would rewrite your code. It is the wrong answer here: one unmarked `.ts`
|
|
3069
|
+
// in a repository would abort the ranking of the other thirty-nine. Caught,
|
|
3070
|
+
// counted, and reported at the end rather than swallowed.
|
|
3071
|
+
let source: { text: string; model?: string } | null;
|
|
3072
|
+
try {
|
|
3073
|
+
source = sourceFileOf(file, raw, pricing, stringFlag(args, 'prompt'));
|
|
3074
|
+
} catch {
|
|
3075
|
+
skipped++;
|
|
3076
|
+
continue;
|
|
3077
|
+
}
|
|
3078
|
+
if (source === null && SOURCE_EXTENSIONS.some((ext) => file.toLowerCase().endsWith(ext))) {
|
|
3079
|
+
skipped++;
|
|
3080
|
+
continue;
|
|
3081
|
+
}
|
|
3082
|
+
const text = source ? source.text : raw;
|
|
3083
|
+
if (text.trim() === '') continue;
|
|
3084
|
+
|
|
3085
|
+
const result = optimize(text, { level, locale, usage, pricing, disableRules: disabledRules(args, config) });
|
|
3086
|
+
ranked.push({
|
|
3087
|
+
path: file,
|
|
3088
|
+
profile: profilePrompt(text),
|
|
3089
|
+
recoverable: result.tokensSaved,
|
|
3090
|
+
recoverableUsd: result.savings.monthlySavingsUsd,
|
|
3091
|
+
promptName: source ? (stringFlag(args, 'prompt') ?? null) : null,
|
|
3092
|
+
});
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
if (ranked.length === 0) {
|
|
3096
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
3097
|
+
}
|
|
3098
|
+
|
|
3099
|
+
ranked.sort((a, b) => b.recoverableUsd - a.recoverableUsd || b.recoverable - a.recoverable);
|
|
3100
|
+
|
|
3101
|
+
// Before the print and independently of --json, as in `check`: the file's whole
|
|
3102
|
+
// job is to survive the run, and a report that only appears on the happy path
|
|
3103
|
+
// is a report nobody can rely on.
|
|
3104
|
+
await writeMarkdown(args, () =>
|
|
3105
|
+
renderRankMarkdown({
|
|
3106
|
+
root,
|
|
3107
|
+
ranked,
|
|
3108
|
+
level,
|
|
3109
|
+
modelDisplayName: pricing.models.find((m) => m.id === usage.model)?.displayName ?? usage.model,
|
|
3110
|
+
callsPerMonth: usage.callsPerMonth,
|
|
3111
|
+
truncated,
|
|
3112
|
+
skipped,
|
|
3113
|
+
t,
|
|
3114
|
+
}),
|
|
3115
|
+
);
|
|
3116
|
+
|
|
3117
|
+
printRank(ranked, { root, args, usage, pricing, t, truncated, skipped });
|
|
3118
|
+
}
|
|
3119
|
+
|
|
3120
|
+
/**
|
|
3121
|
+
* The ranking, and the numbers that explain it.
|
|
3122
|
+
*
|
|
3123
|
+
* Every column is a measurement with a definition in `profile.ts`, printed with
|
|
3124
|
+
* its units. There is deliberately no total, no grade and no index: the reader
|
|
3125
|
+
* is meant to look down the first column, pick a file, and know why.
|
|
3126
|
+
*/
|
|
3127
|
+
function printRank(
|
|
3128
|
+
ranked: RankedPrompt[],
|
|
3129
|
+
context: {
|
|
3130
|
+
root: string;
|
|
3131
|
+
args: Args;
|
|
3132
|
+
usage: UsageProfile;
|
|
3133
|
+
pricing: PricingCatalogue;
|
|
3134
|
+
t: CliMessages;
|
|
3135
|
+
truncated: boolean;
|
|
3136
|
+
skipped: number;
|
|
3137
|
+
},
|
|
3138
|
+
): void {
|
|
3139
|
+
const { root, args, usage, pricing, t, truncated, skipped } = context;
|
|
3140
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
3141
|
+
|
|
3142
|
+
if (boolFlag(args, 'json')) {
|
|
3143
|
+
console.log(
|
|
3144
|
+
JSON.stringify(
|
|
3145
|
+
{
|
|
3146
|
+
root,
|
|
3147
|
+
truncated,
|
|
3148
|
+
skippedSourceFiles: skipped,
|
|
3149
|
+
usage,
|
|
3150
|
+
prompts: ranked.map((entry) => ({
|
|
3151
|
+
path: entry.path,
|
|
3152
|
+
...entry.profile,
|
|
3153
|
+
recoverableTokens: entry.recoverable,
|
|
3154
|
+
recoverableUsdPerMonth: entry.recoverableUsd,
|
|
3155
|
+
})),
|
|
3156
|
+
},
|
|
3157
|
+
null,
|
|
3158
|
+
2,
|
|
3159
|
+
),
|
|
3160
|
+
);
|
|
3161
|
+
return;
|
|
3162
|
+
}
|
|
3163
|
+
|
|
3164
|
+
const model = pricing.models.find((m) => m.id === usage.model);
|
|
3165
|
+
console.log(`\n${c.bold(t.rank.heading(root, ranked.length))}`);
|
|
3166
|
+
console.log(
|
|
3167
|
+
c.dim(t.rank.subheading(model?.displayName ?? usage.model, n(usage.callsPerMonth))),
|
|
3168
|
+
);
|
|
3169
|
+
console.log();
|
|
3170
|
+
|
|
3171
|
+
const cols = t.rank.columns;
|
|
3172
|
+
const widths = {
|
|
3173
|
+
save: Math.max(cols.recoverable.length, ...ranked.map((r) => formatUsd(r.recoverableUsd).length)),
|
|
3174
|
+
back: Math.max(cols.tokensBack.length, ...ranked.map((r) => n(r.recoverable).length)),
|
|
3175
|
+
tokens: Math.max(cols.tokens.length, ...ranked.map((r) => n(r.profile.tokens).length)),
|
|
3176
|
+
density: Math.max(cols.density.length, 6),
|
|
3177
|
+
};
|
|
3178
|
+
|
|
3179
|
+
console.log(
|
|
3180
|
+
c.dim(
|
|
3181
|
+
[
|
|
3182
|
+
cols.recoverable.padStart(widths.save),
|
|
3183
|
+
cols.tokensBack.padStart(widths.back),
|
|
3184
|
+
cols.tokens.padStart(widths.tokens),
|
|
3185
|
+
cols.density.padStart(widths.density),
|
|
3186
|
+
cols.notes,
|
|
3187
|
+
].join(' '),
|
|
3188
|
+
),
|
|
3189
|
+
);
|
|
3190
|
+
|
|
3191
|
+
for (const entry of ranked) {
|
|
3192
|
+
const { profile } = entry;
|
|
3193
|
+
const notes: string[] = [];
|
|
3194
|
+
if (profile.examples > 0) notes.push(t.rank.noteExamples(profile.examples, n(profile.exampleTokens)));
|
|
3195
|
+
if (profile.formatTokens > 0) notes.push(t.rank.noteFormat(n(profile.formatTokens)));
|
|
3196
|
+
// Only when it is a large enough share to change the answer: "3% of this
|
|
3197
|
+
// is code" is true of nearly everything and tells nobody anything.
|
|
3198
|
+
const protectedShare = profile.tokens === 0 ? 0 : profile.protectedTokens / profile.tokens;
|
|
3199
|
+
if (protectedShare >= 0.25) notes.push(t.rank.noteProtected(Math.round(protectedShare * 100)));
|
|
3200
|
+
|
|
3201
|
+
// Money *and* tokens, side by side, and that is the fix for a real
|
|
3202
|
+
// misreading. Four prompts showed "$0.25" and looked like four equivalent
|
|
3203
|
+
// jobs; three of them recovered a single token, which at 50,000 calls is
|
|
3204
|
+
// twenty-five cents and no work worth doing. Rather than invent a threshold
|
|
3205
|
+
// — any cutoff here would be a number nobody could check — the count is
|
|
3206
|
+
// printed beside the money. "1" is self-evidently nothing and "36" is
|
|
3207
|
+
// self-evidently something, with no judgement of ours in between.
|
|
3208
|
+
console.log(
|
|
3209
|
+
[
|
|
3210
|
+
formatUsd(entry.recoverableUsd).padStart(widths.save),
|
|
3211
|
+
n(entry.recoverable).padStart(widths.back),
|
|
3212
|
+
n(profile.tokens).padStart(widths.tokens),
|
|
3213
|
+
profile.tokensPerSentence.toFixed(1).padStart(widths.density),
|
|
3214
|
+
`${entry.path}${notes.length > 0 ? c.dim(` — ${notes.join(', ')}`) : ''}`,
|
|
3215
|
+
].join(' '),
|
|
3216
|
+
);
|
|
3217
|
+
}
|
|
3218
|
+
|
|
3219
|
+
if (truncated) console.log(`\n${c.dim(t.check.walkTruncated())}`);
|
|
3220
|
+
// Named rather than silent: a repository where most prompts live in code
|
|
3221
|
+
// would otherwise show a short list and look complete.
|
|
3222
|
+
if (skipped > 0) console.log(`\n${c.dim(t.rank.skipped(skipped))}`);
|
|
3223
|
+
console.log(`\n${c.dim(t.rank.densityNote())}`);
|
|
3224
|
+
console.log(`${c.dim(t.rank.recoverableNote())}\n`);
|
|
3225
|
+
}
|
|
3226
|
+
|
|
3227
|
+
async function main(): Promise<void> {
|
|
3228
|
+
const argv = process.argv.slice(2);
|
|
3229
|
+
let locale = localeFromArgv(argv);
|
|
3230
|
+
let t = getCliMessages(locale);
|
|
3231
|
+
const args = parseArgs(argv, t);
|
|
3232
|
+
|
|
3233
|
+
/**
|
|
3234
|
+
* An errand, not a mode of a command — so it runs with no command named, and
|
|
3235
|
+
* before the config is loaded.
|
|
3236
|
+
*
|
|
3237
|
+
* Both halves of that are deliberate. `trazum --clear-suggestion-cache` with
|
|
3238
|
+
* nothing else on the line is how somebody will type it, and the first
|
|
3239
|
+
* version sat below the help branch, where `!args.command` had already
|
|
3240
|
+
* printed the usage text and returned: the flag did nothing, and said nothing
|
|
3241
|
+
* about doing nothing. Loading the config first would be the same mistake one
|
|
3242
|
+
* layer down — a cache you cannot empty because an unrelated `trazum.config.json`
|
|
3243
|
+
* fails to parse is a cache somebody deletes by hand, guessing at the path.
|
|
3244
|
+
*/
|
|
3245
|
+
if (boolFlag(args, 'clear-suggestion-cache')) {
|
|
3246
|
+
const dir = cacheDir();
|
|
3247
|
+
const before = cacheStats(dir);
|
|
3248
|
+
const removed = clearCache(dir);
|
|
3249
|
+
console.log(t.cache.cleared(removed, before.bytes, dir));
|
|
3250
|
+
return;
|
|
3251
|
+
}
|
|
3252
|
+
|
|
3253
|
+
if (boolFlag(args, 'help') || boolFlag(args, 'h') || !args.command) {
|
|
3254
|
+
console.log(
|
|
3255
|
+
t.help(
|
|
3256
|
+
{
|
|
3257
|
+
model: DEFAULT_USAGE.model,
|
|
3258
|
+
callsPerMonth: DEFAULT_USAGE.callsPerMonth,
|
|
3259
|
+
avgOutputTokens: DEFAULT_USAGE.avgOutputTokens,
|
|
3260
|
+
cacheHitRate: DEFAULT_USAGE.cacheHitRate,
|
|
3261
|
+
locales: LOCALES,
|
|
3262
|
+
},
|
|
3263
|
+
c.bold,
|
|
3264
|
+
),
|
|
3265
|
+
);
|
|
3266
|
+
return;
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
rejectUnknownFlags(args, t);
|
|
3270
|
+
|
|
3271
|
+
// Loaded before dispatch so every command sees the same settings, and after
|
|
3272
|
+
// flag validation so a typo is reported before any file is touched. An
|
|
3273
|
+
// invalid config throws here rather than quietly reverting to defaults —
|
|
3274
|
+
// "defaults" for a budget means "no budget", which means a green build.
|
|
3275
|
+
const loaded = await loadConfig({ explicit: stringFlag(args, 'config') });
|
|
3276
|
+
const { config } = loaded;
|
|
3277
|
+
const pricing = await pricingFor(args, loaded, t);
|
|
3278
|
+
|
|
3279
|
+
// The config only gets to choose the locale when nothing more explicit did.
|
|
3280
|
+
if (config.locale && !stringFlag(args, 'locale')) {
|
|
3281
|
+
locale = detectLocale(undefined, process.env, config.locale);
|
|
3282
|
+
t = getCliMessages(locale);
|
|
3283
|
+
}
|
|
3284
|
+
|
|
3285
|
+
switch (args.command) {
|
|
3286
|
+
case 'optimize':
|
|
3287
|
+
await commandOptimize(args, config, pricing, t, locale);
|
|
3288
|
+
break;
|
|
3289
|
+
case 'check':
|
|
3290
|
+
await commandCheck(args, config, pricing, t, locale);
|
|
3291
|
+
break;
|
|
3292
|
+
case 'baseline':
|
|
3293
|
+
await commandBaseline(args, config, pricing, t, locale);
|
|
3294
|
+
break;
|
|
3295
|
+
case 'eval':
|
|
3296
|
+
await commandEval(args, config, t, locale);
|
|
3297
|
+
break;
|
|
3298
|
+
case 'prune':
|
|
3299
|
+
await commandPrune(args, t);
|
|
3300
|
+
break;
|
|
3301
|
+
case 'diff':
|
|
3302
|
+
await commandDiff(args, config, pricing, t, locale);
|
|
3303
|
+
break;
|
|
3304
|
+
case 'models':
|
|
3305
|
+
commandModels(t, pricing);
|
|
3306
|
+
break;
|
|
3307
|
+
case 'where':
|
|
3308
|
+
await commandWhere(args, config, pricing, t);
|
|
3309
|
+
break;
|
|
3310
|
+
case 'rules':
|
|
3311
|
+
commandRules(t, locale);
|
|
3312
|
+
break;
|
|
3313
|
+
case 'doctor':
|
|
3314
|
+
await commandDoctor(args, config, pricing, t, locale);
|
|
3315
|
+
return;
|
|
3316
|
+
case 'rank':
|
|
3317
|
+
await commandRank(args, config, pricing, t, locale);
|
|
3318
|
+
break;
|
|
3319
|
+
case 'blame':
|
|
3320
|
+
await commandBlame(args, config, pricing, t);
|
|
3321
|
+
break;
|
|
3322
|
+
default:
|
|
3323
|
+
throw new Error(t.errors.unknownCommand(args.command));
|
|
3324
|
+
}
|
|
3325
|
+
}
|
|
3326
|
+
|
|
3327
|
+
main().catch((error: unknown) => {
|
|
3328
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
3329
|
+
const t = getCliMessages(localeFromArgv(process.argv.slice(2)));
|
|
3330
|
+
console.error(`\n${c.red(t.errors.errorLabel())}: ${message}\n`);
|
|
3331
|
+
process.exitCode = 1;
|
|
3332
|
+
});
|
|
3333
|
+
|
|
3334
|
+
// --------------------------------------------------------------------------
|
|
3335
|
+
// doctor
|
|
3336
|
+
// --------------------------------------------------------------------------
|
|
3337
|
+
|
|
3338
|
+
/** One prompt, as `doctor` sees it. */
|
|
3339
|
+
interface Diagnosis {
|
|
3340
|
+
path: string;
|
|
3341
|
+
tokens: number;
|
|
3342
|
+
/** The budget that applies, or null when no pattern matches. */
|
|
3343
|
+
budget: ResolvedBudget | null;
|
|
3344
|
+
advisories: readonly Advisory[];
|
|
3345
|
+
/**
|
|
3346
|
+
* The prompt as written, kept only for the cross-prompt pass.
|
|
3347
|
+
*
|
|
3348
|
+
* Every other figure here is per prompt and the text could be dropped after
|
|
3349
|
+
* `optimize` returned. Shared cache prefixes cannot be found that way: the
|
|
3350
|
+
* question is whether *these two files* open with the same bytes, and no
|
|
3351
|
+
* summary of either one answers it.
|
|
3352
|
+
*/
|
|
3353
|
+
text: string;
|
|
3354
|
+
}
|
|
3355
|
+
|
|
3356
|
+
/** An advisory rolled up across every prompt that raised it. */
|
|
3357
|
+
interface Finding {
|
|
3358
|
+
id: string;
|
|
3359
|
+
title: string;
|
|
3360
|
+
prompts: number;
|
|
3361
|
+
/** Summed monthly figure, or null when no prompt attached money to it. */
|
|
3362
|
+
monthlyUsd: number | null;
|
|
3363
|
+
}
|
|
3364
|
+
|
|
3365
|
+
/**
|
|
3366
|
+
* `trazum doctor [dir]` — the survey before the gate.
|
|
3367
|
+
*
|
|
3368
|
+
* Every other command answers a question about one prompt, or ranks prompts
|
|
3369
|
+
* against each other. This one answers "what is wrong with this repository", and
|
|
3370
|
+
* it does so **without inventing a single new judgement**.
|
|
3371
|
+
*
|
|
3372
|
+
* That is the design constraint worth stating, because the obvious way to build
|
|
3373
|
+
* this command is the wrong one. A health check invites a score, a grade, a
|
|
3374
|
+
* traffic light — numbers assembled from weights nobody can reproduce, which get
|
|
3375
|
+
* quietly tuned until the output looks right. `rank` already refused that. So
|
|
3376
|
+
* every finding here is an advisory that `optimize` would raise on that prompt on
|
|
3377
|
+
* its own, summed: the "37 prompts only need a cheaper model" line is 37 copies of
|
|
3378
|
+
* the `model-downgrade` advisory, each reproducible by running `trazum optimize`
|
|
3379
|
+
* on the file named. Nothing is computed here that cannot be checked there.
|
|
3380
|
+
*
|
|
3381
|
+
* **It exits 0 even when it finds things.** `trazum check` is the gate and fails
|
|
3382
|
+
* builds; this is the survey. The model recommendation is a keyword heuristic, and
|
|
3383
|
+
* gating a build on a keyword heuristic is how people learn to re-run until green
|
|
3384
|
+
* — which costs more than the tool ever saves.
|
|
3385
|
+
*
|
|
3386
|
+
* Deliberately not included: anything needing a model. "Prompts that exceed their
|
|
3387
|
+
* own `--suggest` recommendations" would mean an LLM call per prompt, and `doctor`
|
|
3388
|
+
* is the command you run on forty files before you have decided to spend anything.
|
|
3389
|
+
*/
|
|
3390
|
+
async function commandDoctor(
|
|
3391
|
+
args: Args,
|
|
3392
|
+
config: TrazumConfig,
|
|
3393
|
+
pricing: PricingCatalogue,
|
|
3394
|
+
t: CliMessages,
|
|
3395
|
+
locale: Locale,
|
|
3396
|
+
): Promise<void> {
|
|
3397
|
+
const root = args.positional[0] ?? '.';
|
|
3398
|
+
const level = levelFlag(args, config, t);
|
|
3399
|
+
const usage = usageFrom(args, config, t);
|
|
3400
|
+
|
|
3401
|
+
const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
|
|
3402
|
+
const { files, truncated } = await walkPrompts(root, { extensions });
|
|
3403
|
+
if (files.length === 0) {
|
|
3404
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
3405
|
+
}
|
|
3406
|
+
|
|
3407
|
+
const seen: Diagnosis[] = [];
|
|
3408
|
+
let skipped = 0;
|
|
3409
|
+
|
|
3410
|
+
for (const file of files) {
|
|
3411
|
+
const raw = await readFile(join(root, file), 'utf8');
|
|
3412
|
+
|
|
3413
|
+
// Same contract as `rank`: a source file contributes its marked prompt or
|
|
3414
|
+
// nothing, and an unmarked one is counted rather than allowed to abort a
|
|
3415
|
+
// survey of the other thirty-nine.
|
|
3416
|
+
let source: { text: string; model?: string } | null;
|
|
3417
|
+
try {
|
|
3418
|
+
source = sourceFileOf(file, raw, pricing, stringFlag(args, 'prompt'));
|
|
3419
|
+
} catch {
|
|
3420
|
+
skipped++;
|
|
3421
|
+
continue;
|
|
3422
|
+
}
|
|
3423
|
+
if (source === null && SOURCE_EXTENSIONS.some((ext) => file.toLowerCase().endsWith(ext))) {
|
|
3424
|
+
skipped++;
|
|
3425
|
+
continue;
|
|
3426
|
+
}
|
|
3427
|
+
const text = source ? source.text : raw;
|
|
3428
|
+
if (text.trim() === '') continue;
|
|
3429
|
+
|
|
3430
|
+
const result = optimize(text, {
|
|
3431
|
+
level,
|
|
3432
|
+
locale,
|
|
3433
|
+
usage,
|
|
3434
|
+
pricing,
|
|
3435
|
+
disableRules: disabledRules(args, config),
|
|
3436
|
+
});
|
|
3437
|
+
|
|
3438
|
+
seen.push({
|
|
3439
|
+
path: file,
|
|
3440
|
+
// As written, not as optimised: a budget governs the file on disk, and this
|
|
3441
|
+
// is the number `check` would compare against it.
|
|
3442
|
+
tokens: result.tokensBefore,
|
|
3443
|
+
budget: budgetFor(file, config.budgets),
|
|
3444
|
+
advisories: result.advisories,
|
|
3445
|
+
text,
|
|
3446
|
+
});
|
|
3447
|
+
}
|
|
3448
|
+
|
|
3449
|
+
if (seen.length === 0) {
|
|
3450
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
3451
|
+
}
|
|
3452
|
+
|
|
3453
|
+
const unbudgeted = seen.filter((d) => d.budget === null);
|
|
3454
|
+
const overBudget = seen.filter((d) => d.budget !== null && d.tokens > d.budget.maxTokens);
|
|
3455
|
+
|
|
3456
|
+
/** Advisories rolled up by id, worst money first. */
|
|
3457
|
+
const findings: Finding[] = [];
|
|
3458
|
+
for (const diagnosis of seen) {
|
|
3459
|
+
for (const advisory of diagnosis.advisories) {
|
|
3460
|
+
let finding = findings.find((f) => f.id === advisory.id);
|
|
3461
|
+
if (!finding) {
|
|
3462
|
+
finding = { id: advisory.id, title: advisory.title, prompts: 0, monthlyUsd: null };
|
|
3463
|
+
findings.push(finding);
|
|
3464
|
+
}
|
|
3465
|
+
finding.prompts++;
|
|
3466
|
+
if (advisory.estimatedMonthlyUsd !== null) {
|
|
3467
|
+
finding.monthlyUsd = (finding.monthlyUsd ?? 0) + advisory.estimatedMonthlyUsd;
|
|
3468
|
+
}
|
|
3469
|
+
}
|
|
3470
|
+
}
|
|
3471
|
+
// Money first, then breadth. An advisory with no figure attached is not
|
|
3472
|
+
// worthless — `context-overflow` means the call fails — so it sorts by how many
|
|
3473
|
+
// prompts raised it rather than falling to the bottom as a zero.
|
|
3474
|
+
findings.sort((a, b) => (b.monthlyUsd ?? 0) - (a.monthlyUsd ?? 0) || b.prompts - a.prompts);
|
|
3475
|
+
|
|
3476
|
+
/**
|
|
3477
|
+
* The one finding here that is not a rolled-up advisory.
|
|
3478
|
+
*
|
|
3479
|
+
* Everything above is `optimize` run on one file and summed, which is the
|
|
3480
|
+
* constraint this command was built around — every line reproducible on a
|
|
3481
|
+
* single prompt. This is the deliberate exception, and it earns it by being
|
|
3482
|
+
* the only question that cannot be asked of one file: whether a preamble
|
|
3483
|
+
* shared by twelve prompts is byte-identical in any two of them.
|
|
3484
|
+
*
|
|
3485
|
+
* Gated on the model's own cacheable minimum, so a shared prefix too short to
|
|
3486
|
+
* cache is not reported as an opportunity — the same refusal `reorderForCache`
|
|
3487
|
+
* makes.
|
|
3488
|
+
*/
|
|
3489
|
+
const prefixGroups = sharedPrefixes(
|
|
3490
|
+
seen.map((d) => ({ path: d.path, text: d.text })),
|
|
3491
|
+
{ minTokens: cacheableMinimum(pricing.models.find((m) => m.id === usage.model)) },
|
|
3492
|
+
);
|
|
3493
|
+
|
|
3494
|
+
// Before the print, like every other file this repository writes: a report that
|
|
3495
|
+
// only appears when the terminal output was also wanted is a report a scheduled
|
|
3496
|
+
// job cannot rely on.
|
|
3497
|
+
await writeOtlp(args, () =>
|
|
3498
|
+
toOtlpMetrics(
|
|
3499
|
+
{
|
|
3500
|
+
prompts: seen.map((d) => ({
|
|
3501
|
+
path: d.path,
|
|
3502
|
+
tokens: d.tokens,
|
|
3503
|
+
overBudget: d.budget !== null && d.tokens > d.budget.maxTokens,
|
|
3504
|
+
budgeted: d.budget !== null,
|
|
3505
|
+
})),
|
|
3506
|
+
findings: findings.map((f) => ({ id: f.id, prompts: f.prompts, monthlyUsd: f.monthlyUsd })),
|
|
3507
|
+
model: usage.model,
|
|
3508
|
+
callsPerMonth: usage.callsPerMonth,
|
|
3509
|
+
},
|
|
3510
|
+
Date.now(),
|
|
3511
|
+
),
|
|
3512
|
+
);
|
|
3513
|
+
|
|
3514
|
+
printDoctor(
|
|
3515
|
+
{ root, seen, unbudgeted, overBudget, findings, prefixGroups, skipped, truncated },
|
|
3516
|
+
{ args, usage, pricing, t },
|
|
3517
|
+
);
|
|
3518
|
+
}
|
|
3519
|
+
|
|
3520
|
+
function printDoctor(
|
|
3521
|
+
report: {
|
|
3522
|
+
root: string;
|
|
3523
|
+
seen: readonly Diagnosis[];
|
|
3524
|
+
unbudgeted: readonly Diagnosis[];
|
|
3525
|
+
overBudget: readonly Diagnosis[];
|
|
3526
|
+
findings: readonly Finding[];
|
|
3527
|
+
prefixGroups: readonly SharedPrefix[];
|
|
3528
|
+
skipped: number;
|
|
3529
|
+
truncated: boolean;
|
|
3530
|
+
},
|
|
3531
|
+
context: { args: Args; usage: UsageProfile; pricing: PricingCatalogue; t: CliMessages },
|
|
3532
|
+
): void {
|
|
3533
|
+
const { root, seen, unbudgeted, overBudget, findings, prefixGroups, skipped, truncated } = report;
|
|
3534
|
+
const { args, usage, pricing, t } = context;
|
|
3535
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
3536
|
+
|
|
3537
|
+
if (boolFlag(args, 'json')) {
|
|
3538
|
+
console.log(
|
|
3539
|
+
JSON.stringify(
|
|
3540
|
+
{
|
|
3541
|
+
root,
|
|
3542
|
+
prompts: seen.length,
|
|
3543
|
+
skippedSourceFiles: skipped,
|
|
3544
|
+
truncated,
|
|
3545
|
+
usage,
|
|
3546
|
+
pricingLastReviewed: pricing.lastReviewed,
|
|
3547
|
+
unbudgeted: unbudgeted.map((d) => d.path),
|
|
3548
|
+
overBudget: overBudget.map((d) => ({
|
|
3549
|
+
path: d.path,
|
|
3550
|
+
tokens: d.tokens,
|
|
3551
|
+
maxTokens: d.budget!.maxTokens,
|
|
3552
|
+
pattern: d.budget!.pattern,
|
|
3553
|
+
})),
|
|
3554
|
+
findings: findings.map((f) => ({
|
|
3555
|
+
id: f.id,
|
|
3556
|
+
prompts: f.prompts,
|
|
3557
|
+
estimatedMonthlyUsd: f.monthlyUsd,
|
|
3558
|
+
})),
|
|
3559
|
+
// No `estimatedMonthlyUsd` here, and consumers should not add one: see
|
|
3560
|
+
// shared-prefix.ts for why the cost model cannot price this.
|
|
3561
|
+
sharedPrefixes: prefixGroups.map((group) => ({
|
|
3562
|
+
paths: group.paths,
|
|
3563
|
+
tokens: group.tokens,
|
|
3564
|
+
blocks: group.blocks,
|
|
3565
|
+
drift: group.drift,
|
|
3566
|
+
})),
|
|
3567
|
+
},
|
|
3568
|
+
null,
|
|
3569
|
+
2,
|
|
3570
|
+
),
|
|
3571
|
+
);
|
|
3572
|
+
return;
|
|
3573
|
+
}
|
|
3574
|
+
|
|
3575
|
+
const model = pricing.models.find((m) => m.id === usage.model);
|
|
3576
|
+
console.log(`\n${c.bold(t.doctor.heading(root, seen.length))}`);
|
|
3577
|
+
console.log(
|
|
3578
|
+
c.dim(t.doctor.subheading(model?.displayName ?? usage.model, n(usage.callsPerMonth))),
|
|
3579
|
+
);
|
|
3580
|
+
console.log(
|
|
3581
|
+
c.dim(
|
|
3582
|
+
t.doctor.pricesReviewed(pricing.lastReviewed, reviewAgeDays(pricing.lastReviewed, new Date())),
|
|
3583
|
+
),
|
|
3584
|
+
);
|
|
3585
|
+
|
|
3586
|
+
// Budgets first. Everything below is money; this is whether anything is
|
|
3587
|
+
// watching at all, and an unwatched prompt is how the money got there.
|
|
3588
|
+
console.log(`\n${c.bold(t.doctor.budgetsHeading())}`);
|
|
3589
|
+
if (unbudgeted.length === 0 && overBudget.length === 0) {
|
|
3590
|
+
console.log(` ${c.green('✓')} ${t.doctor.everyPromptBudgeted(seen.length)}`);
|
|
3591
|
+
}
|
|
3592
|
+
if (overBudget.length > 0) {
|
|
3593
|
+
console.log(` ${c.red('✗')} ${t.doctor.overBudget(overBudget.length)}`);
|
|
3594
|
+
for (const d of overBudget) {
|
|
3595
|
+
console.log(
|
|
3596
|
+
` ${d.path} ${c.red(`${n(d.tokens)} / ${n(d.budget!.maxTokens)}`)} ${c.dim(`(${d.budget!.pattern})`)}`,
|
|
3597
|
+
);
|
|
3598
|
+
}
|
|
3599
|
+
}
|
|
3600
|
+
if (unbudgeted.length > 0) {
|
|
3601
|
+
console.log(` ${c.yellow('!')} ${t.doctor.unbudgeted(unbudgeted.length, seen.length)}`);
|
|
3602
|
+
for (const d of unbudgeted.slice(0, DOCTOR_LIST_LIMIT)) {
|
|
3603
|
+
console.log(` ${c.dim(d.path)}`);
|
|
3604
|
+
}
|
|
3605
|
+
if (unbudgeted.length > DOCTOR_LIST_LIMIT) {
|
|
3606
|
+
console.log(` ${c.dim(t.doctor.andMore(unbudgeted.length - DOCTOR_LIST_LIMIT))}`);
|
|
3607
|
+
}
|
|
3608
|
+
}
|
|
3609
|
+
|
|
3610
|
+
if (findings.length > 0) {
|
|
3611
|
+
console.log(`\n${c.bold(t.doctor.findingsHeading())}`);
|
|
3612
|
+
const width = Math.max(
|
|
3613
|
+
...findings.map((f) => (f.monthlyUsd === null ? 1 : formatUsd(f.monthlyUsd).length)),
|
|
3614
|
+
);
|
|
3615
|
+
for (const finding of findings) {
|
|
3616
|
+
const money =
|
|
3617
|
+
finding.monthlyUsd === null
|
|
3618
|
+
? ' '.repeat(width + 1)
|
|
3619
|
+
: c.green(`~${formatUsd(finding.monthlyUsd).padStart(width)}`);
|
|
3620
|
+
console.log(
|
|
3621
|
+
` ${money} ${finding.title} ${c.dim(t.doctor.acrossPrompts(finding.prompts))}`,
|
|
3622
|
+
);
|
|
3623
|
+
}
|
|
3624
|
+
console.log(`\n ${c.dim(t.doctor.findingsNote())}`);
|
|
3625
|
+
}
|
|
3626
|
+
|
|
3627
|
+
/**
|
|
3628
|
+
* Its own section, below the money, and not among the findings.
|
|
3629
|
+
*
|
|
3630
|
+
* Every line above carries a dollar figure or is one advisory `optimize` would
|
|
3631
|
+
* raise on a single file. This is neither, and putting it in that list would
|
|
3632
|
+
* make it look like a finding with the money left off — which is how a reader
|
|
3633
|
+
* concludes the tool forgot to compute something rather than that it declined
|
|
3634
|
+
* to guess.
|
|
3635
|
+
*/
|
|
3636
|
+
if (prefixGroups.length > 0) {
|
|
3637
|
+
console.log(`\n${c.bold(t.doctor.sharedPrefixHeading())}`);
|
|
3638
|
+
for (const group of prefixGroups) {
|
|
3639
|
+
console.log(
|
|
3640
|
+
` ${c.yellow('!')} ${t.doctor.sharedPrefixGroup(group.paths.length, n(group.tokens), group.drift)}`,
|
|
3641
|
+
);
|
|
3642
|
+
for (const path of group.paths.slice(0, DOCTOR_LIST_LIMIT)) {
|
|
3643
|
+
console.log(` ${c.dim(path)}`);
|
|
3644
|
+
}
|
|
3645
|
+
if (group.paths.length > DOCTOR_LIST_LIMIT) {
|
|
3646
|
+
console.log(` ${c.dim(t.doctor.andMore(group.paths.length - DOCTOR_LIST_LIMIT))}`);
|
|
3647
|
+
}
|
|
3648
|
+
console.log(` ${c.dim(t.doctor.sharedPrefixFix(group.drift))}`);
|
|
3649
|
+
}
|
|
3650
|
+
console.log(`\n ${c.dim(t.doctor.sharedPrefixNoFigure())}`);
|
|
3651
|
+
}
|
|
3652
|
+
|
|
3653
|
+
if (skipped > 0) console.log(`\n${c.dim(t.rank.skipped(skipped))}`);
|
|
3654
|
+
if (truncated) console.log(`\n${c.dim(t.check.walkTruncated())}`);
|
|
3655
|
+
|
|
3656
|
+
// Stated at the end, where somebody deciding what to do with the output is
|
|
3657
|
+
// looking. A survey that exits 1 becomes a gate, and a gate on a keyword
|
|
3658
|
+
// heuristic teaches people to re-run until green.
|
|
3659
|
+
console.log(`\n${c.dim(t.doctor.notAGate())}\n`);
|
|
3660
|
+
}
|
|
3661
|
+
|
|
3662
|
+
/** One prompt that exists on both sides, and what the edit did to it. */
|
|
3663
|
+
interface PairedDiff {
|
|
3664
|
+
path: string;
|
|
3665
|
+
comparison: PromptComparison;
|
|
3666
|
+
}
|
|
3667
|
+
|
|
3668
|
+
/**
|
|
3669
|
+
* `trazum diff --all <before> <after>` — a whole prompt library, before and after.
|
|
3670
|
+
*
|
|
3671
|
+
* `diff` answers the question for one prompt. A team refactoring forty of them
|
|
3672
|
+
* wants the same question answered forty times and totalled, and running the
|
|
3673
|
+
* command forty times by hand loses the total — which is the figure the decision
|
|
3674
|
+
* actually turns on.
|
|
3675
|
+
*
|
|
3676
|
+
* **Prompts that exist on only one side are named, not silently skipped.** A
|
|
3677
|
+
* refactor that deletes a prompt and a refactor that renames one look identical
|
|
3678
|
+
* from a token count, and both are things a reviewer has to know about. Reporting
|
|
3679
|
+
* only the pairs would let a deletion read as a saving.
|
|
3680
|
+
*
|
|
3681
|
+
* `--max-growth` applies **per prompt**, not to the total, which follows the rule
|
|
3682
|
+
* `check` already states about budgets: a library is forty things to govern, and
|
|
3683
|
+
* summing them would pass a refactor that quietly doubled one prompt because
|
|
3684
|
+
* another shrank.
|
|
3685
|
+
*/
|
|
3686
|
+
async function diffDirectories(
|
|
3687
|
+
beforeRoot: string,
|
|
3688
|
+
afterRoot: string,
|
|
3689
|
+
args: Args,
|
|
3690
|
+
config: TrazumConfig,
|
|
3691
|
+
pricing: PricingCatalogue,
|
|
3692
|
+
t: CliMessages,
|
|
3693
|
+
locale: Locale,
|
|
3694
|
+
): Promise<void> {
|
|
3695
|
+
const level = levelFlag(args, config, t);
|
|
3696
|
+
const usage = usageFrom(args, config, t);
|
|
3697
|
+
const optimizeBoth = boolFlag(args, 'optimized');
|
|
3698
|
+
const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
|
|
3699
|
+
|
|
3700
|
+
const [beforeWalk, afterWalk] = await Promise.all([
|
|
3701
|
+
walkPrompts(beforeRoot, { extensions }),
|
|
3702
|
+
walkPrompts(afterRoot, { extensions }),
|
|
3703
|
+
]);
|
|
3704
|
+
if (beforeWalk.files.length === 0 && afterWalk.files.length === 0) {
|
|
3705
|
+
throw new Error(t.errors.noPromptsFound(`${beforeRoot}, ${afterRoot}`, extensions.join(' ')));
|
|
3706
|
+
}
|
|
3707
|
+
|
|
3708
|
+
const beforeFiles = new Set(beforeWalk.files);
|
|
3709
|
+
const afterFiles = new Set(afterWalk.files);
|
|
3710
|
+
|
|
3711
|
+
/** The prompt at a path, or null when the file holds no marked prompt. */
|
|
3712
|
+
const textAt = async (root: string, file: string): Promise<string | null> => {
|
|
3713
|
+
const raw = await readFile(join(root, file), 'utf8');
|
|
3714
|
+
let source: { text: string; model?: string } | null;
|
|
3715
|
+
try {
|
|
3716
|
+
source = sourceFileOf(file, raw, pricing, stringFlag(args, 'prompt'));
|
|
3717
|
+
} catch {
|
|
3718
|
+
return null;
|
|
3719
|
+
}
|
|
3720
|
+
if (source === null && SOURCE_EXTENSIONS.some((ext) => file.toLowerCase().endsWith(ext))) {
|
|
3721
|
+
return null;
|
|
3722
|
+
}
|
|
3723
|
+
return source ? source.text : raw;
|
|
3724
|
+
};
|
|
3725
|
+
|
|
3726
|
+
const pairs: PairedDiff[] = [];
|
|
3727
|
+
let skipped = 0;
|
|
3728
|
+
|
|
3729
|
+
for (const file of [...beforeFiles].filter((f) => afterFiles.has(f)).sort()) {
|
|
3730
|
+
const [before, after] = await Promise.all([textAt(beforeRoot, file), textAt(afterRoot, file)]);
|
|
3731
|
+
if (before === null || after === null) {
|
|
3732
|
+
skipped++;
|
|
3733
|
+
continue;
|
|
3734
|
+
}
|
|
3735
|
+
pairs.push({
|
|
3736
|
+
path: file,
|
|
3737
|
+
comparison: comparePrompts(before, after, { level, locale, optimizeBoth, usage, pricing }),
|
|
3738
|
+
});
|
|
3739
|
+
}
|
|
3740
|
+
|
|
3741
|
+
const removed = [...beforeFiles].filter((f) => !afterFiles.has(f)).sort();
|
|
3742
|
+
const added = [...afterFiles].filter((f) => !beforeFiles.has(f)).sort();
|
|
3743
|
+
|
|
3744
|
+
if (pairs.length === 0 && removed.length === 0 && added.length === 0) {
|
|
3745
|
+
throw new Error(t.errors.noPromptsFound(`${beforeRoot}, ${afterRoot}`, extensions.join(' ')));
|
|
3746
|
+
}
|
|
3747
|
+
|
|
3748
|
+
// Worst first: a reviewer reads the top of this list and stops.
|
|
3749
|
+
pairs.sort((a, b) => b.comparison.tokenDelta - a.comparison.tokenDelta);
|
|
3750
|
+
|
|
3751
|
+
printDirectoryDiff(
|
|
3752
|
+
{ beforeRoot, afterRoot, pairs, removed, added, skipped, optimizeBoth },
|
|
3753
|
+
{ args, usage, pricing, t },
|
|
3754
|
+
);
|
|
3755
|
+
|
|
3756
|
+
const limit =
|
|
3757
|
+
typeof args.flags.get('max-growth') === 'string'
|
|
3758
|
+
? numberFlag(args, 'max-growth', 0, t)
|
|
3759
|
+
: config.maxGrowth;
|
|
3760
|
+
|
|
3761
|
+
if (limit !== undefined) {
|
|
3762
|
+
// Per prompt, not on the total. Summing would pass a refactor that doubled one
|
|
3763
|
+
// prompt because another happened to shrink — and the prompt that doubled is
|
|
3764
|
+
// the one somebody has to look at.
|
|
3765
|
+
const over = pairs.filter((p) => p.comparison.tokenDelta > limit);
|
|
3766
|
+
if (over.length > 0) {
|
|
3767
|
+
console.error(`\n${c.red(t.diff.someOverLimit(over.length, limit))}`);
|
|
3768
|
+
for (const p of over) {
|
|
3769
|
+
console.error(` ${p.path} ${c.red(`+${p.comparison.tokenDelta}`)}`);
|
|
3770
|
+
}
|
|
3771
|
+
process.exitCode = 1;
|
|
3772
|
+
}
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
|
|
3776
|
+
function printDirectoryDiff(
|
|
3777
|
+
report: {
|
|
3778
|
+
beforeRoot: string;
|
|
3779
|
+
afterRoot: string;
|
|
3780
|
+
pairs: readonly PairedDiff[];
|
|
3781
|
+
removed: readonly string[];
|
|
3782
|
+
added: readonly string[];
|
|
3783
|
+
skipped: number;
|
|
3784
|
+
optimizeBoth: boolean;
|
|
3785
|
+
},
|
|
3786
|
+
context: { args: Args; usage: UsageProfile; pricing: PricingCatalogue; t: CliMessages },
|
|
3787
|
+
): void {
|
|
3788
|
+
const { beforeRoot, afterRoot, pairs, removed, added, skipped, optimizeBoth } = report;
|
|
3789
|
+
const { args, usage, pricing, t } = context;
|
|
3790
|
+
const n = (value: number): string => value.toLocaleString(t.numberLocale);
|
|
3791
|
+
|
|
3792
|
+
const totalTokens = pairs.reduce((sum, p) => sum + p.comparison.tokenDelta, 0);
|
|
3793
|
+
const totalMonthly = pairs.reduce((sum, p) => sum + p.comparison.monthlyDeltaUsd, 0);
|
|
3794
|
+
|
|
3795
|
+
if (boolFlag(args, 'json')) {
|
|
3796
|
+
console.log(
|
|
3797
|
+
JSON.stringify(
|
|
3798
|
+
{
|
|
3799
|
+
before: beforeRoot,
|
|
3800
|
+
after: afterRoot,
|
|
3801
|
+
optimized: optimizeBoth,
|
|
3802
|
+
usage,
|
|
3803
|
+
totals: { tokenDelta: totalTokens, monthlyDeltaUsd: totalMonthly, prompts: pairs.length },
|
|
3804
|
+
prompts: pairs.map((p) => ({
|
|
3805
|
+
path: p.path,
|
|
3806
|
+
tokensBefore: p.comparison.tokensBefore,
|
|
3807
|
+
tokensAfter: p.comparison.tokensAfter,
|
|
3808
|
+
tokenDelta: p.comparison.tokenDelta,
|
|
3809
|
+
monthlyDeltaUsd: p.comparison.monthlyDeltaUsd,
|
|
3810
|
+
})),
|
|
3811
|
+
removed,
|
|
3812
|
+
added,
|
|
3813
|
+
skippedSourceFiles: skipped,
|
|
3814
|
+
},
|
|
3815
|
+
null,
|
|
3816
|
+
2,
|
|
3817
|
+
),
|
|
3818
|
+
);
|
|
3819
|
+
return;
|
|
3820
|
+
}
|
|
3821
|
+
|
|
3822
|
+
const model = pricing.models.find((m) => m.id === usage.model);
|
|
3823
|
+
console.log(`\n${c.bold(t.diff.heading(beforeRoot, afterRoot))}`);
|
|
3824
|
+
console.log(c.dim(t.diff.allSubheading(pairs.length)));
|
|
3825
|
+
if (optimizeBoth) console.log(c.dim(t.diff.measuringOptimised()));
|
|
3826
|
+
|
|
3827
|
+
// The convention, before any number. Every figure here is after minus before,
|
|
3828
|
+
// which is the opposite of the rest of Trazum, and a reader arriving from
|
|
3829
|
+
// `optimize` has the other one loaded.
|
|
3830
|
+
console.log(`\n${c.dim(t.diff.signConvention())}`);
|
|
3831
|
+
|
|
3832
|
+
if (pairs.length > 0) {
|
|
3833
|
+
console.log();
|
|
3834
|
+
const width = Math.max(...pairs.map((p) => signedTokens(p.comparison.tokenDelta, n).length));
|
|
3835
|
+
for (const pair of pairs) {
|
|
3836
|
+
const delta = pair.comparison.tokenDelta;
|
|
3837
|
+
const text = signedTokens(delta, n).padStart(width);
|
|
3838
|
+
const paint = delta > 0 ? c.red : delta < 0 ? c.green : c.dim;
|
|
3839
|
+
console.log(` ${paint(text)} ${pair.path}`);
|
|
3840
|
+
}
|
|
3841
|
+
}
|
|
3842
|
+
|
|
3843
|
+
if (removed.length > 0 || added.length > 0) {
|
|
3844
|
+
// Named rather than folded into the totals. A prompt that vanished is not a
|
|
3845
|
+
// saving of its whole token count; it is a question.
|
|
3846
|
+
console.log();
|
|
3847
|
+
for (const path of removed) console.log(` ${c.dim(t.diff.onlyBefore())} ${path}`);
|
|
3848
|
+
for (const path of added) console.log(` ${c.dim(t.diff.onlyAfter())} ${path}`);
|
|
3849
|
+
console.log(` ${c.dim(t.diff.onlyOneSideNote())}`);
|
|
3850
|
+
}
|
|
3851
|
+
|
|
3852
|
+
if (skipped > 0) console.log(`\n${c.dim(t.rank.skipped(skipped))}`);
|
|
3853
|
+
|
|
3854
|
+
if (pairs.length > 0) {
|
|
3855
|
+
const paint = totalTokens > 0 ? c.red : totalTokens < 0 ? c.green : c.dim;
|
|
3856
|
+
console.log(`\n${c.bold(t.diff.allTotal(signedTokens(totalTokens, n), pairs.length))}`);
|
|
3857
|
+
console.log(
|
|
3858
|
+
paint(
|
|
3859
|
+
t.diff.monthly(
|
|
3860
|
+
formatSignedUsd(totalMonthly),
|
|
3861
|
+
n(usage.callsPerMonth),
|
|
3862
|
+
model?.displayName ?? usage.model,
|
|
3863
|
+
),
|
|
3864
|
+
),
|
|
3865
|
+
);
|
|
3866
|
+
}
|
|
3867
|
+
console.log();
|
|
3868
|
+
}
|
|
3869
|
+
|
|
3870
|
+
/** A token delta with its sign, always. A bare `40` is unreadable either way. */
|
|
3871
|
+
function signedTokens(delta: number, n: (value: number) => string): string {
|
|
3872
|
+
return `${delta > 0 ? '+' : ''}${n(delta)}`;
|
|
3873
|
+
}
|