@trazum/cli 1.35.0 → 1.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +60 -0
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +62 -0
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +49 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +311 -26
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/i18n/en.ts +85 -0
- package/src/i18n/es.ts +87 -0
- package/src/i18n/types.ts +49 -0
- package/src/index.ts +381 -31
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
3
3
|
import { join, resolve as resolvePath } from 'node:path';
|
|
4
4
|
import { gunzipSync } from 'node:zlib';
|
|
5
|
-
import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
5
|
+
import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
6
6
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
7
7
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
8
8
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
@@ -25,6 +25,7 @@ const c = {
|
|
|
25
25
|
};
|
|
26
26
|
const VALUE_FLAGS = new Set([
|
|
27
27
|
'against',
|
|
28
|
+
'from-log',
|
|
28
29
|
// `route` takes a path here, and the flag is deliberately not `--prompt`:
|
|
29
30
|
// everywhere else in this tool `--prompt` names a marked prompt *inside* a
|
|
30
31
|
// source file, and reusing it for a path would be a trap laid for the reader.
|
|
@@ -158,6 +159,24 @@ function levelFlag(args, config, t) {
|
|
|
158
159
|
* model id. It beats the default because reading the code is better than
|
|
159
160
|
* assuming, and loses to config because being told is better than reading.
|
|
160
161
|
*/
|
|
162
|
+
/**
|
|
163
|
+
* One usage log, gzip included, shared by every command that reads one.
|
|
164
|
+
*
|
|
165
|
+
* A `.gz` that will not decompress is an error naming the file — skipping it
|
|
166
|
+
* would be a figure quietly missing a day, the failure this repository
|
|
167
|
+
* refuses everywhere it can occur.
|
|
168
|
+
*/
|
|
169
|
+
async function readUsageLog(file, t) {
|
|
170
|
+
if (!file.endsWith('.gz'))
|
|
171
|
+
return readFile(file, 'utf8');
|
|
172
|
+
const compressed = await readFile(file);
|
|
173
|
+
try {
|
|
174
|
+
return gunzipSync(compressed).toString('utf8');
|
|
175
|
+
}
|
|
176
|
+
catch (error) {
|
|
177
|
+
throw new Error(t.profile.badGzip(file, error instanceof Error ? error.message : String(error)));
|
|
178
|
+
}
|
|
179
|
+
}
|
|
161
180
|
function usageFrom(args, config, t, detected) {
|
|
162
181
|
const fromConfig = config.usage ?? {};
|
|
163
182
|
const model = stringFlag(args, 'model') ?? fromConfig.model ?? detected ?? DEFAULT_USAGE.model;
|
|
@@ -255,11 +274,11 @@ const COMMAND_FLAGS = {
|
|
|
255
274
|
'level', 'model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch',
|
|
256
275
|
'disable', 'llm', 'exact-tokens', 'diff', 'reorder', 'out', 'o',
|
|
257
276
|
'tokens-only', 'cost', 'prompt', 'suggest', 'apply-suggestions',
|
|
258
|
-
'cache-suggestions',
|
|
277
|
+
'cache-suggestions', 'from-log', 'label', 'all-labels',
|
|
259
278
|
],
|
|
260
279
|
check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
|
|
261
280
|
baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
|
|
262
|
-
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary'],
|
|
281
|
+
profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
|
|
263
282
|
route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
|
|
264
283
|
eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
|
|
265
284
|
prune: ['cases', 'concurrency', 'json', 'yes'],
|
|
@@ -420,7 +439,9 @@ function biggestLever(result, tokensOnly, t) {
|
|
|
420
439
|
}
|
|
421
440
|
function printReport(result, showDiff, t, examplesReview = null, reorder = null, tokensOnly = false, host = { id: 'terminal', displayName: 'terminal', billing: 'unknown', evidence: null }, suggestions = null,
|
|
422
441
|
/** They named a scenario, and the host is suppressing the money anyway. */
|
|
423
|
-
namedScenario = false
|
|
442
|
+
namedScenario = false,
|
|
443
|
+
/** Present when the usage came from a log rather than from typing. */
|
|
444
|
+
measured = null) {
|
|
424
445
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
425
446
|
const sourceNote = result.tokenSource === 'heuristic'
|
|
426
447
|
? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
|
|
@@ -542,7 +563,7 @@ namedScenario = false) {
|
|
|
542
563
|
printTokensOnly(result, host, t, n, namedScenario);
|
|
543
564
|
}
|
|
544
565
|
else {
|
|
545
|
-
printMoney(result, t, n);
|
|
566
|
+
printMoney(result, t, n, measured);
|
|
546
567
|
}
|
|
547
568
|
// On a subscription, an advisory whose entire pitch is money is not weaker
|
|
548
569
|
// advice — it is not advice. "Use a cheaper model" saves nothing on a flat
|
|
@@ -606,11 +627,36 @@ namedScenario = false) {
|
|
|
606
627
|
console.log(` ${c.dim(wrap(tokensOnly ? t.report.beyondThisPromptTokensOnly() : t.report.beyondThisPrompt(), 74, ' '))}`);
|
|
607
628
|
}
|
|
608
629
|
/** The cost section, for anyone billed by the token. */
|
|
609
|
-
function printMoney(result, t, n
|
|
630
|
+
function printMoney(result, t, n,
|
|
631
|
+
/** Present when the usage came from a log rather than from typing. */
|
|
632
|
+
measured = null) {
|
|
610
633
|
const { savings } = result;
|
|
611
634
|
console.log();
|
|
612
635
|
console.log(c.bold(t.report.costWith(savings.modelDisplayName)));
|
|
613
|
-
|
|
636
|
+
/**
|
|
637
|
+
* The usage line names its provenance. "1,000 calls/month" typed and
|
|
638
|
+
* "1,043 calls measured over 12 days, scaled" are different claims about
|
|
639
|
+
* the same multiplication, and the reader budgeting on the result must
|
|
640
|
+
* know which one they are holding. Under the week floor nothing is scaled
|
|
641
|
+
* and nothing says "month": the figures cover exactly the period measured.
|
|
642
|
+
*/
|
|
643
|
+
if (measured !== null) {
|
|
644
|
+
if (measured.scaled !== null) {
|
|
645
|
+
console.log(` ${t.report.usageLineMeasured(n(measured.calls), measured.scaled.fromDays.toFixed(1), n(result.usage.callsPerMonth), result.usage.avgOutputTokens, result.usage.batchEligible)}`);
|
|
646
|
+
}
|
|
647
|
+
else {
|
|
648
|
+
console.log(` ${t.report.usageLineMeasuredPeriod(n(measured.calls), measured.spanDays === null ? null : measured.spanDays.toFixed(1), result.usage.avgOutputTokens, result.usage.batchEligible)}`);
|
|
649
|
+
}
|
|
650
|
+
if (measured.models.count > 1) {
|
|
651
|
+
console.log(` ${c.dim(wrap(t.report.measuredModelShare(measured.models.chosen, `${(measured.models.chosenShareOfSpend * 100).toFixed(0)}%`, n(measured.models.count)), 74, ' '))}`);
|
|
652
|
+
}
|
|
653
|
+
if (measured.outputUnmeasured) {
|
|
654
|
+
console.log(` ${c.dim(wrap(t.report.measuredNoOutput(), 74, ' '))}`);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
else {
|
|
658
|
+
console.log(` ${t.report.usageLine(n(result.usage.callsPerMonth), result.usage.avgOutputTokens, result.usage.batchEligible)}`);
|
|
659
|
+
}
|
|
614
660
|
// Said, not assumed. Once prices can be overlaid locally, a figure from the
|
|
615
661
|
// bundled catalogue and a figure from somebody's JSON file look identical, and
|
|
616
662
|
// the reader has to be able to tell which one they are about to budget against.
|
|
@@ -621,9 +667,15 @@ function printMoney(result, t, n) {
|
|
|
621
667
|
if (touched.length > 0) {
|
|
622
668
|
console.log(` ${c.yellow(t.report.pricingOverlaid(touched.join(', '), result.pricingSource.lastReviewed))}`);
|
|
623
669
|
}
|
|
670
|
+
const periodOnly = measured !== null && measured.scaled === null;
|
|
624
671
|
console.log(` ${formatUsd(savings.perMonth.before.totalUsd)} → ` +
|
|
625
672
|
`${c.green(formatUsd(savings.perMonth.after.totalUsd))} ` +
|
|
626
|
-
c.bold(
|
|
673
|
+
c.bold(periodOnly
|
|
674
|
+
? t.report.perPeriodSaving(formatUsd(savings.monthlySavingsUsd), savings.monthlySavingsPct.toFixed(1))
|
|
675
|
+
: t.report.perMonthSaving(formatUsd(savings.monthlySavingsUsd), savings.monthlySavingsPct.toFixed(1))));
|
|
676
|
+
if (periodOnly) {
|
|
677
|
+
console.log(` ${c.dim(wrap(t.report.periodNotScaled(measured.spanDays === null ? null : measured.spanDays.toFixed(1)), 74, ' '))}`);
|
|
678
|
+
}
|
|
627
679
|
}
|
|
628
680
|
/**
|
|
629
681
|
* What the saving buys when there is no bill: room.
|
|
@@ -998,6 +1050,80 @@ async function readInput(source, t) {
|
|
|
998
1050
|
return readFile(source, 'utf8');
|
|
999
1051
|
}
|
|
1000
1052
|
async function commandOptimize(args, config, pricing, t, locale) {
|
|
1053
|
+
/**
|
|
1054
|
+
* `--all-labels`: every mapped prompt against its own measured traffic,
|
|
1055
|
+
* ranked by what the change is worth — the list a person actually wants,
|
|
1056
|
+
* which is "which prompt do I edit first".
|
|
1057
|
+
*
|
|
1058
|
+
* Requires `--from-log`, because ranking estimated savings that were all
|
|
1059
|
+
* multiplied by the same typed guess ranks the prompts by length, and calls
|
|
1060
|
+
* that a priority. And it renders both coverage mismatches at the end: a
|
|
1061
|
+
* prompt mapped to a label with no traffic is dead weight or a rename, and
|
|
1062
|
+
* a label carrying real money with no prompt mapped is the workload nobody
|
|
1063
|
+
* can optimise because nobody said where it lives.
|
|
1064
|
+
*/
|
|
1065
|
+
if (boolFlag(args, 'all-labels')) {
|
|
1066
|
+
const fromLogPath = stringFlag(args, 'from-log');
|
|
1067
|
+
if (fromLogPath === undefined)
|
|
1068
|
+
throw new Error(t.errors.allLabelsNeedsLog());
|
|
1069
|
+
const labelsMap = config.labels ?? {};
|
|
1070
|
+
if (Object.keys(labelsMap).length === 0)
|
|
1071
|
+
throw new Error(t.errors.allLabelsNeedsMap());
|
|
1072
|
+
const report = profileUsage(await readUsageLog(fromLogPath, t), { catalogue: pricing });
|
|
1073
|
+
const coverage = labelCoverage(report, labelsMap);
|
|
1074
|
+
const level = levelFlag(args, config, t);
|
|
1075
|
+
const rows = [];
|
|
1076
|
+
const unreadable = [];
|
|
1077
|
+
for (const { label, promptPath } of coverage.joined) {
|
|
1078
|
+
const m = measuredUsage(report, label, { batchEligible: config.usage?.batchEligible ?? false });
|
|
1079
|
+
if (m === null)
|
|
1080
|
+
continue;
|
|
1081
|
+
let text;
|
|
1082
|
+
try {
|
|
1083
|
+
text = await readFile(promptPath, 'utf8');
|
|
1084
|
+
}
|
|
1085
|
+
catch {
|
|
1086
|
+
unreadable.push({ label, path: promptPath });
|
|
1087
|
+
continue;
|
|
1088
|
+
}
|
|
1089
|
+
const r = optimize(text, { level, usage: m.profile, locale, pricing });
|
|
1090
|
+
rows.push({
|
|
1091
|
+
label,
|
|
1092
|
+
path: promptPath,
|
|
1093
|
+
tokensBefore: r.tokensBefore,
|
|
1094
|
+
tokensAfter: r.tokensAfter,
|
|
1095
|
+
savingUsd: r.savings.monthlySavingsUsd,
|
|
1096
|
+
periodOnly: m.scaled === null,
|
|
1097
|
+
spentUsd: m.spentUsd,
|
|
1098
|
+
});
|
|
1099
|
+
}
|
|
1100
|
+
rows.sort((a, b) => b.savingUsd - a.savingUsd);
|
|
1101
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
1102
|
+
console.log(c.bold(t.report.allLabelsHeading(n(rows.length))));
|
|
1103
|
+
for (const row of rows) {
|
|
1104
|
+
const saving = row.periodOnly
|
|
1105
|
+
? t.report.allLabelsRowPeriod(formatUsd(row.savingUsd))
|
|
1106
|
+
: t.report.allLabelsRow(formatUsd(row.savingUsd));
|
|
1107
|
+
console.log(` ${row.savingUsd > 0 ? c.green('→') : c.dim('·')} ${c.bold(row.label)} ${saving} ${c.dim(`${row.path} · ${n(row.tokensBefore)} → ${n(row.tokensAfter)} tokens · ${formatUsd(row.spentUsd)} measured`)}`);
|
|
1108
|
+
}
|
|
1109
|
+
if (rows.length > 0) {
|
|
1110
|
+
console.log(` ${c.dim(wrap(t.report.allLabelsFooter(), 74, ' '))}`);
|
|
1111
|
+
}
|
|
1112
|
+
/**
|
|
1113
|
+
* The mismatches, both directions, never silently. These are the two
|
|
1114
|
+
* failures neither side can see alone.
|
|
1115
|
+
*/
|
|
1116
|
+
for (const gap of coverage.trafficWithoutPrompt.slice(0, 5)) {
|
|
1117
|
+
console.log(` ${c.yellow('!')} ${wrap(t.report.allLabelsUnmapped(gap.label, formatUsd(gap.spentUsd)), 74, ' ')}`);
|
|
1118
|
+
}
|
|
1119
|
+
for (const dead of coverage.mappedWithoutTraffic) {
|
|
1120
|
+
console.log(` ${c.dim(wrap(t.report.allLabelsDead(dead.label, dead.promptPath), 74, ' '))}`);
|
|
1121
|
+
}
|
|
1122
|
+
for (const miss of unreadable) {
|
|
1123
|
+
console.log(` ${c.yellow('!')} ${wrap(t.report.allLabelsUnreadable(miss.label, miss.path), 74, ' ')}`);
|
|
1124
|
+
}
|
|
1125
|
+
return;
|
|
1126
|
+
}
|
|
1001
1127
|
const target = args.positional[0];
|
|
1002
1128
|
const raw = await readInput(target, t);
|
|
1003
1129
|
const level = levelFlag(args, config, t);
|
|
@@ -1016,7 +1142,59 @@ async function commandOptimize(args, config, pricing, t, locale) {
|
|
|
1016
1142
|
// Detection sits between config and defaults, as everywhere: a flag beats
|
|
1017
1143
|
// config, config beats what the code says, and what the code says beats a
|
|
1018
1144
|
// built-in default that has no idea which provider you use.
|
|
1019
|
-
|
|
1145
|
+
let usage = usageFrom(args, config, t, source?.model);
|
|
1146
|
+
/**
|
|
1147
|
+
* `--from-log`: the multiplication stops guessing.
|
|
1148
|
+
*
|
|
1149
|
+
* The saving printed below is `token delta × usage`, and until now every
|
|
1150
|
+
* part of `usage` was typed by a human. A usage log knows the real call
|
|
1151
|
+
* count, the real output size, the real cache share and the model the
|
|
1152
|
+
* calls actually went to — so `--from-log` measures them, and the typed
|
|
1153
|
+
* flags are refused beside it rather than merged: measuring and typing the
|
|
1154
|
+
* same figure is a contradiction, not a preference order.
|
|
1155
|
+
*/
|
|
1156
|
+
const fromLog = stringFlag(args, 'from-log');
|
|
1157
|
+
let measured = null;
|
|
1158
|
+
if (fromLog !== undefined) {
|
|
1159
|
+
for (const flag of ['calls', 'output-tokens', 'cache-hit-rate', 'model']) {
|
|
1160
|
+
if (args.flags.get(flag) !== undefined) {
|
|
1161
|
+
throw new Error(t.errors.fromLogConflict(flag));
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
const report = profileUsage(await readUsageLog(fromLog, t), { catalogue: pricing });
|
|
1165
|
+
/**
|
|
1166
|
+
* Which label this prompt is. `--label` says it outright; otherwise the
|
|
1167
|
+
* config's `labels` map is read in reverse — it maps labels to prompt
|
|
1168
|
+
* files, and the file on the command line is looked up among its values.
|
|
1169
|
+
* Ambiguity (two labels mapped to one file) is an error naming both,
|
|
1170
|
+
* never a silent first match.
|
|
1171
|
+
*/
|
|
1172
|
+
let label = stringFlag(args, 'label');
|
|
1173
|
+
if (label === undefined && target !== undefined && config.labels !== undefined) {
|
|
1174
|
+
const hits = Object.entries(config.labels)
|
|
1175
|
+
.filter(([, path]) => resolvePath(path) === resolvePath(target))
|
|
1176
|
+
.map(([name]) => name);
|
|
1177
|
+
if (hits.length > 1)
|
|
1178
|
+
throw new Error(t.errors.fromLogAmbiguousLabel(target, hits.join(', ')));
|
|
1179
|
+
label = hits[0];
|
|
1180
|
+
}
|
|
1181
|
+
if (label === undefined) {
|
|
1182
|
+
const available = report.byLabel
|
|
1183
|
+
.map((row) => (row.label === UNLABELLED ? t.profile.unlabelled() : row.label))
|
|
1184
|
+
.join(', ');
|
|
1185
|
+
throw new Error(t.errors.fromLogNeedsLabel(available || '—'));
|
|
1186
|
+
}
|
|
1187
|
+
measured = measuredUsage(report, label, {
|
|
1188
|
+
batchEligible: boolFlag(args, 'batch', config.usage?.batchEligible ?? false),
|
|
1189
|
+
});
|
|
1190
|
+
if (measured === null) {
|
|
1191
|
+
const available = report.byLabel
|
|
1192
|
+
.map((row) => (row.label === UNLABELLED ? t.profile.unlabelled() : row.label))
|
|
1193
|
+
.join(', ');
|
|
1194
|
+
throw new Error(t.errors.fromLogLabelEmpty(label, available || '—'));
|
|
1195
|
+
}
|
|
1196
|
+
usage = measured.profile;
|
|
1197
|
+
}
|
|
1020
1198
|
const disableRules = disabledRules(args, config) ?? [];
|
|
1021
1199
|
for (const id of disableRules) {
|
|
1022
1200
|
if (!RULES.some((r) => r.id === id)) {
|
|
@@ -1176,7 +1354,15 @@ async function commandOptimize(args, config, pricing, t, locale) {
|
|
|
1176
1354
|
// Cursor wants the dollars, and they should not have to leave the editor to
|
|
1177
1355
|
// see them.
|
|
1178
1356
|
const host = detectHost();
|
|
1179
|
-
|
|
1357
|
+
/**
|
|
1358
|
+
* `--from-log` implies `--cost`, and the reasoning is different from the
|
|
1359
|
+
* `--calls` case documented below: `--calls` is a typed scenario parameter,
|
|
1360
|
+
* but a usage log with billed token counts is *evidence* — proof this
|
|
1361
|
+
* prompt's traffic goes to a metered API, whatever the terminal running
|
|
1362
|
+
* the command bills like. Withholding the money there would suppress
|
|
1363
|
+
* exactly the figures the person measured in order to see.
|
|
1364
|
+
*/
|
|
1365
|
+
const tokensOnly = boolFlag(args, 'cost') || measured !== null
|
|
1180
1366
|
? false
|
|
1181
1367
|
: boolFlag(args, 'tokens-only') || host.billing === 'subscription';
|
|
1182
1368
|
/**
|
|
@@ -1195,7 +1381,7 @@ async function commandOptimize(args, config, pricing, t, locale) {
|
|
|
1195
1381
|
const namedScenario = args.flags.has('calls') || args.flags.has('output-tokens');
|
|
1196
1382
|
printReport(result, boolFlag(args, 'diff'), t, examplesReview, reorder, tokensOnly, host, suggestions
|
|
1197
1383
|
? { result: suggestions, applied: boolFlag(args, 'apply-suggestions'), locale }
|
|
1198
|
-
: null, namedScenario);
|
|
1384
|
+
: null, namedScenario, measured);
|
|
1199
1385
|
if (outPath) {
|
|
1200
1386
|
console.log(c.dim(t.report.wroteTo(outPath)));
|
|
1201
1387
|
console.log();
|
|
@@ -1489,10 +1675,17 @@ async function commandProfile(args, config, pricing, t) {
|
|
|
1489
1675
|
const target = await stat(path).catch(() => null);
|
|
1490
1676
|
let logFiles = [path];
|
|
1491
1677
|
if (target?.isDirectory()) {
|
|
1492
|
-
|
|
1678
|
+
/**
|
|
1679
|
+
* Recursive under `--by-source`, flat otherwise. The fleet's whole point
|
|
1680
|
+
* is one directory per service, so the walk must descend; the flat mode
|
|
1681
|
+
* keeps its long-standing behaviour because a directory of rotated logs
|
|
1682
|
+
* with an unrelated subfolder should not quietly absorb it.
|
|
1683
|
+
*/
|
|
1684
|
+
const bySourceMode = boolFlag(args, 'by-source');
|
|
1685
|
+
const entries = await readdir(path, { withFileTypes: true, recursive: bySourceMode });
|
|
1493
1686
|
logFiles = entries
|
|
1494
1687
|
.filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
|
|
1495
|
-
.map((entry) => join(path, entry.name))
|
|
1688
|
+
.map((entry) => join(entry.parentPath ?? path, entry.name))
|
|
1496
1689
|
.sort((a, b) => a.localeCompare(b));
|
|
1497
1690
|
if (logFiles.length === 0) {
|
|
1498
1691
|
throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
|
|
@@ -1510,18 +1703,7 @@ async function commandProfile(args, config, pricing, t) {
|
|
|
1510
1703
|
* alternative — skipping it — is a total quietly missing a day, which is
|
|
1511
1704
|
* the failure this repository refuses in every other place it can occur.
|
|
1512
1705
|
*/
|
|
1513
|
-
const
|
|
1514
|
-
if (!file.endsWith('.gz'))
|
|
1515
|
-
return readFile(file, 'utf8');
|
|
1516
|
-
const compressed = await readFile(file);
|
|
1517
|
-
try {
|
|
1518
|
-
return gunzipSync(compressed).toString('utf8');
|
|
1519
|
-
}
|
|
1520
|
-
catch (error) {
|
|
1521
|
-
throw new Error(t.profile.badGzip(file, error instanceof Error ? error.message : String(error)));
|
|
1522
|
-
}
|
|
1523
|
-
};
|
|
1524
|
-
const logTexts = await Promise.all(logFiles.map((file) => readLog(file)));
|
|
1706
|
+
const logTexts = await Promise.all(logFiles.map((file) => readUsageLog(file, t)));
|
|
1525
1707
|
// A file that does not end in a newline would otherwise glue its last record
|
|
1526
1708
|
// to the next file's first one, and both would be reported as unreadable.
|
|
1527
1709
|
const raw = logTexts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
|
|
@@ -1626,6 +1808,109 @@ async function commandProfile(args, config, pricing, t) {
|
|
|
1626
1808
|
}
|
|
1627
1809
|
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
1628
1810
|
const pct = (share) => `${(share * 100).toFixed(1)}%`;
|
|
1811
|
+
/**
|
|
1812
|
+
* `--by-source`: one report per service, plus the rollup — the fleet.
|
|
1813
|
+
*
|
|
1814
|
+
* A merged bill is right for one service and wrong for twelve: it hides
|
|
1815
|
+
* which service the money comes from, per-service budgets cannot exist,
|
|
1816
|
+
* and the findings a comparison between services could make are invisible.
|
|
1817
|
+
* Files are assigned to sources by the most specific matching glob from the
|
|
1818
|
+
* config's `sources` block; a file matching no source is named loudly,
|
|
1819
|
+
* because a log that silently joined no report is spend missing from every
|
|
1820
|
+
* bill.
|
|
1821
|
+
*/
|
|
1822
|
+
if (boolFlag(args, 'by-source')) {
|
|
1823
|
+
const sourceDefs = config.sources;
|
|
1824
|
+
if (sourceDefs === undefined || Object.keys(sourceDefs).length === 0) {
|
|
1825
|
+
throw new Error(t.profile.bySourceNeedsConfig());
|
|
1826
|
+
}
|
|
1827
|
+
const { bySource, unmatched } = assignSources(logFiles, sourceDefs);
|
|
1828
|
+
if (bySource.size === 0) {
|
|
1829
|
+
throw new Error(t.profile.bySourceNothingMatched(Object.keys(sourceDefs).join(', ')));
|
|
1830
|
+
}
|
|
1831
|
+
const textByFile = new Map(logFiles.map((file, i) => [file, logTexts[i]]));
|
|
1832
|
+
const fleetSources = [];
|
|
1833
|
+
const cacheDeltas = new Map();
|
|
1834
|
+
for (const [name, files] of [...bySource.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
1835
|
+
const text = files
|
|
1836
|
+
.map((file) => textByFile.get(file))
|
|
1837
|
+
.map((chunk) => (chunk.endsWith('\n') ? chunk : `${chunk}\n`))
|
|
1838
|
+
.join('');
|
|
1839
|
+
const sourceReport = profileUsage(text, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
1840
|
+
fleetSources.push({ name, report: sourceReport });
|
|
1841
|
+
cacheDeltas.set(name, cacheEconomics(sourceReport.total).deltaUsd);
|
|
1842
|
+
}
|
|
1843
|
+
const aggregate = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
|
|
1844
|
+
const rollup = fleetRollup(fleetSources, {
|
|
1845
|
+
cacheDeltas,
|
|
1846
|
+
aggregateCacheDelta: cacheEconomics(aggregate.total).deltaUsd,
|
|
1847
|
+
});
|
|
1848
|
+
if (boolFlag(args, 'json')) {
|
|
1849
|
+
console.log(JSON.stringify({
|
|
1850
|
+
schemaVersion: 1,
|
|
1851
|
+
bySource: fleetSources.map((source) => ({ name: source.name, report: source.report })),
|
|
1852
|
+
rollup: {
|
|
1853
|
+
totalUsd: rollup.totalUsd,
|
|
1854
|
+
calls: rollup.calls,
|
|
1855
|
+
sources: rollup.sources,
|
|
1856
|
+
worst: rollup.worst,
|
|
1857
|
+
mismatchedSpans: rollup.mismatchedSpans,
|
|
1858
|
+
splitBrains: rollup.splitBrains,
|
|
1859
|
+
cacheUnderwater: rollup.cacheUnderwater,
|
|
1860
|
+
unmatchedFiles: unmatched,
|
|
1861
|
+
},
|
|
1862
|
+
}, (key, value) => (value instanceof Map ? undefined : value), 2));
|
|
1863
|
+
}
|
|
1864
|
+
else {
|
|
1865
|
+
console.log(c.bold(t.profile.fleetHeading(n(rollup.sources.length), formatUsd(rollup.totalUsd), t.profile.calls(rollup.calls))));
|
|
1866
|
+
for (const row of rollup.sources) {
|
|
1867
|
+
const span = row.spanDays === null ? t.profile.fleetNoClock() : t.profile.fleetSpan(row.spanDays.toFixed(1));
|
|
1868
|
+
console.log(` ${t.profile.fleetRow(row.name, formatUsd(row.usd), pct(row.share), t.profile.calls(row.calls), span)}`);
|
|
1869
|
+
}
|
|
1870
|
+
if (rollup.worst !== null && rollup.sources.length > 1) {
|
|
1871
|
+
console.log();
|
|
1872
|
+
console.log(` ${c.yellow('!')} ${c.bold(wrap(t.profile.fleetWorst(rollup.worst.name, formatUsd(rollup.worst.usd), pct(rollup.worst.share)), 74, ' '))}`);
|
|
1873
|
+
}
|
|
1874
|
+
if (rollup.mismatchedSpans) {
|
|
1875
|
+
console.log(` ${c.dim(wrap(t.profile.fleetMismatchedSpans(), 74, ' '))}`);
|
|
1876
|
+
}
|
|
1877
|
+
for (const split of rollup.splitBrains.slice(0, 3)) {
|
|
1878
|
+
console.log();
|
|
1879
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetSplitBrain(split.label, split.sources.map((v) => `${v.name} → ${v.model} (${formatUsd(v.usd)})`).join(', ')), 74, ' ')}`);
|
|
1880
|
+
}
|
|
1881
|
+
for (const under of rollup.cacheUnderwater.slice(0, 3)) {
|
|
1882
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetCacheUnderwater(under.name, formatUsd(under.deltaUsd)), 74, ' ')}`);
|
|
1883
|
+
}
|
|
1884
|
+
for (const file of unmatched) {
|
|
1885
|
+
console.log(` ${c.yellow('!')} ${wrap(t.profile.fleetUnmatched(file), 74, ' ')}`);
|
|
1886
|
+
}
|
|
1887
|
+
console.log();
|
|
1888
|
+
console.log(` ${c.dim(wrap(t.profile.fleetFooter(), 74, ' '))}`);
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* The per-source gates. Each budget judges its own service and the run
|
|
1892
|
+
* fails naming the service — a total that hides which source crossed its
|
|
1893
|
+
* line is the rendering this mode exists to end. Waivable per source
|
|
1894
|
+
* through `bySource:<name>`, under the same expiry discipline.
|
|
1895
|
+
*/
|
|
1896
|
+
const bySourceBudgets = config.spend?.bySource ?? {};
|
|
1897
|
+
for (const [name, limit] of Object.entries(bySourceBudgets)) {
|
|
1898
|
+
const found = fleetSources.find((source) => source.name === name);
|
|
1899
|
+
if (found === undefined) {
|
|
1900
|
+
console.error(c.dim(t.profile.fleetBudgetMissing(name)));
|
|
1901
|
+
continue;
|
|
1902
|
+
}
|
|
1903
|
+
const usd = found.report.total.totalUsd;
|
|
1904
|
+
if (usd > limit) {
|
|
1905
|
+
console.error(c.red(t.profile.fleetBudgetFailed(name, formatUsd(usd), formatUsd(limit))));
|
|
1906
|
+
process.exitCode = 1;
|
|
1907
|
+
}
|
|
1908
|
+
else {
|
|
1909
|
+
console.error(c.dim(t.profile.fleetBudgetOk(name, formatUsd(usd), formatUsd(limit))));
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
return;
|
|
1913
|
+
}
|
|
1629
1914
|
/**
|
|
1630
1915
|
* `--dry-run`: what this log could and could not answer, and no bill.
|
|
1631
1916
|
*
|
|
@@ -1686,7 +1971,7 @@ async function commandProfile(args, config, pricing, t) {
|
|
|
1686
1971
|
// The same reader as the log itself, so `--against last-month.jsonl.gz`
|
|
1687
1972
|
// works: a comparison that could only read one of the two formats would
|
|
1688
1973
|
// be a flag that fails on exactly the rotated file it exists to read.
|
|
1689
|
-
? profileUsage(await
|
|
1974
|
+
? profileUsage(await readUsageLog(againstPath, t), {
|
|
1690
1975
|
catalogue: pricing,
|
|
1691
1976
|
label: onlyLabel,
|
|
1692
1977
|
// The same window on both sides, for the same reason as the label:
|