@trazum/cli 1.9.0 → 1.10.0

package/dist/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { readFile, stat, writeFile } from 'node:fs/promises';
 import { join, resolve as resolvePath } from 'node:path';
-import { BASELINE_FILENAME, BASELINE_VERSION, MAX_BASELINE_BYTES, breaches, compareToBaseline, formatBaseline, moneyIsComparable, parseBaseline, DEFAULT_USAGE, LOCALES, reviewAgeDays, RULES, comparePrompts, countTokensAnthropic, getModel, applyRewrites, computeSavings, profilePrompt, toPromptfoo, PHRASE_LANGUAGES, estimateTokens, formatUsd, formatSignedUsd, getMessages, nearestName, optimize, toOtlpMetrics, providerFromEnv, reorderForCache, sharedPrefixes, cacheableMinimum, findExamples, plannedCalls, pruneExamples, extractPrompts, promptId, hasMarker, SOURCE_EXTENSIONS, detectFromSource, evaluate, refineWithLlm, rejectionText, suggestRewrites, reviewExamples, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, cacheHitRate, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 // Everything that reads the filesystem, on its own entry point so the web
 // bundle cannot reach it. See packages/core/src/node.ts.
@@ -241,6 +241,7 @@ const COMMAND_FLAGS = {
     ],
     check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
     baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
+    profile: ['json', 'pricing', 'pricing-live'],
     eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
     prune: ['cases', 'concurrency', 'json', 'yes'],
     diff: ['level', 'model', 'calls', 'output-tokens', 'batch', 'max-growth', 'optimized', 'markdown-out', 'all', 'prompt'],
@@ -330,7 +331,7 @@ function renderDiff(before, after, t) {
  * The provider's name when the estimator was not calibrated for it.
  *
  * `estimateTokens` is a heuristic tuned against Claude's tokenizer, and the
- * ±15% band descends from that. Printing the same band beside a GPT or Kimi
+ * ±10% band descends from that. Printing the same band beside a GPT or Kimi
  * figure states a precision nobody has measured for that family — and since the
  * catalogue grew past Anthropic, that is most of it. Returns null when the model
  * is Anthropic's, where the band is at least the claim it was written for.
@@ -353,11 +354,81 @@ function languageNames(codes, t) {
         return names[0] ?? '';
     return `${names.slice(0, -1).join(', ')} ${t.languages.and} ${names[names.length - 1]}`;
 }
+/**
+ * Advisories whose entire pitch is money.
+ *
+ * On a subscription these are not weaker advice, they are not advice: "use a
+ * cheaper model" saves nothing on a flat plan, and its detail text quotes dollars
+ * per month, so suppressing only the price tag beside the title left the money in
+ * the sentence underneath.
+ */
+const MONEY_ONLY_ADVISORIES = new Set([
+    'model-downgrade',
+    'batch-api',
+    'output-dominated',
+    'promo-pricing',
+    'prompt-caching-not-worth-it',
+]);
+/**
+ * The one thing worth doing about this prompt, and how it compares to shortening it.
+ *
+ * `null` when there is nothing to say: no advisory carries a figure, or the
+ * reader is on a subscription where a monthly saving is meaningless. A heading
+ * with a shrug under it is worse than no heading.
+ */
+function biggestLever(result, tokensOnly, t) {
+    /**
+     * One guard, and it is the only thing deciding.
+     *
+     * The first version also filtered the candidate list by `!tokensOnly`, which
+     * duplicated this and made it untestable: removing the guard left the filter
+     * still suppressing the line, so a mutation that priced a subscription passed
+     * the suite. Two checks for one condition is one check and one place for a bug.
+     */
+    if (tokensOnly)
+        return null;
+    const best = result.advisories.find((a) => (a.estimatedMonthlyUsd ?? 0) > 0);
+    if (!best?.estimatedMonthlyUsd)
+        return null;
+    const ruleSaving = result.savings.monthlySavingsUsd;
+    return {
+        line: t.report.biggestLeverDetail(best.title, formatUsd(best.estimatedMonthlyUsd), 
+        // The multiple is the point of the line, and it is only honest when there
+        // is something to divide by. A prompt the rules could not improve at all
+        // gets the amount and no ratio rather than a division by zero dressed up.
+        ruleSaving > 0 ? Math.round(best.estimatedMonthlyUsd / ruleSaving) : null),
+    };
+}
 function printReport(result, showDiff, t, examplesReview = null, reorder = null, tokensOnly = false, host = { id: 'terminal', displayName: 'terminal', billing: 'unknown', evidence: null }, suggestions = null) {
     const n = (value) => value.toLocaleString(t.numberLocale);
     const sourceNote = result.tokenSource === 'heuristic'
         ? c.dim(t.report.estimated(offFamilyName(result.usage.model)))
         : c.dim(t.report.exactCount());
+    /**
+     * The largest lever, first.
+     *
+     * This line used to be the last thing in the report and it is the most useful
+     * thing in it. Measured on an ordinary support prompt — already reasonably
+     * written, which is what a real one is — the rules recover **three tokens of
+     * 306**, worth $0.75 a month, while the cache reorder sitting below them is
+     * worth $48. The report opened with the 1.3% and closed with the 64×.
+     *
+     * That ordering is not a presentation quibble. It teaches the reader that
+     * shortening the prompt is what this tool is for, and on any prompt somebody
+     * competent wrote, shortening it is the smallest thing available. The rules
+     * earn their keep on genuine bloat — a duplicated paragraph, "due to the fact
+     * that" — and recover close to nothing once that is gone, because they recover
+     * waste rather than creating savings.
+     *
+     * So the answer to "what should I do about this prompt" goes at the top, and
+     * the token count follows as the detail it is.
+     */
+    const best = biggestLever(result, tokensOnly, t);
+    if (best) {
+        console.log();
+        console.log(c.bold(t.report.biggestLever()));
+        console.log(`  ${c.dim(wrap(best.line, 74, '  '))}`);
+    }
     console.log();
     console.log(c.bold(t.report.inputTokens()));
     console.log(`  ${n(result.tokensBefore)} → ${c.green(n(result.tokensAfter))}   ${c.bold(`-${result.reductionPct.toFixed(1)}%`)}${sourceNote}`);
@@ -460,13 +531,7 @@ function printReport(result, showDiff, t, examplesReview = null, reorder = null,
     // The rest stay: an overflowing context window still fails the call, a
     // contradiction is still wrong, redundant examples still cost tokens, and
     // caching still buys latency and rate-limit headroom.
-    const MONEY_ONLY = new Set([
-        'model-downgrade',
-        'batch-api',
-        'output-dominated',
-        'promo-pricing',
-        'prompt-caching-not-worth-it',
-    ]);
+    const MONEY_ONLY = MONEY_ONLY_ADVISORIES;
     const advisories = tokensOnly
         ? result.advisories.filter((a) => !MONEY_ONLY.has(a.id))
         : result.advisories;
@@ -497,19 +562,8 @@ function printReport(result, showDiff, t, examplesReview = null, reorder = null,
             console.log(`  ${marker} ${column}${c.bold(advisory.title)}`);
             console.log(`${gutter}${c.dim(wrap(advisory.detail, 78 - gutter.length, gutter))}`);
         }
-        // What to do first. The rules trimmed $1.25 and the top advisory is worth
-        // $506; leaving the reader to notice that by comparing four numbers in four
-        // sentences is how the most valuable line in the report gets skipped.
-        const best = advisories.find((a) => (a.estimatedMonthlyUsd ?? 0) > 0);
-        if (!tokensOnly && best?.estimatedMonthlyUsd) {
-            const ruleSaving = result.savings.monthlySavingsUsd;
-            const line = t.report.biggestLeverDetail(best.title, formatUsd(best.estimatedMonthlyUsd), ruleSaving > 0 ? Math.round(best.estimatedMonthlyUsd / ruleSaving) : null);
-            console.log();
-            // Wrapped to the same width as everything else. An unwrapped closing line
-            // is the one that runs off a narrow terminal, and it is the line most
-            // worth reading.
-            console.log(`  ${c.bold(t.report.biggestLever())} ${c.dim(wrap(line, 62, '  '))}`);
-        }
+        // The "start here" line is printed at the top of the report now, where a
+        // reader who stops after four lines still sees it.
     }
     printSuggestions(suggestions, t, n);
     printRest(result, showDiff, t, examplesReview, n);
@@ -1304,6 +1358,134 @@ function monthlyCostOf(tokens, usage, pricing) {
 function isoDate() {
     return new Date().toISOString().slice(0, 10);
 }
+/**
+ * `trazum profile <log.jsonl>` — where the money actually went.
+ *
+ * Every other command in this file reads a prompt and reasons forward about what
+ * it would cost. This one reads what the provider charged and reasons backward,
+ * and it exists because the forward direction can only see the smallest line item:
+ * on an ordinary support prompt the rules recover about 1% of the monthly figure
+ * while output alone was 87% of it.
+ *
+ * **Money is never suppressed here, unlike every other report.** The rest of the
+ * CLI hides dollar figures on a subscription host, because a saving quoted to
+ * somebody on a flat plan is money that does not exist. This log is a record of
+ * metered API calls somebody was actually billed for — the bill exists wherever
+ * Trazum happens to be running, so the host has no bearing on it.
+ */
+async function commandProfile(args, pricing, t) {
+    const path = args.positional[0];
+    if (path === undefined) {
+        console.log();
+        console.log(c.dim(wrap(t.profile.noTarget(), 74, '  ')));
+        console.log();
+        return;
+    }
+    const raw = await readFile(path, 'utf8');
+    const report = profileUsage(raw, { catalogue: pricing });
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    const pct = (share) => `${(share * 100).toFixed(1)}%`;
+    if (boolFlag(args, 'json')) {
+        console.log(JSON.stringify(report, null, 2));
+        return;
+    }
+    /**
+     * Nothing priced means there is no report, not a report of zero.
+     *
+     * The guard was `total.calls === 0 && unpriced.calls === 0`, so a log whose every
+     * model was unknown fell through and printed a full report built from a zeroed
+     * total: `0 calls · $0`, four `$0 / 0.0%` rows, a meaningless "Input is 0.0% of
+     * this bill", and — on a log containing a hundred thousand cache-read tokens —
+     * the flatly false "Caching was never used on these calls".
+     *
+     * Two affirmatively wrong claims and a $0 headline for a real bill. The trailing
+     * unpriced note was the only correct line on screen, and it was the quietest.
+     */
+    if (report.total.calls === 0) {
+        console.log();
+        console.log(c.dim(report.unpriced.calls === 0 ? t.profile.empty() : t.profile.nothingPriced()));
+        reportProfileGaps(report, t, n);
+        return;
+    }
+    const shares = sharesOf(report.total);
+    const parts = [
+        [t.profile.partInput(), report.total.inputUsd, shares.input, report.total.inputTokens],
+        [t.profile.partCacheRead(), report.total.cacheReadUsd, shares.cacheRead, report.total.cacheReadTokens],
+        [t.profile.partCacheWrite(), report.total.cacheWriteUsd, shares.cacheWrite, report.total.cacheWriteTokens],
+        [t.profile.partOutput(), report.total.outputUsd, shares.output, report.total.outputTokens],
+    ];
+    console.log();
+    console.log(c.bold(t.profile.heading()));
+    console.log(`  ${t.profile.spent(n(report.total.calls), formatUsd(report.total.totalUsd))}`);
+    console.log();
+    // Every part, including the zero ones. A row missing because it was zero reads
+    // as a row somebody forgot, and "you are not caching at all" is a finding.
+    for (const [name, usd, share, tokens] of parts) {
+        console.log(`  ${c.dim(t.profile.part(name, formatUsd(usd), pct(share), n(tokens)))}`);
+    }
+    /**
+     * The line the command exists for: which part of the bill to argue with.
+     *
+     * When output is both the biggest part and over half, the two sentences say the
+     * same thing and the second says more — so only the second prints. Reporting a
+     * fact twice in adjacent lines reads as a bug, and it was one.
+     */
+    const [biggestName, , biggestShare] = parts.reduce((a, b) => (b[1] > a[1] ? b : a));
+    const outputDominates = shares.output > 0.5;
+    console.log();
+    if (outputDominates) {
+        console.log(`  ${c.bold(wrap(t.profile.outputDominates(pct(shares.output)), 74, '  '))}`);
+    }
+    else {
+        console.log(`  ${c.bold(t.profile.biggestPart(biggestName, pct(biggestShare)))}`);
+    }
+    const hitRate = cacheHitRate(report.total);
+    console.log(hitRate === null
+        ? `  ${c.dim(wrap(t.profile.cacheNever(), 74, '  '))}`
+        : `  ${c.dim(t.profile.cacheHit(pct(hitRate)))}`);
+    /**
+     * A total that assumed a cache-write rate is a floor, and says so.
+     *
+     * Anthropic's 1-hour entry costs 2x input against the 5-minute entry's 1.25x. A
+     * log carrying only the flat `cache_creation_input_tokens` cannot say which, so
+     * the cheaper one is used — and the flattering direction is exactly the one this
+     * tool refuses to take quietly.
+     */
+    if (report.total.assumedWriteTtlCalls > 0) {
+        console.log(`  ${c.dim(wrap(t.profile.assumedWriteTtl(report.total.assumedWriteTtlCalls), 74, '  '))}`);
+    }
+    for (const [heading, rows] of [
+        [t.profile.byLabelHeading(), report.byLabel.map((r) => [r.label === UNLABELLED ? t.profile.unlabelled() : r.label, r.breakdown])],
+        [t.profile.byModelHeading(), report.byModel.map((r) => [r.model, r.breakdown])],
+    ]) {
+        if (rows.length <= 1)
+            continue; // One row is the total again, said twice.
+        console.log();
+        console.log(c.bold(heading));
+        for (const [name, breakdown] of rows) {
+            const share = report.total.totalUsd > 0 ? breakdown.totalUsd / report.total.totalUsd : 0;
+            console.log(`  ${t.profile.row(name, formatUsd(breakdown.totalUsd), pct(share), n(breakdown.calls))}`);
+        }
+    }
+    reportProfileGaps(report, t, n);
+}
+/**
+ * What the profile could not account for, said out loud.
+ *
+ * Separated so both the empty and the populated path print it. A total that
+ * silently omits calls is wrong in the flattering direction, which is the fault
+ * this repository keeps finding in itself.
+ */
+function reportProfileGaps(report, t, n) {
+    if (report.unpricedModels.length > 0) {
+        console.log();
+        console.log(`  ${c.yellow('!')} ${c.dim(wrap(t.profile.unpriced(report.unpricedModels.join(', '), report.unpriced.calls), 74, '    '))}`);
+    }
+    if (report.skippedLines.length > 0) {
+        const shown = report.skippedLines.slice(0, 5).join(', ');
+        console.log(`  ${c.dim(t.profile.skipped(report.skippedLines.length, report.skippedLines.length > 5 ? `${shown}…` : shown))}`);
+    }
+}
 /**
  * `trazum baseline <dir>` — record what the estate costs now.
  *
@@ -2401,6 +2583,9 @@ async function main() {
         case 'baseline':
             await commandBaseline(args, config, pricing, t, locale);
             break;
+        case 'profile':
+            await commandProfile(args, pricing, t);
+            break;
         case 'eval':
             await commandEval(args, config, t, locale);
             break;