@trazum/cli 1.29.0 → 1.31.0

package/dist/index.js

@@ -2,7 +2,7 @@
 import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { join, resolve as resolvePath } from 'node:path';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, driversBetween, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -1718,6 +1718,19 @@ async function commandProfile(args, config, pricing, t) {
      * profiles yesterday's log has a daily budget without Trazum ever
      * guessing what a day is.
      */
+    /**
+     * The gate verdicts, kept so the markdown summary can carry them.
+     *
+     * Collected by wrapping `console.error` for the duration of `applyGates`
+     * rather than by threading a return value through every gate. That is the
+     * unusual choice here and it is deliberate: a gate added later reaches the
+     * summary without anyone remembering to register it, and the alternative —
+     * one push per verdict at a dozen call sites — is a list that goes stale
+     * silently. Colour is stripped, because a summary is markdown and an
+     * escape sequence in it is noise a reader has to look past.
+     */
+    const gateVerdicts = [];
+    let gateFailed = false;
     const applyGates = () => {
         /**
          * Before any verdict: whether the gated figure is the whole bill. A gate
@@ -1777,21 +1790,85 @@ async function commandProfile(args, config, pricing, t) {
             // for a period the caller did not name would gate against a slice.
             console.error(c.dim(t.profile.labelBudgetWindowed()));
         }
+        /**
+         * Why a gate failed and how much room a pass had — written once, called by
+         * every gate, because four hand-rolled copies of the same three sentences
+         * is four chances for one of them to soften.
+         */
+        const explainFailure = (overUsd, { namesLargest = false } = {}) => {
+            const why = explainGateFailure(report, levers, overUsd);
+            // The day gate already names its own day's biggest label; repeating the
+            // whole bill's biggest slice under it reads as the same sentence twice.
+            if (why.largest !== null && !namesLargest) {
+                const name = why.largest.label === UNLABELLED ? t.profile.unlabelled() : why.largest.label;
+                console.error(c.dim(wrap(t.profile.gateLargest(name, why.largest.model, formatUsd(why.largest.usd), pct(why.largest.share)), 74, '  ')));
+            }
+            if (why.lever !== null) {
+                const leverName = why.lever.label === UNLABELLED ? t.profile.unlabelled() : why.lever.label;
+                // The action, not the slice's current model: a slice with only a batch
+                // price has no destination, and naming the model it already runs on as
+                // somewhere to move it would be plainly false.
+                const route = why.lever.route;
+                const action = route !== null && why.lever.batch !== null
+                    ? t.profile.gateLeverBoth(route.candidate.displayName)
+                    : route !== null
+                        ? t.profile.gateLeverRoute(route.candidate.displayName)
+                        : t.profile.gateLeverBatch();
+                console.error(c.dim(wrap(t.profile.gateLever(leverName, action, formatUsd(why.lever.combinedUsd), formatUsd(why.overageUsd), why.coversIt), 74, '  ')));
+            }
+        };
+        /** How much room a pass had, said only when tight, threshold in the copy. */
+        const explainMargin = (judgedUsd, limitUsd) => {
+            const margin = gateMargin(judgedUsd, limitUsd);
+            if (margin !== null && margin < GATE_MARGIN_TIGHT) {
+                console.error(c.yellow(wrap(t.profile.gateMarginTight(pct(margin), formatUsd(limitUsd - judgedUsd)), 74, '  ')));
+            }
+        };
         if (typeof args.flags.get('max-usd') === 'string' || config.spend?.maxUsd !== undefined) {
             const maxUsd = typeof args.flags.get('max-usd') === 'string'
                 ? numberFlag(args, 'max-usd', 0, t)
                 : config.spend.maxUsd;
             if (report.total.totalUsd > maxUsd) {
                 console.error(c.red(t.profile.maxUsdFailed(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
+                /**
+                 * What to change, next to the fact that something must. A red build in
+                 * CI is the one place nobody opens the full report, so the failure
+                 * carries its own next step: which slice holds the money, and the one
+                 * lever the report already priced. Nothing here is a recommendation —
+                 * whether that model can do the work is the reader's to judge, and the
+                 * copy says so.
+                 */
+                explainFailure(report.total.totalUsd - maxUsd);
                 process.exitCode = 1;
             }
             else {
                 console.error(c.dim(t.profile.maxUsdOk(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
+                explainMargin(report.total.totalUsd, maxUsd);
             }
         }
         if (typeof args.flags.get('max-growth-usd') === 'string' && againstDelta !== null) {
             const maxGrowth = numberFlag(args, 'max-growth-usd', 0, t);
-            if (againstDelta > maxGrowth) {
+            /**
+             * A comparison that went blind fails before it is judged.
+             *
+             * The dollars can hold flat while the current log stopped recording a
+             * field the previous one carried — and every finding that needed the
+             * field is now silent for a reason that has nothing to do with spend.
+             * A gate passing there would be certifying a comparison it could not
+             * make: "not measured" is not "did not grow", the same refusal
+             * --max-day-usd makes on a clockless log and --max-session-usd on a
+             * sessionless one. Only a collapse fails; a field that appeared means
+             * this side can see more, which is never a reason to refuse.
+             */
+            const blinded = previous !== null
+                ? coverageDrift(previous.fieldCoverage, report.fieldCoverage).filter((d) => d.delta < 0)
+                : [];
+            const worst = blinded[0];
+            if (worst !== undefined) {
+                console.error(c.red(t.profile.maxGrowthCoverageLost(blinded.map((d) => t.profile.coverageField(d.field)).join(', '), pct(worst.was), pct(worst.now))));
+                process.exitCode = 1;
+            }
+            else if (againstDelta > maxGrowth) {
                 console.error(c.red(t.profile.maxGrowthUsdFailed(formatSignedUsd(againstDelta), formatUsd(maxGrowth))));
                 process.exitCode = 1;
             }
@@ -1857,10 +1934,12 @@ async function commandProfile(args, config, pricing, t) {
                     : '';
                 if (worst.usd > maxDay) {
                     console.error(c.red(`${t.profile.maxDayFailed(worst.day, formatUsd(worst.usd), formatUsd(maxDay))}${suspect}`));
+                    explainFailure(worst.usd - maxDay, { namesLargest: true });
                     process.exitCode = 1;
                 }
                 else {
                     console.error(c.dim(t.profile.maxDayOk(worst.day, formatUsd(worst.usd), formatUsd(maxDay))));
+                    explainMargin(worst.usd, maxDay);
                     /**
                      * Calls with no clock are in the bill above and in no day below, so
                      * the worst day is a floor by exactly that much. Said only on a
@@ -1896,13 +1975,39 @@ async function commandProfile(args, config, pricing, t) {
             }
             else if (report.sessionSpend.maxUsd > maxSession) {
                 console.error(c.red(t.profile.maxSessionFailed(formatUsd(report.sessionSpend.maxUsd), formatUsd(maxSession), n(report.sessionSpend.sessions))));
+                explainFailure(report.sessionSpend.maxUsd - maxSession);
                 process.exitCode = 1;
             }
             else {
                 console.error(c.dim(t.profile.maxSessionOk(formatUsd(report.sessionSpend.maxUsd), formatUsd(maxSession), n(report.sessionSpend.sessions))));
+                explainMargin(report.sessionSpend.maxUsd, maxSession);
             }
         }
     };
+    /**
+     * Run the gates, keeping what they said. Exit codes and stderr behave
+     * exactly as before — this only also remembers, so `--markdown-out` can put
+     * the verdict where the person reading CI will actually see it.
+     */
+    const recordGates = () => {
+        const original = console.error;
+        console.error = (...parts) => {
+            const text = parts.map((part) => String(part)).join(' ');
+            // Colour stripped and the terminal's wrap collapsed: markdown re-wraps
+            // to its own width, and the escape sequences and hanging indents that
+            // make a terminal readable are noise a summary reader looks past.
+            // eslint-disable-next-line no-control-regex
+            gateVerdicts.push(text.replace(/\u001b\[[0-9;]*m/g, '').replace(/\s+/g, ' ').trim());
+            original(...parts);
+        };
+        try {
+            applyGates();
+        }
+        finally {
+            console.error = original;
+            gateFailed = process.exitCode === 1;
+        }
+    };
     /**
      * The side files the caller asked for. Written on **both** output paths:
      * under --json the human rendering returns early, and the first version of
@@ -1936,6 +2041,9 @@ async function commandProfile(args, config, pricing, t) {
                     ? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
                     : {}),
                 ...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
+                // The verdict, where the person reading CI will see it. recordGates()
+                // runs before the side files for exactly this.
+                ...(gateVerdicts.length > 0 ? { gates: { failed: gateFailed, lines: gateVerdicts } } : {}),
                 // The repricing, when --what-if was given: computed once above and
                 // handed over, so the summary in a pull request cannot disagree
                 // with the terminal about what a move would cost.
@@ -2045,8 +2153,8 @@ async function commandProfile(args, config, pricing, t) {
             // without the caveat being in the same object.
             ...(whatIf !== null ? { whatIf } : {}),
         }, null, 2));
+        recordGates();
         await writeSideFiles();
-        applyGates();
         return;
     }
     /**
@@ -2945,6 +3053,40 @@ async function commandProfile(args, config, pricing, t) {
                     console.log(`  ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
                 }
             }
+            /**
+             * What the comparison stopped being able to see.
+             *
+             * Every figure above is dollars, and dollars cannot tell a finding that
+             * was fixed from a finding whose field the log stopped recording — both
+             * are silence. This is the only section that can, so it is loud: a
+             * collapse in coverage invalidates whichever findings depended on it,
+             * and reading the drop as good news is the specific mistake it exists
+             * to prevent.
+             */
+            const drifts = coverageDrift(previous.fieldCoverage, report.fieldCoverage);
+            if (drifts.length > 0) {
+                console.log();
+                for (const drift of drifts) {
+                    const line = t.profile.coverageDrift(t.profile.coverageField(drift.field), pct(drift.was), pct(drift.now));
+                    console.log(drift.delta < 0
+                        ? `  ${c.yellow('!')} ${c.bold(wrap(line, 74, '    '))}`
+                        : `  ${c.dim(wrap(line, 74, '  '))}`);
+                    /**
+                     * Which findings went with it, named. "Some findings are silent" is
+                     * not something a reader can act on; knowing that conversation
+                     * growth and the cache-TTL fit are now silence rather than absence
+                     * tells them exactly which sections of this report to distrust.
+                     */
+                    if (drift.delta < 0) {
+                        const silenced = t.profile.coverageSilenced(drift.field);
+                        if (silenced !== '')
+                            console.log(`    ${c.dim(wrap(silenced, 72, '    '))}`);
+                    }
+                }
+                if (drifts.some((d) => d.delta < 0)) {
+                    console.log(`  ${c.dim(wrap(t.profile.coverageDriftWhy(), 74, '    '))}`);
+                }
+            }
         }
     }
     for (const [heading, rows] of [
@@ -3000,8 +3142,8 @@ async function commandProfile(args, config, pricing, t) {
         }
     }
     reportProfileGaps(report, t, n, pricingStale);
+    recordGates();
     await writeSideFiles();
-    applyGates();
 }
 /**
  * What the profile could not account for, said out loud.