@trazum/cli 1.29.0 → 1.31.0

package/src/index.ts

@@ -20,7 +20,11 @@ import {
   countTokensAnthropic,
   DEFAULT_USAGE,
   detectFromSource,
+  coverageDrift,
   driversBetween,
+  explainGateFailure,
+  gateMargin,
+  GATE_MARGIN_TIGHT,
   estimateTokens,
   evaluate,
   extractPrompts,
@@ -2249,6 +2253,19 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
    * profiles yesterday's log has a daily budget without Trazum ever
    * guessing what a day is.
    */
+  /**
+   * The gate verdicts, kept so the markdown summary can carry them.
+   *
+   * Collected by wrapping `console.error` for the duration of `applyGates`
+   * rather than by threading a return value through every gate. That is the
+   * unusual choice here and it is deliberate: a gate added later reaches the
+   * summary without anyone remembering to register it, and the alternative —
+   * one push per verdict at a dozen call sites — is a list that goes stale
+   * silently. Colour is stripped, because a summary is markdown and an
+   * escape sequence in it is noise a reader has to look past.
+   */
+  const gateVerdicts: string[] = [];
+  let gateFailed = false;
   const applyGates = (): void => {
     /**
      * Before any verdict: whether the gated figure is the whole bill. A gate
@@ -2306,6 +2323,52 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
       console.error(c.dim(t.profile.labelBudgetWindowed()));
     }
 
+    /**
+     * Why a gate failed and how much room a pass had — written once, called by
+     * every gate, because four hand-rolled copies of the same three sentences
+     * is four chances for one of them to soften.
+     */
+    const explainFailure = (overUsd: number, { namesLargest = false } = {}): void => {
+      const why = explainGateFailure(report, levers, overUsd);
+      // The day gate already names its own day's biggest label; repeating the
+      // whole bill's biggest slice under it reads as the same sentence twice.
+      if (why.largest !== null && !namesLargest) {
+        const name = why.largest.label === UNLABELLED ? t.profile.unlabelled() : why.largest.label;
+        console.error(
+          c.dim(wrap(t.profile.gateLargest(name, why.largest.model, formatUsd(why.largest.usd), pct(why.largest.share)), 74, '  ')),
+        );
+      }
+      if (why.lever !== null) {
+        const leverName = why.lever.label === UNLABELLED ? t.profile.unlabelled() : why.lever.label;
+        // The action, not the slice's current model: a slice with only a batch
+        // price has no destination, and naming the model it already runs on as
+        // somewhere to move it would be plainly false.
+        const route = why.lever.route;
+        const action =
+          route !== null && why.lever.batch !== null
+            ? t.profile.gateLeverBoth(route.candidate.displayName)
+            : route !== null
+              ? t.profile.gateLeverRoute(route.candidate.displayName)
+              : t.profile.gateLeverBatch();
+        console.error(
+          c.dim(
+            wrap(
+              t.profile.gateLever(leverName, action, formatUsd(why.lever.combinedUsd), formatUsd(why.overageUsd), why.coversIt),
+              74,
+              '  ',
+            ),
+          ),
+        );
+      }
+    };
+    /** How much room a pass had, said only when tight, threshold in the copy. */
+    const explainMargin = (judgedUsd: number, limitUsd: number): void => {
+      const margin = gateMargin(judgedUsd, limitUsd);
+      if (margin !== null && margin < GATE_MARGIN_TIGHT) {
+        console.error(c.yellow(wrap(t.profile.gateMarginTight(pct(margin), formatUsd(limitUsd - judgedUsd)), 74, '  ')));
+      }
+    };
+
     if (typeof args.flags.get('max-usd') === 'string' || config.spend?.maxUsd !== undefined) {
       const maxUsd =
         typeof args.flags.get('max-usd') === 'string'
@@ -2313,14 +2376,51 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
           : config.spend!.maxUsd!;
       if (report.total.totalUsd > maxUsd) {
         console.error(c.red(t.profile.maxUsdFailed(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
+        /**
+         * What to change, next to the fact that something must. A red build in
+         * CI is the one place nobody opens the full report, so the failure
+         * carries its own next step: which slice holds the money, and the one
+         * lever the report already priced. Nothing here is a recommendation —
+         * whether that model can do the work is the reader's to judge, and the
+         * copy says so.
+         */
+        explainFailure(report.total.totalUsd - maxUsd);
         process.exitCode = 1;
       } else {
         console.error(c.dim(t.profile.maxUsdOk(formatUsd(report.total.totalUsd), formatUsd(maxUsd))));
+        explainMargin(report.total.totalUsd, maxUsd);
       }
     }
     if (typeof args.flags.get('max-growth-usd') === 'string' && againstDelta !== null) {
       const maxGrowth = numberFlag(args, 'max-growth-usd', 0, t);
-      if (againstDelta > maxGrowth) {
+      /**
+       * A comparison that went blind fails before it is judged.
+       *
+       * The dollars can hold flat while the current log stopped recording a
+       * field the previous one carried — and every finding that needed the
+       * field is now silent for a reason that has nothing to do with spend.
+       * A gate passing there would be certifying a comparison it could not
+       * make: "not measured" is not "did not grow", the same refusal
+       * --max-day-usd makes on a clockless log and --max-session-usd on a
+       * sessionless one. Only a collapse fails; a field that appeared means
+       * this side can see more, which is never a reason to refuse.
+       */
+      const blinded = previous !== null
+        ? coverageDrift(previous.fieldCoverage, report.fieldCoverage).filter((d) => d.delta < 0)
+        : [];
+      const worst = blinded[0];
+      if (worst !== undefined) {
+        console.error(
+          c.red(
+            t.profile.maxGrowthCoverageLost(
+              blinded.map((d) => t.profile.coverageField(d.field)).join(', '),
+              pct(worst.was),
+              pct(worst.now),
+            ),
+          ),
+        );
+        process.exitCode = 1;
+      } else if (againstDelta > maxGrowth) {
         console.error(c.red(t.profile.maxGrowthUsdFailed(formatSignedUsd(againstDelta), formatUsd(maxGrowth))));
         process.exitCode = 1;
       }
@@ -2399,9 +2499,11 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
           console.error(
             c.red(`${t.profile.maxDayFailed(worst.day, formatUsd(worst.usd), formatUsd(maxDay))}${suspect}`),
           );
+          explainFailure(worst.usd - maxDay, { namesLargest: true });
           process.exitCode = 1;
         } else {
           console.error(c.dim(t.profile.maxDayOk(worst.day, formatUsd(worst.usd), formatUsd(maxDay))));
+          explainMargin(worst.usd, maxDay);
           /**
            * Calls with no clock are in the bill above and in no day below, so
            * the worst day is a floor by exactly that much. Said only on a
@@ -2439,15 +2541,41 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
         console.error(
           c.red(t.profile.maxSessionFailed(formatUsd(report.sessionSpend.maxUsd), formatUsd(maxSession), n(report.sessionSpend.sessions))),
         );
+        explainFailure(report.sessionSpend.maxUsd - maxSession);
         process.exitCode = 1;
       } else {
         console.error(
           c.dim(t.profile.maxSessionOk(formatUsd(report.sessionSpend.maxUsd), formatUsd(maxSession), n(report.sessionSpend.sessions))),
         );
+        explainMargin(report.sessionSpend.maxUsd, maxSession);
       }
     }
   };
 
+  /**
+   * Run the gates, keeping what they said. Exit codes and stderr behave
+   * exactly as before — this only also remembers, so `--markdown-out` can put
+   * the verdict where the person reading CI will actually see it.
+   */
+  const recordGates = (): void => {
+    const original = console.error;
+    console.error = (...parts: unknown[]): void => {
+      const text = parts.map((part) => String(part)).join(' ');
+      // Colour stripped and the terminal's wrap collapsed: markdown re-wraps
+      // to its own width, and the escape sequences and hanging indents that
+      // make a terminal readable are noise a summary reader looks past.
+      // eslint-disable-next-line no-control-regex
+      gateVerdicts.push(text.replace(/\u001b\[[0-9;]*m/g, '').replace(/\s+/g, ' ').trim());
+      original(...(parts as []));
+    };
+    try {
+      applyGates();
+    } finally {
+      console.error = original;
+      gateFailed = process.exitCode === 1;
+    }
+  };
+
   /**
    * The side files the caller asked for. Written on **both** output paths:
    * under --json the human rendering returns early, and the first version of
@@ -2483,6 +2611,9 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
             ? { window: { since: stringFlag(args, 'since') ?? '—', until: stringFlag(args, 'until') ?? '—' } }
             : {}),
           ...(pricingStale !== null ? { stalePricing: pricingStale } : {}),
+          // The verdict, where the person reading CI will see it. recordGates()
+          // runs before the side files for exactly this.
+          ...(gateVerdicts.length > 0 ? { gates: { failed: gateFailed, lines: gateVerdicts } } : {}),
           // The repricing, when --what-if was given: computed once above and
           // handed over, so the summary in a pull request cannot disagree
           // with the terminal about what a move would cost.
@@ -2608,8 +2739,8 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
         2,
       ),
     );
+    recordGates();
     await writeSideFiles();
-    applyGates();
     return;
   }
 
@@ -3643,6 +3774,46 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
           console.log(`  ${d.delta > 0 ? c.yellow(line) : c.dim(line)}`);
         }
       }
+
+      /**
+       * What the comparison stopped being able to see.
+       *
+       * Every figure above is dollars, and dollars cannot tell a finding that
+       * was fixed from a finding whose field the log stopped recording — both
+       * are silence. This is the only section that can, so it is loud: a
+       * collapse in coverage invalidates whichever findings depended on it,
+       * and reading the drop as good news is the specific mistake it exists
+       * to prevent.
+       */
+      const drifts = coverageDrift(previous.fieldCoverage, report.fieldCoverage);
+      if (drifts.length > 0) {
+        console.log();
+        for (const drift of drifts) {
+          const line = t.profile.coverageDrift(
+            t.profile.coverageField(drift.field),
+            pct(drift.was),
+            pct(drift.now),
+          );
+          console.log(
+            drift.delta < 0
+              ? `  ${c.yellow('!')} ${c.bold(wrap(line, 74, '    '))}`
+              : `  ${c.dim(wrap(line, 74, '  '))}`,
+          );
+          /**
+           * Which findings went with it, named. "Some findings are silent" is
+           * not something a reader can act on; knowing that conversation
+           * growth and the cache-TTL fit are now silence rather than absence
+           * tells them exactly which sections of this report to distrust.
+           */
+          if (drift.delta < 0) {
+            const silenced = t.profile.coverageSilenced(drift.field);
+            if (silenced !== '') console.log(`    ${c.dim(wrap(silenced, 72, '    '))}`);
+          }
+        }
+        if (drifts.some((d) => d.delta < 0)) {
+          console.log(`  ${c.dim(wrap(t.profile.coverageDriftWhy(), 74, '    '))}`);
+        }
+      }
     }
   }
 
@@ -3700,9 +3871,9 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
 
   reportProfileGaps(report, t, n, pricingStale);
 
-  await writeSideFiles();
+  recordGates();
 
-  applyGates();
+  await writeSideFiles();
 }
 
 /**

package/src/markdown.ts

@@ -723,6 +723,16 @@ export interface ProfileMarkdownInput {
   levers: BillLevers;
   cache: CacheEconomics;
   t: CliMessages;
+  /**
+   * The gate verdicts, when any gate was armed.
+   *
+   * They reached the terminal on stderr and stopped there, so a CI run
+   * summary carried the whole report and not the one sentence explaining why
+   * the build was red — the reader had to open the raw log to find it. These
+   * arrive already rendered by the caller, which owns the thresholds and the
+   * copy; this is a rendering and must not decide anything a gate decides.
+   */
+  gates?: { failed: boolean; lines: string[] };
   /**
    * The `--since`/`--until` values as the user typed them, when a window was
    * applied. Passed through rather than re-derived from `timeWindow`'s epoch
@@ -783,7 +793,7 @@ export interface ProfileMarkdownInput {
  * reading CI instead of machines.
  */
 export function renderProfileMarkdown(input: ProfileMarkdownInput): string {
-  const { report, levers, cache, t, window, stalePricing, against, whatIf, pressure = [] } = input;
+  const { report, levers, cache, t, window, stalePricing, against, whatIf, pressure = [], gates } = input;
   const n = (value: number): string => value.toLocaleString(t.numberLocale);
   const pct = (share: number): string => `${(share * 100).toFixed(1)}%`;
   const shares = sharesOf(report.total);
@@ -793,6 +803,24 @@ export function renderProfileMarkdown(input: ProfileMarkdownInput): string {
   const lines: string[] = [];
   lines.push(`### ${t.profile.heading()}`);
   lines.push('');
+  /**
+   * The verdict first, when a gate was armed.
+   *
+   * A run summary that carried the whole report and not the sentence
+   * explaining why the build is red made the reader open the raw log to find
+   * it — which is the same failure the explanation itself exists to fix, one
+   * surface further out. A failure is quoted so it survives being skimmed; a
+   * pass is stated plainly and does not shout.
+   */
+  if (gates !== undefined && gates.lines.length > 0) {
+    // One mark, on the verdict. The lines under it explain that verdict and
+    // are not themselves failures — marking each would turn one red build
+    // into a wall of crosses and make the actual verdict harder to find.
+    const [verdict, ...rest] = gates.lines;
+    lines.push(gates.failed ? `> ❌ **${mdText(verdict!)}**` : `_${mdText(verdict!)}_`);
+    for (const line of rest) lines.push(gates.failed ? `> ${mdText(line)}` : `_${mdText(line)}_`);
+    lines.push('');
+  }
   lines.push(`**${t.profile.spent(t.profile.calls(report.total.calls), formatUsd(report.total.totalUsd))}**`);
   lines.push('');
   // The span, under the same rule as the terminal: stated, never extrapolated,