@trazum/cli 1.37.0 → 1.39.0

package/src/index.ts

@@ -11,6 +11,8 @@ import {
   cacheableMinimum,
   analyzeCachePrefix,
   billLevers,
+  buildPlan,
+  verifyPlan,
   cacheEconomics,
   cacheHitRate,
   contextPressure,
@@ -78,6 +80,8 @@ import { dayOf, formatGap, median, spanDays } from './time.js';
 import type {
   FleetSource,
   MeasuredUsage,
+  PlanDocument,
+  VerifiedAction,
   BaselineBreach,
   BaselineChange,
   BaselineComparison,
@@ -172,6 +176,7 @@ interface Args {
 const VALUE_FLAGS = new Set([
   'against',
   'from-log',
+  'min-usd',
   // `route` takes a path here, and the flag is deliberately not `--prompt`:
   // everywhere else in this tool `--prompt` names a marked prompt *inside* a
   // source file, and reusing it for a path would be a trap laid for the reader.
@@ -312,6 +317,13 @@ function levelFlag(args: Args, config: TrazumConfig, t: CliMessages): RuleLevel
  * model id. It beats the default because reading the code is better than
  * assuming, and loses to config because being told is better than reading.
  */
+/**
+ * The file names a usage log answers to, shared by every command that reads a
+ * directory of them. One list, because two commands disagreeing on what counts
+ * as a log would be the same directory billing differently by verb.
+ */
+const LOG_EXTENSIONS = ['.jsonl', '.ndjson', '.log', '.json'];
+
 /**
  * One usage log, gzip included, shared by every command that reads one.
  *
@@ -465,6 +477,8 @@ const COMMAND_FLAGS: Record<string, string[]> = {
   check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
   baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
   profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
+  plan: ['json', 'out', 'markdown-out', 'min-usd', 'pricing', 'pricing-live'],
+  verify: ['against', 'gate', 'json', 'markdown-out', 'pricing', 'pricing-live'],
   route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
   eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
   prune: ['cases', 'concurrency', 'json', 'yes'],
@@ -2204,6 +2218,274 @@ function isoDate(): string {
  * metered API calls somebody was actually billed for — the bill exists wherever
  * Trazum happens to be running, so the host has no bearing on it.
  */
+/**
+ * `trazum verify <plan.json> --against <newer.jsonl|dir>` — did it work?
+ *
+ * The plan predicted; this holds the prediction to the log that came after
+ * it. Three outcomes and never two — arrived, did not arrive, cannot be told
+ * — because "cannot be told" rendered as "arrived" is how every other tool
+ * congratulates a team for a workload that merely vanished. With `--gate`,
+ * a broken promise is a failing exit code: a different and more useful gate
+ * than "spend went up".
+ */
+async function commandVerify(
+  args: Args,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
+  const planPath = args.positional[0];
+  if (planPath === undefined) throw new Error(t.verify.noTarget());
+  const againstPath = stringFlag(args, 'against');
+  if (againstPath === undefined) throw new Error(t.verify.needsAgainst());
+
+  let plan: PlanDocument & { createdAt?: string };
+  try {
+    const parsed = JSON.parse(await readFile(planPath, 'utf8'));
+    if (parsed?.schemaVersion !== 1 || !Array.isArray(parsed.actions)) {
+      throw new Error(t.verify.badPlan(planPath));
+    }
+    plan = parsed;
+  } catch (error) {
+    if (error instanceof SyntaxError) throw new Error(t.verify.badPlan(planPath));
+    throw error;
+  }
+
+  const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
+  const READABLE = [...LOG_EXTENSIONS, ...GZ];
+  const target = await stat(againstPath).catch(() => null);
+  let files: string[] = [againstPath];
+  if (target?.isDirectory()) {
+    const entries = await readdir(againstPath, { withFileTypes: true });
+    files = entries
+      .filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
+      .map((entry) => join(againstPath, entry.name))
+      .sort((a, b) => a.localeCompare(b));
+    if (files.length === 0) throw new Error(t.profile.noLogsInDirectory(againstPath, READABLE.join(', ')));
+  }
+  const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
+  const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
+  const report = profileUsage(raw, { catalogue: pricing });
+
+  const verification = verifyPlan(plan, report, { currentPricingLastReviewed: pricing.lastReviewed });
+  const gate = boolFlag(args, 'gate');
+  const n = (value: number): string => value.toLocaleString(t.numberLocale);
+
+  const lines = (md: boolean): string[] => {
+    const out: string[] = [];
+    const actionLine = (v: VerifiedAction): string[] => {
+      const name = v.action.label === UNLABELLED ? t.profile.unlabelled() : v.action.label;
+      const rows: string[] = [];
+      rows.push(t.verify.action(v.action.kind, name, v.action.model, v.outcome));
+      if (v.outcome === 'cannot-tell' && v.reason !== null) rows.push(t.verify.reason(v.reason));
+      if (v.action.kind === 'route' || v.action.kind === 'route+batch') {
+        if (v.outcome !== 'cannot-tell') {
+          rows.push(
+            t.verify.routeObserved(
+              String(v.observed.dearestModel ?? ''),
+              formatUsd(Number(v.observed.onTargetUsd ?? 0)),
+              formatUsd(Number(v.observed.onOldModelUsd ?? 0)),
+            ),
+          );
+        }
+        if (v.action.kind === 'route+batch' && v.outcome !== 'cannot-tell') rows.push(t.verify.batchUnobservable());
+      }
+      if (v.action.kind === 'fix-truncation' && v.outcome === 'not-arrived') {
+        rows.push(t.verify.truncationObserved(formatUsd(Number(v.observed.retryBillUsd ?? 0))));
+      }
+      if (v.action.kind === 'fix-caching' && v.outcome !== 'cannot-tell') {
+        rows.push(t.verify.cacheObserved(formatUsd(Number(v.observed.deltaUsd ?? 0)), v.outcome));
+      }
+      if (v.attribution?.calls !== undefined) {
+        rows.push(
+          t.verify.attribution(
+            n(Math.round(v.attribution.calls.before)),
+            n(Math.round(v.attribution.calls.after)),
+            n(Math.round(v.attribution.outputPerCallTokens?.before ?? 0)),
+            n(Math.round(v.attribution.outputPerCallTokens?.after ?? 0)),
+          ),
+        );
+      }
+      return rows;
+    };
+
+    const heading = t.verify.heading(
+      n(verification.actions.length),
+      verification.planCreatedAt === null ? null : verification.planCreatedAt.slice(0, 10),
+    );
+    out.push(md ? `## ${heading}` : heading);
+    out.push(
+      t.verify.counts(n(verification.arrived), n(verification.notArrived), n(verification.cannotTell)),
+    );
+    if (verification.pricesChanged) {
+      out.push(t.verify.pricesChanged(verification.planPricing, verification.currentPricing));
+    }
+    for (const v of verification.actions) {
+      out.push('');
+      const [head, ...rest] = actionLine(v);
+      out.push(md ? `### ${head}` : `→ ${head}`);
+      for (const row of rest) out.push(md ? `- ${row}` : `  · ${row}`);
+    }
+    out.push('');
+    out.push(t.verify.footer());
+    return out;
+  };
+
+  await writeMarkdown(args, () => lines(true).join('\n'));
+
+  if (boolFlag(args, 'json')) {
+    console.log(JSON.stringify(verification, null, 2));
+  } else {
+    const [head, ...rest] = lines(false);
+    console.log(c.bold(head!));
+    for (const row of rest) {
+      console.log(row === '' ? '' : `  ${wrap(row, 74, '    ')}`);
+    }
+  }
+
+  if (gate) {
+    if (verification.gateFailures > 0) {
+      console.error(c.red(t.verify.gateFailed(n(verification.gateFailures), n(verification.actions.length))));
+      process.exitCode = 1;
+    } else {
+      console.log(c.green(t.verify.gateOk()));
+    }
+  }
+}
+
+/**
+ * `trazum plan <log>` — not a list of findings, a ranked plan of what to do.
+ *
+ * The composition (route and batch on one slice never summed) happens in
+ * core's `buildPlan`; this command owns the I/O and the rendering. The plan
+ * saves as a dated JSON file on request, which is what makes verifying it
+ * against a later log possible at all — a prediction nobody wrote down is a
+ * prediction nobody can be held to.
+ */
+async function commandPlan(
+  args: Args,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
+  const path = args.positional[0];
+  if (path === undefined) throw new Error(t.plan.noTarget());
+
+  const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
+  const READABLE = [...LOG_EXTENSIONS, ...GZ];
+  const target = await stat(path).catch(() => null);
+  let files: string[] = [path];
+  if (target?.isDirectory()) {
+    const entries = await readdir(path, { withFileTypes: true });
+    files = entries
+      .filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
+      .map((entry) => join(path, entry.name))
+      .sort((a, b) => a.localeCompare(b));
+    if (files.length === 0) throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
+  }
+  const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
+  const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
+
+  const report = profileUsage(raw, { catalogue: pricing });
+  if (report.total.calls === 0) throw new Error(t.plan.nothingPriced());
+  const levers = billLevers(report, { catalogue: pricing });
+  const plan = buildPlan(report, levers, pricing.lastReviewed);
+
+  const minUsd = typeof args.flags.get('min-usd') === 'string' ? numberFlag(args, 'min-usd', 0, t) : 0;
+  const actions = plan.actions.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) >= minUsd);
+  const filtered = plan.actions.length - actions.length;
+  const droppedUsd = plan.actions
+    .filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) < minUsd)
+    .reduce((sum, a) => sum + (a.savingUsd ?? a.stakeUsd ?? 0), 0);
+
+  const n = (value: number): string => value.toLocaleString(t.numberLocale);
+  /**
+   * The document's totals cover the actions the document holds — a filtered
+   * plan whose totals still counted the filtered actions would be a file
+   * that contradicts itself, and 1.39's verify would hold it to money it
+   * cannot see. What --min-usd dropped is stated with its worth, never
+   * silently.
+   */
+  const stamped = {
+    ...plan,
+    actions,
+    projectedSavingUsd: actions.reduce((sum, a) => sum + (a.savingUsd ?? 0), 0),
+    measuredStakeUsd: actions.reduce((sum, a) => sum + (a.stakeUsd ?? 0), 0),
+    createdAt: new Date().toISOString(),
+  };
+
+  const outPath = stringFlag(args, 'out');
+  if (outPath !== undefined) {
+    await writeFile(outPath, `${JSON.stringify(stamped, null, 2)}\n`);
+  }
+
+  await writeMarkdown(args, () => {
+    const lines: string[] = [];
+    lines.push(`## ${t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))}`);
+    lines.push('');
+    lines.push(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)));
+    if (plan.span === null) {
+      lines.push('');
+      lines.push(`_${t.plan.noClock()}_`);
+    }
+    for (const action of actions) {
+      const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
+      const money =
+        action.savingUsd !== null
+          ? t.plan.projected(formatUsd(action.savingUsd))
+          : t.plan.staked(formatUsd(action.stakeUsd ?? 0));
+      lines.push('');
+      lines.push(`### ${t.plan.action(action.kind, name, action.model)} — ${money}`);
+      if (action.detail.routeTo !== undefined) lines.push(`- ${t.plan.routeTo(action.detail.routeTo.displayName)}`);
+      for (const assumption of action.assumes) lines.push(`- ${t.plan.assume(assumption)}`);
+      if (action.check !== null) lines.push(`- ${t.plan.check(action.check)}`);
+    }
+    if (filtered > 0) {
+      lines.push('');
+      lines.push(`_${t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd))}_`);
+    }
+    lines.push('');
+    lines.push(`_${t.plan.footer()}_`);
+    return lines.join('\n');
+  });
+
+  if (boolFlag(args, 'json')) {
+    console.log(JSON.stringify(stamped, null, 2));
+    return;
+  }
+
+  console.log(c.bold(t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))));
+  console.log(
+    `  ${wrap(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)), 74, '  ')}`,
+  );
+  if (plan.span === null) {
+    console.log(`  ${c.dim(wrap(t.plan.noClock(), 74, '  '))}`);
+  }
+  for (const action of actions) {
+    const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
+    const money =
+      action.savingUsd !== null
+        ? t.plan.projected(formatUsd(action.savingUsd))
+        : t.plan.staked(formatUsd(action.stakeUsd ?? 0));
+    console.log();
+    console.log(`  ${c.green('→')} ${c.bold(t.plan.action(action.kind, name, action.model))}  ${money}`);
+    if (action.detail.routeTo !== undefined) {
+      console.log(`    ${c.dim(t.plan.routeTo(action.detail.routeTo.displayName))}`);
+    }
+    for (const assumption of action.assumes) {
+      console.log(`    ${c.yellow('?')} ${c.dim(wrap(t.plan.assume(assumption), 72, '      '))}`);
+    }
+    if (action.check !== null) {
+      console.log(`    ${c.dim(wrap(t.plan.check(action.check), 72, '      '))}`);
+    }
+  }
+  if (filtered > 0) {
+    console.log();
+    console.log(`  ${c.dim(wrap(t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd)), 74, '  '))}`);
+  }
+  console.log();
+  console.log(`  ${c.dim(wrap(t.plan.footer(), 74, '  '))}`);
+  if (outPath !== undefined) console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
+}
+
 async function commandProfile(args: Args, config: TrazumConfig, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
   const path = args.positional[0];
   if (path === undefined) {
@@ -2227,7 +2509,6 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
    * directory holding nothing readable is an error naming what it looked for,
    * not an empty report.
    */
-  const LOG_EXTENSIONS = ['.jsonl', '.ndjson', '.log', '.json'];
   /**
    * The same names, gzipped — which is what a rotated log actually looks like
    * a day after it rotates.
@@ -6015,6 +6296,12 @@ async function main(): Promise<void> {
     case 'profile':
       await commandProfile(args, config, pricing, t);
       break;
+    case 'plan':
+      await commandPlan(args, pricing, t);
+      break;
+    case 'verify':
+      await commandVerify(args, pricing, t);
+      break;
     case 'route':
       await commandRoute(args, pricing, t);
       break;