@trazum/cli 1.38.0 → 1.39.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trazum/cli",
-  "version": "1.38.0",
+  "version": "1.39.0",
   "description": "Trazum CLI: find where your LLM bill goes, price every finding per month, and enforce token budgets in CI.",
   "license": "MIT",
   "author": "David Mu\u00f1oz Rey",
@@ -37,7 +37,7 @@
     "prepublishOnly": "npm run build && npm test"
   },
   "dependencies": {
-    "@trazum/core": "1.38.0"
+    "@trazum/core": "1.39.0"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",

package/src/i18n/en.ts

@@ -40,6 +40,7 @@ ${bold('USAGE')}
   trazum eval <file> --cases <file> --export promptfoo -o suite.json
   trazum route <log.jsonl> --prompt-file <file> --cases <file> --yes
   trazum plan <log.jsonl|dir> [options]
+  trazum verify <plan.json> --against <newer.jsonl|dir> [options]
   trazum diff <before> <after> [options]
   trazum diff --all <dir> <dir> [options]
   trazum rank <dir> [options]
@@ -317,6 +318,25 @@ ${bold('OPTIONS FOR plan')}
   a plan that hides its assumptions is advice pretending to be arithmetic.
   Projected savings and money already spent are separate totals throughout.
 
+${bold('OPTIONS FOR verify')}
+  --against <log|dir>         The newer usage log the plan is held to. Required.
+  --gate                      Exit 1 when an action did not produce what the
+                              plan promised, or its fields stopped being
+                              recorded — "not recorded" must not read as
+                              "fixed". A vanished workload fails nothing.
+  --markdown-out <file>       Also write the verdicts as Markdown, for a CI
+                              job summary or a pull request comment.
+  --pricing <file>            Local price overlay, as everywhere else.
+  --json                      The verification as data.
+
+  Holds a saved plan to the log that came after it, with three outcomes and
+  never two: the change arrived, it did not, or it cannot be told — because
+  the workload vanished, the fields the detection needs stopped being
+  recorded, or tokens cannot say which tier billed them. Differences carry
+  the world's measured movement (calls, output per call) from the plan's own
+  baseline, and a plan priced under a different catalogue says so rather
+  than blaming a team for a saving that arithmetic revoked.
+
 ${bold('OPTIONS FOR route')}
   --prompt-file <file>        The prompt those calls send. Not --prompt, which
                               names a marked prompt inside a source file.
@@ -1412,6 +1432,61 @@ ${bold('EXAMPLES')}
       `Plan written to ${path}, dated. Keep it: a prediction nobody wrote down is a prediction nobody can be held to.`,
   },
 
+  verify: {
+    noTarget: () =>
+      'Point this at a saved plan and a newer log: trazum verify plan.json --against usage.jsonl. It says, per action, whether the change arrived, did not arrive, or cannot be told — and never fewer than those three.',
+    needsAgainst: () =>
+      '--against <newer.jsonl|dir> is required. A plan can only be verified against a log that came after it; without one there is nothing to hold the prediction to.',
+    badPlan: (path) =>
+      `${path} is not a plan document this tool can verify — expected the JSON that "trazum plan -o" writes (schemaVersion 1, with an actions array).`,
+    heading: (actions, planDate) =>
+      planDate === null
+        ? `Did it work? ${actions} actions from an undated plan, against this log`
+        : `Did it work? ${actions} actions from the plan of ${planDate}, against this log`,
+    counts: (arrived, notArrived, cannotTell) =>
+      `${arrived} arrived · ${notArrived} did not arrive · ${cannotTell} cannot be told. The third is not a soft version of the second: it means this log cannot answer, which is its own finding.`,
+    pricesChanged: (planReviewed, nowReviewed) =>
+      `Prices were reviewed ${planReviewed} when the plan was made and ${nowReviewed} now, so every dollar comparison here is two price lists, not one measurement — a team must not be blamed for a saving that arithmetic revoked.`,
+    action: (kind, label, model, outcome) => {
+      const what =
+        kind === 'route'
+          ? `Route ${label} (${model})`
+          : kind === 'batch'
+            ? `Batch ${label} (${model})`
+            : kind === 'route+batch'
+              ? `Route and batch ${label} (${model})`
+              : kind === 'fix-truncation'
+                ? `Fix the truncation retries on ${label} (${model})`
+                : `Fix the cache on ${label} (${model})`;
+      const verdict =
+        outcome === 'arrived' ? 'ARRIVED' : outcome === 'not-arrived' ? 'DID NOT ARRIVE' : 'CANNOT BE TOLD';
+      return `${what} — ${verdict}`;
+    },
+    reason: (reason) =>
+      reason === 'workload-vanished'
+        ? 'the label carries no priced traffic in this log — a vanished workload is not a fixed one, and not a broken one either'
+        : reason === 'fields-stopped'
+          ? 'the fields the detection needs are not in this log — "not recorded" must not read as "fixed", so with --gate this fails'
+          : 'the log records tokens, and tokens do not say which tier billed them — the Batch API cannot be seen from here',
+    routeObserved: (dearestModel, onTargetUsd, onOldUsd) =>
+      `the label's dearest model is now ${dearestModel} · ${onTargetUsd} on the target, ${onOldUsd} still on the old model`,
+    batchUnobservable: () =>
+      'the batch half of this action cannot be seen in token counts; the verdict above is the route half alone',
+    truncationObserved: (retryBillUsd) =>
+      `this log still shows ${retryBillUsd} of truncation waste and retries`,
+    cacheObserved: (deltaUsd, outcome) =>
+      outcome === 'arrived'
+        ? `caching now pays for itself on this slice (${deltaUsd} against the no-cache bill)`
+        : `caching still adds ${deltaUsd} to this slice's bill`,
+    attribution: (callsBefore, callsAfter, outBefore, outAfter) =>
+      `the world moved too: calls ${callsBefore} → ${callsAfter}, output/call ${outBefore} → ${outAfter} tokens — stated so the verdict is not read as the whole story`,
+    gateFailed: (failures, total) =>
+      `GATE FAILED — ${failures} of ${total} actions did not produce what the plan promised, or stopped being measurable by the team's own log.`,
+    gateOk: () => 'Gate passed: every verifiable action arrived, and nothing became unverifiable.',
+    footer: () =>
+      'Arrived and did-not-arrive are measurements; cannot-be-told is the log refusing to guess. All three are the verification working, not failing.',
+  },
+
   route: {
     noTarget: () =>
       'Point this at a usage log and a prompt: trazum route usage.jsonl --prompt-file prompts/support.txt --cases cases.txt --yes. It finds the slice worth the most, then measures whether the cheaper model still does the job. The flag is --prompt-file and not --prompt, because --prompt names a marked prompt inside a source file everywhere else in this tool.',

package/src/i18n/es.ts

@@ -27,6 +27,7 @@ ${bold('USO')}
   trazum eval <fichero> --cases <fichero> --export promptfoo -o suite.json
   trazum route <log.jsonl> --prompt-file <fichero> --cases <fichero> --yes
   trazum plan <log.jsonl|dir> [opciones]
+  trazum verify <plan.json> --against <nuevo.jsonl|dir> [opciones]
   trazum diff <antes> <después> [opciones]
   trazum diff --all <dir> <dir> [opciones]
   trazum rank <dir> [opciones]
@@ -324,6 +325,27 @@ ${bold('OPCIONES DE plan')}
   consejo haciéndose pasar por aritmética. El ahorro proyectado y el dinero ya
   gastado son totales separados en todas partes.
 
+${bold('OPCIONES DE verify')}
+  --against <log|dir>         El registro de uso posterior al que se somete el
+                              plan. Obligatorio.
+  --gate                      Sale con 1 cuando una acción no produjo lo que el
+                              plan prometió, o sus campos dejaron de
+                              registrarse — "no registrado" no puede leerse
+                              como "arreglado". Una carga de trabajo
+                              desaparecida no suspende nada.
+  --markdown-out <fichero>    Escribe además los veredictos como Markdown, para
+                              un resumen de CI o un comentario de pull request.
+  --pricing <fichero>         Tarifas locales superpuestas, como en el resto.
+  --json                      La verificación como datos.
+
+  Somete un plan guardado al registro que vino después, con tres resultados y
+  nunca dos: el cambio llegó, no llegó, o no se puede saber — porque la carga
+  desapareció, los campos que la detección necesita dejaron de registrarse, o
+  los tokens no dicen con qué tarifa se facturaron. Las diferencias llevan el
+  movimiento medido del mundo (llamadas, salida por llamada) desde la línea
+  base del propio plan, y un plan tasado con otro catálogo lo dice en vez de
+  culpar a un equipo por un ahorro que la aritmética revocó.
+
 ${bold('OPCIONES DE route')}
   --prompt-file <fichero>     El prompt que mandan esas llamadas. No --prompt,
                               que nombra un prompt marcado dentro de un fuente.
@@ -1432,6 +1454,61 @@ ${bold('EJEMPLOS')}
       `Plan escrito en ${path}, con fecha. Guárdalo: una predicción que nadie apuntó es una predicción que no se le puede exigir a nadie.`,
   },
 
+  verify: {
+    noTarget: () =>
+      'Apunta esto a un plan guardado y a un registro posterior: trazum verify plan.json --against usage.jsonl. Dice, por acción, si el cambio llegó, no llegó o no se puede saber — y nunca menos de esos tres.',
+    needsAgainst: () =>
+      '--against <nuevo.jsonl|dir> es obligatorio. Un plan solo puede verificarse contra un registro posterior; sin uno no hay nada a lo que someter la predicción.',
+    badPlan: (path) =>
+      `${path} no es un documento de plan que esta herramienta pueda verificar — se esperaba el JSON que escribe "trazum plan -o" (schemaVersion 1, con un array actions).`,
+    heading: (actions, planDate) =>
+      planDate === null
+        ? `¿Funcionó? ${actions} acciones de un plan sin fecha, contra este registro`
+        : `¿Funcionó? ${actions} acciones del plan del ${planDate}, contra este registro`,
+    counts: (arrived, notArrived, cannotTell) =>
+      `${arrived} llegaron · ${notArrived} no llegaron · ${cannotTell} no se puede saber. Lo tercero no es una versión suave de lo segundo: significa que este registro no puede responder, y eso es un hallazgo en sí.`,
+    pricesChanged: (planReviewed, nowReviewed) =>
+      `Las tarifas se revisaron el ${planReviewed} cuando se hizo el plan y el ${nowReviewed} ahora, así que cada comparación en dólares aquí son dos listas de precios, no una medición — no se puede culpar a un equipo por un ahorro que la aritmética revocó.`,
+    action: (kind, label, model, outcome) => {
+      const what =
+        kind === 'route'
+          ? `Enrutar ${label} (${model})`
+          : kind === 'batch'
+            ? `Agrupar en batch ${label} (${model})`
+            : kind === 'route+batch'
+              ? `Enrutar y agrupar ${label} (${model})`
+              : kind === 'fix-truncation'
+                ? `Arreglar los reintentos por truncado de ${label} (${model})`
+                : `Arreglar la caché de ${label} (${model})`;
+      const verdict =
+        outcome === 'arrived' ? 'LLEGÓ' : outcome === 'not-arrived' ? 'NO LLEGÓ' : 'NO SE PUEDE SABER';
+      return `${what} — ${verdict}`;
+    },
+    reason: (reason) =>
+      reason === 'workload-vanished'
+        ? 'el label no lleva tráfico tasado en este registro — una carga desaparecida no está arreglada, y tampoco rota'
+        : reason === 'fields-stopped'
+          ? 'los campos que la detección necesita no están en este registro — "no registrado" no puede leerse como "arreglado", así que con --gate esto suspende'
+          : 'el registro guarda tokens, y los tokens no dicen con qué tarifa se facturaron — el Batch API no se puede ver desde aquí',
+    routeObserved: (dearestModel, onTargetUsd, onOldUsd) =>
+      `el modelo más caro del label ahora es ${dearestModel} · ${onTargetUsd} en el destino, ${onOldUsd} todavía en el modelo antiguo`,
+    batchUnobservable: () =>
+      'la mitad batch de esta acción no se puede ver en los recuentos de tokens; el veredicto de arriba es solo la mitad de la ruta',
+    truncationObserved: (retryBillUsd) =>
+      `este registro todavía muestra ${retryBillUsd} de desperdicio y reintentos por truncado`,
+    cacheObserved: (deltaUsd, outcome) =>
+      outcome === 'arrived'
+        ? `la caché ahora se paga sola en esta porción (${deltaUsd} contra la factura sin caché)`
+        : `la caché todavía añade ${deltaUsd} a la factura de esta porción`,
+    attribution: (callsBefore, callsAfter, outBefore, outAfter) =>
+      `el mundo también se movió: llamadas ${callsBefore} → ${callsAfter}, salida/llamada ${outBefore} → ${outAfter} tokens — dicho para que el veredicto no se lea como toda la historia`,
+    gateFailed: (failures, total) =>
+      `GATE SUSPENDIDO — ${failures} de ${total} acciones no produjeron lo que el plan prometió, o dejaron de poder medirse por el propio registro del equipo.`,
+    gateOk: () => 'Gate superado: toda acción verificable llegó, y nada se volvió inverificable.',
+    footer: () =>
+      'Llegó y no-llegó son mediciones; no-se-puede-saber es el registro negándose a adivinar. Los tres son la verificación funcionando, no fallando.',
+  },
+
   route: {
     noTarget: () =>
       'Apunta esto a un registro de uso y a un prompt: trazum route usage.jsonl --prompt-file prompts/soporte.txt --cases casos.txt --yes. Busca la porción que más vale y mide si el modelo más barato sigue haciendo el trabajo. El flag es --prompt-file y no --prompt, porque en el resto de la herramienta --prompt nombra un prompt marcado dentro de un fichero fuente.',

package/src/i18n/types.ts

@@ -1,4 +1,4 @@
-import type { EvalVerdict, Locale, PlanActionKind, PlanAssumption, RuleLevel } from '@trazum/core';
+import type { CannotTellReason, EvalVerdict, Locale, PlanActionKind, PlanAssumption, RuleLevel, VerifyOutcome } from '@trazum/core';
 
 /**
  * The CLI's own message catalogue.
@@ -1072,6 +1072,38 @@ export interface CliMessages {
     wrote(path: string): string;
   };
 
+  /**
+   * `trazum verify` — the plan held to the log that came after it.
+   *
+   * Three outcomes and never two: arrived, did not arrive, cannot be told.
+   * The third is the honest one, and the one every other tool renders as
+   * the first. Differences carry the world's measured movement, and a plan
+   * priced under another catalogue says so rather than blaming a team for
+   * a saving that arithmetic revoked.
+   */
+  verify: {
+    noTarget(): string;
+    needsAgainst(): string;
+    /** Not a plan document: wrong shape, wrong version, or not JSON at all. */
+    badPlan(path: string): string;
+    heading(actions: string, planDate: string | null): string;
+    counts(arrived: string, notArrived: string, cannotTell: string): string;
+    /** Two price lists are two measurements; every dollar line inherits this. */
+    pricesChanged(planReviewed: string, nowReviewed: string): string;
+    action(kind: PlanActionKind, label: string, model: string, outcome: VerifyOutcome): string;
+    reason(reason: CannotTellReason): string;
+    routeObserved(dearestModel: string, onTargetUsd: string, onOldUsd: string): string;
+    /** Tokens do not say which tier billed them; named, never assumed arrived. */
+    batchUnobservable(): string;
+    truncationObserved(retryBillUsd: string): string;
+    cacheObserved(deltaUsd: string, outcome: VerifyOutcome): string;
+    /** The world's movement, measured: never a verdict, always both numbers. */
+    attribution(callsBefore: string, callsAfter: string, outBefore: string, outAfter: string): string;
+    gateFailed(failures: string, total: string): string;
+    gateOk(): string;
+    footer(): string;
+  };
+
   /**
    * `trazum route` — the loop the levers section could only point at.
    *

package/src/index.ts

@@ -12,6 +12,7 @@ import {
   analyzeCachePrefix,
   billLevers,
   buildPlan,
+  verifyPlan,
   cacheEconomics,
   cacheHitRate,
   contextPressure,
@@ -79,6 +80,8 @@ import { dayOf, formatGap, median, spanDays } from './time.js';
 import type {
   FleetSource,
   MeasuredUsage,
+  PlanDocument,
+  VerifiedAction,
   BaselineBreach,
   BaselineChange,
   BaselineComparison,
@@ -475,6 +478,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
   baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
   profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
   plan: ['json', 'out', 'markdown-out', 'min-usd', 'pricing', 'pricing-live'],
+  verify: ['against', 'gate', 'json', 'markdown-out', 'pricing', 'pricing-live'],
   route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
   eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
   prune: ['cases', 'concurrency', 'json', 'yes'],
@@ -2214,6 +2218,140 @@ function isoDate(): string {
  * metered API calls somebody was actually billed for — the bill exists wherever
  * Trazum happens to be running, so the host has no bearing on it.
  */
+/**
+ * `trazum verify <plan.json> --against <newer.jsonl|dir>` — did it work?
+ *
+ * The plan predicted; this holds the prediction to the log that came after
+ * it. Three outcomes and never two — arrived, did not arrive, cannot be told
+ * — because "cannot be told" rendered as "arrived" is how every other tool
+ * congratulates a team for a workload that merely vanished. With `--gate`,
+ * a broken promise is a failing exit code: a different and more useful gate
+ * than "spend went up".
+ */
+async function commandVerify(
+  args: Args,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
+  const planPath = args.positional[0];
+  if (planPath === undefined) throw new Error(t.verify.noTarget());
+  const againstPath = stringFlag(args, 'against');
+  if (againstPath === undefined) throw new Error(t.verify.needsAgainst());
+
+  let plan: PlanDocument & { createdAt?: string };
+  try {
+    const parsed = JSON.parse(await readFile(planPath, 'utf8'));
+    if (parsed?.schemaVersion !== 1 || !Array.isArray(parsed.actions)) {
+      throw new Error(t.verify.badPlan(planPath));
+    }
+    plan = parsed;
+  } catch (error) {
+    if (error instanceof SyntaxError) throw new Error(t.verify.badPlan(planPath));
+    throw error;
+  }
+
+  const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
+  const READABLE = [...LOG_EXTENSIONS, ...GZ];
+  const target = await stat(againstPath).catch(() => null);
+  let files: string[] = [againstPath];
+  if (target?.isDirectory()) {
+    const entries = await readdir(againstPath, { withFileTypes: true });
+    files = entries
+      .filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
+      .map((entry) => join(againstPath, entry.name))
+      .sort((a, b) => a.localeCompare(b));
+    if (files.length === 0) throw new Error(t.profile.noLogsInDirectory(againstPath, READABLE.join(', ')));
+  }
+  const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
+  const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
+  const report = profileUsage(raw, { catalogue: pricing });
+
+  const verification = verifyPlan(plan, report, { currentPricingLastReviewed: pricing.lastReviewed });
+  const gate = boolFlag(args, 'gate');
+  const n = (value: number): string => value.toLocaleString(t.numberLocale);
+
+  const lines = (md: boolean): string[] => {
+    const out: string[] = [];
+    const actionLine = (v: VerifiedAction): string[] => {
+      const name = v.action.label === UNLABELLED ? t.profile.unlabelled() : v.action.label;
+      const rows: string[] = [];
+      rows.push(t.verify.action(v.action.kind, name, v.action.model, v.outcome));
+      if (v.outcome === 'cannot-tell' && v.reason !== null) rows.push(t.verify.reason(v.reason));
+      if (v.action.kind === 'route' || v.action.kind === 'route+batch') {
+        if (v.outcome !== 'cannot-tell') {
+          rows.push(
+            t.verify.routeObserved(
+              String(v.observed.dearestModel ?? ''),
+              formatUsd(Number(v.observed.onTargetUsd ?? 0)),
+              formatUsd(Number(v.observed.onOldModelUsd ?? 0)),
+            ),
+          );
+        }
+        if (v.action.kind === 'route+batch' && v.outcome !== 'cannot-tell') rows.push(t.verify.batchUnobservable());
+      }
+      if (v.action.kind === 'fix-truncation' && v.outcome === 'not-arrived') {
+        rows.push(t.verify.truncationObserved(formatUsd(Number(v.observed.retryBillUsd ?? 0))));
+      }
+      if (v.action.kind === 'fix-caching' && v.outcome !== 'cannot-tell') {
+        rows.push(t.verify.cacheObserved(formatUsd(Number(v.observed.deltaUsd ?? 0)), v.outcome));
+      }
+      if (v.attribution?.calls !== undefined) {
+        rows.push(
+          t.verify.attribution(
+            n(Math.round(v.attribution.calls.before)),
+            n(Math.round(v.attribution.calls.after)),
+            n(Math.round(v.attribution.outputPerCallTokens?.before ?? 0)),
+            n(Math.round(v.attribution.outputPerCallTokens?.after ?? 0)),
+          ),
+        );
+      }
+      return rows;
+    };
+
+    const heading = t.verify.heading(
+      n(verification.actions.length),
+      verification.planCreatedAt === null ? null : verification.planCreatedAt.slice(0, 10),
+    );
+    out.push(md ? `## ${heading}` : heading);
+    out.push(
+      t.verify.counts(n(verification.arrived), n(verification.notArrived), n(verification.cannotTell)),
+    );
+    if (verification.pricesChanged) {
+      out.push(t.verify.pricesChanged(verification.planPricing, verification.currentPricing));
+    }
+    for (const v of verification.actions) {
+      out.push('');
+      const [head, ...rest] = actionLine(v);
+      out.push(md ? `### ${head}` : `→ ${head}`);
+      for (const row of rest) out.push(md ? `- ${row}` : `  · ${row}`);
+    }
+    out.push('');
+    out.push(t.verify.footer());
+    return out;
+  };
+
+  await writeMarkdown(args, () => lines(true).join('\n'));
+
+  if (boolFlag(args, 'json')) {
+    console.log(JSON.stringify(verification, null, 2));
+  } else {
+    const [head, ...rest] = lines(false);
+    console.log(c.bold(head!));
+    for (const row of rest) {
+      console.log(row === '' ? '' : `  ${wrap(row, 74, '    ')}`);
+    }
+  }
+
+  if (gate) {
+    if (verification.gateFailures > 0) {
+      console.error(c.red(t.verify.gateFailed(n(verification.gateFailures), n(verification.actions.length))));
+      process.exitCode = 1;
+    } else {
+      console.log(c.green(t.verify.gateOk()));
+    }
+  }
+}
+
 /**
  * `trazum plan <log>` — not a list of findings, a ranked plan of what to do.
  *
@@ -6161,6 +6299,9 @@ async function main(): Promise<void> {
     case 'plan':
       await commandPlan(args, pricing, t);
       break;
+    case 'verify':
+      await commandVerify(args, pricing, t);
+      break;
     case 'route':
       await commandRoute(args, pricing, t);
       break;