@trazum/cli 1.50.6 → 1.50.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trazum/cli",
-  "version": "1.50.6",
+  "version": "1.50.8",
   "description": "Trazum CLI: find where your LLM bill goes, price every finding per month, and enforce token budgets in CI.",
   "license": "MIT",
   "author": "David Mu\u00f1oz Rey",
@@ -37,7 +37,7 @@
     "prepublishOnly": "npm run build && npm test"
   },
   "dependencies": {
-    "@trazum/core": "1.50.6"
+    "@trazum/core": "1.50.8"
   },
   "devDependencies": {
     "@types/node": "^26.2.0",

package/src/i18n/en.ts

@@ -58,6 +58,8 @@ ${bold('USAGE')}
   trazum models
   trazum rules
   trazum gateway <anthropic|openai> --on-cannot-tell <fail-open|fail-closed>
+  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
+  trazum quality <log> --label <name> --at <iso> [--gate]
   trazum ladder <log>
   trazum feedback
   trazum --version
@@ -119,6 +121,32 @@ ${bold('OPTIONS FOR gateway')}
 
   Your credential is forwarded untouched and never read. See docs/gateway.md.
 
+${bold('OPTIONS FOR experiment')}
+  --a <label>, --b <label>    The two workloads to compare.
+  --min-outcomes <n>          Required: how many outcomes each arm must record
+                              before the result may be read. A stopping rule
+                              declared after looking at the numbers is not a
+                              stopping rule, and the report says whether it was
+                              honoured.
+
+  Judges recorded outcomes and cost together. Three-valued: A wins, B wins, or
+  not separable — with the number of outcomes per arm that would settle it, so
+  "run it longer" is an instruction rather than a shrug. Nothing is promoted.
+
+${bold('OPTIONS FOR quality')}
+  --label <name>              The workload to judge. Required: a mixture would
+                              average a regression away.
+  --at <iso>                  When the change landed. Required: without a
+                              boundary there is nothing to compare across, and
+                              picking one would be this tool choosing which
+                              change to blame.
+  --gate                      Exit 1 on a measured drop, 2 on "cannot tell".
+                              Three outcomes, never two.
+
+  A before-and-after rather than an experiment, so it reports "cannot tell"
+  whenever the model mix, the call volume or the outcome coverage moved across
+  the boundary — the prompt is not the only variable and it says so.
+
 ${bold('OPTIONS FOR prune')}
   --cases <file>              One input per line, or a JSON array. Required.
   --yes                       Actually spend the calls. Without it the estimate is
@@ -964,6 +992,69 @@ ${bold('EXAMPLES')}
       `${path} exists and could not be parsed, so nothing was written over it. Fix or move it first.`,
   },
 
+  quality: {
+    heading: (label) => `Quality across the change: ${label}`,
+    needsLabel: () => 'Name the workload with --label: this compares one label before and after a change, and a mixture of workloads would average a regression away.',
+    needsAt: () =>
+      '--at is required: give the moment the change landed, as an ISO timestamp. Without it there is no boundary to compare across, and picking one from the log would be this tool choosing which change to blame.',
+    sides: (beforeRate, afterRate, before, after) =>
+      `before ${beforeRate} (${before} outcomes)   after ${afterRate} (${after} outcomes)`,
+    dropped: (from, to, outcomes, cost) =>
+      `The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes, and this change ${cost}. Both halves are measured; neither is an estimate.`,
+    held: (from, to, outcomes) =>
+      `The resolution rate moved from ${from} to ${to} on ${outcomes} measured outcomes \u2014 up, measurably.`,
+    cannotTell: (why, need) =>
+      why === 'too-few-before'
+        ? `Cannot tell: only ${need} outcomes before the change, and this gate needs 100 a side. It fails builds, so the threshold is not the one a rate uses elsewhere.`
+        : why === 'too-few-after'
+          ? `Cannot tell yet: only ${need} outcomes since the change, and this gate needs 100 a side. Run it again once the traffic has caught up.`
+          : why === 'not-separable'
+            ? 'Cannot tell: the rate did not measurably move. That is NOT the same as "it held" \u2014 a gate that spelled them the same way would pass a real regression it merely lacked the power to see.'
+            : why === 'no-vocabulary'
+              ? 'Cannot tell: "outcomes.success" declares nothing, so there is no resolution rate to compare.'
+              : 'Cannot tell: something other than the prompt moved across the boundary. See below.',
+    confoundersHeading: () => 'The prompt is not the only thing that changed',
+    confounder: (kind, detail) =>
+      kind === 'model-mix-moved'
+        ? `The model mix moved by ${detail}. The drop may be entirely somebody else's migration, and this tool cannot separate the two.`
+        : kind === 'volume-moved'
+          ? `The call volume moved ${detail}. A workload whose traffic moved that much is usually a workload whose population changed \u2014 a new surface, a new customer, a campaign \u2014 and the questions being asked are not the questions from before.`
+          : `Outcome coverage moved from ${detail}. The two rates describe different populations: a team that starts instrumenting its hard cases sees its measured rate fall without anything having got worse.`,
+    notRandomised: () =>
+      'This is a before-and-after, not an experiment. It splits traffic by time rather than at random, so everything else that changed at the same time is in the difference too \u2014 which is why it says "cannot tell" far more readily than an A/B would.',
+    cannotSee: () =>
+      'It cannot see anything else you deployed that day. A "dropped" verdict says the rate fell and the three things it can check did not move. That is a smaller claim than "the prompt did it", and it is the largest one the evidence supports.',
+    gateFailed: () => 'Gate failed: a measured drop with nothing else to explain it.',
+    gateHeldOpen: () =>
+      'Gate not passed and not failed. "Cannot tell" holds the claim open rather than exiting green \u2014 the posture verify has had since 1.39.',
+  },
+
+  experiment: {
+    heading: (a, b) => `Experiment: ${a} against ${b}`,
+    needsTwo: () =>
+      'Name two labels to compare, for example: trazum experiment <log> --a prompt-v1 --b prompt-v2',
+    needsRule: () =>
+      '--min-outcomes is required, and it is the point: a stopping rule declared after looking at the numbers is not a stopping rule. Say how many outcomes each arm must record before the result may be read.',
+    arm: (name, rate, successes, recorded, interval) =>
+      `${name}  ${rate}  (${successes} of ${recorded} recorded)  95% ${interval}`,
+    wins: (name, low, high) =>
+      `${name} wins. The difference is between ${low} and ${high} at 95% confidence \u2014 the whole interval is on one side of zero, which is what "wins" means here.`,
+    notSeparable: (why, needed) =>
+      why === 'no-difference-observed'
+        ? 'Not separable: both arms recorded the same rate. No sample size separates a difference of zero, so there is no "run it longer" to offer \u2014 there is nothing here to find.'
+        : why === 'nothing-recorded'
+          ? 'Not separable: an arm recorded no outcomes at all, so there is no rate to compare.'
+          : `Not separable on this traffic: the 95% interval on the difference includes zero. One number is larger, and that is not a finding. About ${needed} outcomes per arm would settle the difference observed so far.`,
+    peeked: (short, declared, recorded) =>
+      `Read early. The declared rule was ${declared} outcomes per arm and ${short} has ${recorded}. Nothing can stop a number being read early; this line exists so whoever reads the result later can see that it was.`,
+    honoured: (declared) => `Stopping rule honoured: both arms cleared ${declared} recorded outcomes.`,
+    marginalDearer: (better, usd) =>
+      `${better} resolves more and costs more. One extra success costs ${usd} \u2014 that figure, not the rate, is what the decision turns on.`,
+    marginalCheaper: (better) => `${better} resolves more AND costs less per call. Nothing is being traded.`,
+    neverPromotes: () =>
+      'Nothing was changed. A winner is a finding; taking it is a decision with a name attached, and it belongs in the plan like everything else.',
+  },
+
   ladder: {
     heading: () => 'Escalation ladders',
     noLadders: () =>

package/src/i18n/es.ts

@@ -45,6 +45,8 @@ ${bold('USO')}
   trazum models
   trazum rules
   trazum gateway <anthropic|openai> --on-cannot-tell <fail-open|fail-closed>
+  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
+  trazum quality <log> --label <name> --at <iso> [--gate]
   trazum ladder <log>
   trazum feedback
   trazum --version
@@ -109,6 +111,32 @@ ${bold('OPCIONES DE gateway')}
 
   Tu credencial se reenvía intacta y nunca se lee. Ver docs/gateway.md.
 
+${bold('OPCIONES DE experiment')}
+  --a <etiqueta>, --b <etiqueta>  Las dos cargas a comparar.
+  --min-outcomes <n>          Obligatorio: cuántos resultados debe registrar
+                              cada brazo antes de poder leer el resultado. Una
+                              regla de parada declarada después de mirar los
+                              números no es una regla de parada, y el informe
+                              dice si se respetó.
+
+  Juzga resultados registrados y coste a la vez. Tres valores: gana A, gana B,
+  o no separables — con cuántos resultados por brazo lo zanjarían, para que
+  "déjalo correr más" sea una instrucción y no un encogimiento de hombros.
+
+${bold('OPCIONES DE quality')}
+  --label <nombre>            La carga a juzgar. Obligatorio: una mezcla
+                              promediaría una regresión hasta hacerla
+                              desaparecer.
+  --at <iso>                  Cuándo aterrizó el cambio. Obligatorio: sin
+                              frontera no hay nada que comparar, y elegir una
+                              sería que esta herramienta decida a qué culpar.
+  --gate                      Sale con 1 si hay caída medida, 2 si no se puede
+                              decir. Tres resultados, nunca dos.
+
+  Un antes y después, no un experimento, así que dice "no se puede decir"
+  siempre que la mezcla de modelos, el volumen o la cobertura se movieran a
+  través de la frontera — el prompt no es la única variable y lo dice.
+
 ${bold('OPCIONES DE prune')}
   --cases <fichero>           Una entrada por línea, o un array JSON. Obligatorio.
   --yes                       Gasta las llamadas de verdad. Sin él se imprime la
@@ -1000,6 +1028,69 @@ ${bold('EJEMPLOS')}
       `${path} existe y no se pudo interpretar, así que no se escribió nada encima. Arréglalo o muévelo primero.`,
   },
 
+  quality: {
+    heading: (label) => `Calidad a trav\u00e9s del cambio: ${label}`,
+    needsLabel: () => 'Nombra la carga con --label: esto compara una etiqueta antes y despu\u00e9s de un cambio, y una mezcla de cargas promediar\u00eda una regresi\u00f3n hasta hacerla desaparecer.',
+    needsAt: () =>
+      '--at es obligatorio: da el momento en que aterriz\u00f3 el cambio, como marca ISO. Sin \u00e9l no hay frontera que comparar, y elegir una del log ser\u00eda que esta herramienta decida a qu\u00e9 cambio culpar.',
+    sides: (beforeRate, afterRate, before, after) =>
+      `antes ${beforeRate} (${before} resultados)   despu\u00e9s ${afterRate} (${after} resultados)`,
+    dropped: (from, to, outcomes, cost) =>
+      `La tasa de resoluci\u00f3n pas\u00f3 de ${from} a ${to} sobre ${outcomes} resultados medidos, y este cambio ${cost}. Las dos mitades son medidas; ninguna es una estimaci\u00f3n.`,
+    held: (from, to, outcomes) =>
+      `La tasa de resoluci\u00f3n pas\u00f3 de ${from} a ${to} sobre ${outcomes} resultados medidos \u2014 arriba, de forma medible.`,
+    cannotTell: (why, need) =>
+      why === 'too-few-before'
+        ? `No se puede decir: solo ${need} resultados antes del cambio, y esta puerta necesita 100 por lado. Falla builds, as\u00ed que el umbral no es el que usa una tasa en otros sitios.`
+        : why === 'too-few-after'
+          ? `A\u00fan no se puede decir: solo ${need} resultados desde el cambio, y esta puerta necesita 100 por lado. Vuelve a correrlo cuando el tr\u00e1fico se ponga al d\u00eda.`
+          : why === 'not-separable'
+            ? 'No se puede decir: la tasa no se movi\u00f3 de forma medible. Eso NO es lo mismo que "se mantuvo" \u2014 una puerta que las escribiera igual dejar\u00eda pasar una regresi\u00f3n real que simplemente no tuvo potencia para ver.'
+            : why === 'no-vocabulary'
+              ? 'No se puede decir: "outcomes.success" no declara nada, as\u00ed que no hay tasa de resoluci\u00f3n que comparar.'
+              : 'No se puede decir: algo que no es el prompt se movi\u00f3 a trav\u00e9s de la frontera. Ver abajo.',
+    confoundersHeading: () => 'El prompt no es lo \u00fanico que cambi\u00f3',
+    confounder: (kind, detail) =>
+      kind === 'model-mix-moved'
+        ? `La mezcla de modelos se movi\u00f3 un ${detail}. La ca\u00edda puede ser enteramente la migraci\u00f3n de otra persona, y esta herramienta no puede separarlas.`
+        : kind === 'volume-moved'
+          ? `El volumen de llamadas se movi\u00f3 ${detail}. Una carga cuyo tr\u00e1fico se mueve tanto suele ser una carga cuya poblaci\u00f3n cambi\u00f3 \u2014 una superficie nueva, un cliente nuevo, una campa\u00f1a \u2014 y las preguntas que se hacen no son las de antes.`
+          : `La cobertura de resultados pas\u00f3 de ${detail}. Las dos tasas describen poblaciones distintas: un equipo que empieza a instrumentar sus casos dif\u00edciles ve caer su tasa medida sin que nada haya empeorado.`,
+    notRandomised: () =>
+      'Esto es un antes y despu\u00e9s, no un experimento. Parte el tr\u00e1fico por tiempo y no al azar, as\u00ed que todo lo dem\u00e1s que cambi\u00f3 a la vez est\u00e1 tambi\u00e9n en la diferencia \u2014 por eso dice "no se puede decir" mucho m\u00e1s f\u00e1cilmente que un A/B.',
+    cannotSee: () =>
+      'No puede ver nada m\u00e1s que desplegaras ese d\u00eda. Un veredicto de "ca\u00edda" dice que la tasa baj\u00f3 y que las tres cosas que puede comprobar no se movieron. Es una afirmaci\u00f3n m\u00e1s peque\u00f1a que "lo hizo el prompt", y es la mayor que sostiene la evidencia.',
+    gateFailed: () => 'Puerta fallada: una ca\u00edda medida sin nada m\u00e1s que la explique.',
+    gateHeldOpen: () =>
+      'Puerta ni pasada ni fallada. "No se puede decir" mantiene la afirmaci\u00f3n abierta en vez de salir en verde \u2014 la postura de verify desde 1.39.',
+  },
+
+  experiment: {
+    heading: (a, b) => `Experimento: ${a} contra ${b}`,
+    needsTwo: () =>
+      'Nombra dos etiquetas a comparar, por ejemplo: trazum experiment <log> --a prompt-v1 --b prompt-v2',
+    needsRule: () =>
+      '--min-outcomes es obligatorio, y ese es justo el punto: una regla de parada declarada despu\u00e9s de mirar los n\u00fameros no es una regla de parada. Di cu\u00e1ntos resultados debe registrar cada brazo antes de poder leer el resultado.',
+    arm: (name, rate, successes, recorded, interval) =>
+      `${name}  ${rate}  (${successes} de ${recorded} registrados)  95% ${interval}`,
+    wins: (name, low, high) =>
+      `Gana ${name}. La diferencia est\u00e1 entre ${low} y ${high} con un 95% de confianza \u2014 el intervalo entero est\u00e1 a un lado del cero, que es lo que significa "gana" aqu\u00ed.`,
+    notSeparable: (why, needed) =>
+      why === 'no-difference-observed'
+        ? 'No separables: los dos brazos registraron la misma tasa. Ning\u00fan tama\u00f1o de muestra separa una diferencia de cero, as\u00ed que no hay ning\u00fan "d\u00e9jalo correr m\u00e1s" que ofrecer \u2014 no hay nada que encontrar.'
+        : why === 'nothing-recorded'
+          ? 'No separables: un brazo no registr\u00f3 ning\u00fan resultado, as\u00ed que no hay tasa que comparar.'
+          : `No separables con este tr\u00e1fico: el intervalo del 95% sobre la diferencia incluye el cero. Un n\u00famero es mayor, y eso no es un hallazgo. Unos ${needed} resultados por brazo zanjar\u00edan la diferencia observada hasta ahora.`,
+    peeked: (short, declared, recorded) =>
+      `Le\u00eddo antes de tiempo. La regla declarada eran ${declared} resultados por brazo y ${short} tiene ${recorded}. Nada puede impedir que se lea un n\u00famero antes de tiempo; esta l\u00ednea existe para que quien lea el resultado despu\u00e9s vea que se hizo.`,
+    honoured: (declared) => `Regla de parada respetada: los dos brazos superaron ${declared} resultados registrados.`,
+    marginalDearer: (better, usd) =>
+      `${better} resuelve m\u00e1s y cuesta m\u00e1s. Un acierto extra cuesta ${usd} \u2014 esa cifra, y no la tasa, es de lo que depende la decisi\u00f3n.`,
+    marginalCheaper: (better) => `${better} resuelve m\u00e1s Y cuesta menos por llamada. No se est\u00e1 cambiando nada por nada.`,
+    neverPromotes: () =>
+      'No se cambi\u00f3 nada. Un ganador es un hallazgo; tomarlo es una decisi\u00f3n con un nombre detr\u00e1s, y va en el plan como todo lo dem\u00e1s.',
+  },
+
   ladder: {
     heading: () => 'Escaleras de escalado',
     noLadders: () =>

package/src/i18n/types.ts

@@ -268,6 +268,36 @@ export interface CliMessages {
    * The ladder. Every line here exists to stop somebody reading "we route to
    * the cheap model first" as a saving without the number that decides it.
    */
+  quality: {
+    heading(label: string): string;
+    needsLabel(): string;
+    needsAt(): string;
+    sides(beforeRate: string, afterRate: string, before: string, after: string): string;
+    dropped(from: string, to: string, outcomes: string, cost: string): string;
+    held(from: string, to: string, outcomes: string): string;
+    cannotTell(why: string, need: string): string;
+    confounder(kind: string, detail: string): string;
+    confoundersHeading(): string;
+    notRandomised(): string;
+    cannotSee(): string;
+    gateFailed(): string;
+    gateHeldOpen(): string;
+  };
+
+  experiment: {
+    heading(a: string, b: string): string;
+    needsTwo(): string;
+    needsRule(): string;
+    arm(name: string, rate: string, successes: string, recorded: string, interval: string): string;
+    wins(name: string, low: string, high: string): string;
+    notSeparable(why: string, needed: string): string;
+    peeked(short: string, declared: string, recorded: string): string;
+    honoured(declared: string): string;
+    marginalDearer(better: string, usd: string): string;
+    marginalCheaper(better: string): string;
+    neverPromotes(): string;
+  };
+
   ladder: {
     heading(): string;
     noLadders(): string;

package/src/index.ts

@@ -40,6 +40,8 @@ import {
   budgetPositions,
   conform,
   BREAK_EVEN_BAND,
+  runExperiment,
+  qualityGate,
   ladderPosition,
   validateLadder,
   outcomeReport,
@@ -139,6 +141,8 @@ import type {
 import type {
   BudgetReport,
   ContractName,
+  ExperimentArm,
+  GateSide,
   FailurePolicy,
   GatewayStanding,
   UsageProfileReport,
@@ -245,6 +249,10 @@ interface Args {
 }
 
 const VALUE_FLAGS = new Set([
+  'a',
+  'at',
+  'b',
+  'min-outcomes',
   'against',
   'contract',
   'on-cannot-tell',
@@ -574,6 +582,8 @@ const COMMAND_FLAGS: Record<string, string[]> = {
   feedback: [],
   gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
   ladder: ['pricing', 'pricing-live', 'since', 'until', 'label'],
+  experiment: ['a', 'b', 'min-outcomes', 'pricing', 'pricing-live'],
+  quality: ['label', 'at', 'gate', 'pricing', 'pricing-live'],
   where: [],
   rules: [],
   blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -2324,6 +2334,281 @@ async function commandLadder(
   if (anyProblem) process.exitCode = 1;
 }
 
+/**
+ * `trazum experiment <log> --a <label> --b <label> --min-outcomes <n>`
+ *
+ * Two arms on real traffic, judged on recorded outcomes and cost together.
+ *
+ * `--min-outcomes` is required and that is the entire point of it. A stopping
+ * rule declared after looking at the numbers is not a stopping rule, and
+ * nothing here can stop somebody reading a result early — what it can do is
+ * make the early read **visible to whoever reads the result later**, which is
+ * the part that survives the afternoon.
+ */
+async function commandExperiment(
+  args: Args,
+  config: TrazumConfig,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
+  const path = args.positional[0];
+  if (path === undefined) throw new Error(t.errors.missingInputFile());
+
+  const aName = stringFlag(args, 'a');
+  const bName = stringFlag(args, 'b');
+  if (aName === undefined || bName === undefined) throw new Error(t.experiment.needsTwo());
+
+  const minRaw = stringFlag(args, 'min-outcomes');
+  const minOutcomesPerArm = minRaw === undefined ? Number.NaN : Number(minRaw);
+  if (!Number.isInteger(minOutcomesPerArm) || minOutcomesPerArm < 1) {
+    throw new Error(t.experiment.needsRule());
+  }
+
+  const report = profileUsage(await readUsageLog(path, t), { catalogue: pricing });
+  const n = (value: number): string => value.toLocaleString(t.numberLocale);
+  const pct = (value: number): string => `${(value * 100).toFixed(1)}%`;
+
+  const armOf = (label: string): ExperimentArm => {
+    const slice = report.outcomeTallyByLabel.find((entry) => entry.label === label);
+    return {
+      name: label,
+      totalUsd: slice?.totalUsd ?? 0,
+      tally: slice?.tally ?? { byValue: [], recorded: 0, parsed: 0, unrecordedUsd: 0 },
+    };
+  };
+
+  const result = runExperiment(
+    { arms: [aName, bName], minOutcomesPerArm },
+    { a: armOf(aName), b: armOf(bName) },
+    config.outcomes ?? null,
+  );
+
+  console.log();
+  console.log(c.bold(t.experiment.heading(aName, bName)));
+  console.log();
+  for (const side of [result.a, result.b]) {
+    console.log(
+      `  ${t.experiment.arm(
+        side.name,
+        side.rate === null ? '—' : pct(side.rate),
+        n(side.successes),
+        n(side.recorded),
+        side.interval === null ? '—' : `[${pct(side.interval.low)}, ${pct(side.interval.high)}]`,
+      )}`,
+    );
+  }
+  console.log();
+
+  if (result.separation === 'not-separable') {
+    console.log(
+      `  ${c.dim('·')} ${wrap(
+        t.experiment.notSeparable(
+          result.notSeparable ?? '',
+          result.outcomesNeededPerArm === null ? '—' : n(result.outcomesNeededPerArm),
+        ),
+        74,
+        '    ',
+      )}`,
+    );
+  } else {
+    const winner = result.separation === 'a-wins' ? result.a.name : result.b.name;
+    const d = result.difference as { low: number; high: number };
+    // Reported as a magnitude: the sign is carried by which arm is named, and
+    // printing "-30.0% to -18.0%" beside "b wins" is two ways of saying the
+    // same thing that a reader has to reconcile.
+    const lo = Math.min(Math.abs(d.low), Math.abs(d.high));
+    const hi = Math.max(Math.abs(d.low), Math.abs(d.high));
+    console.log(`  ${c.green('✓')} ${wrap(t.experiment.wins(winner, pct(lo), pct(hi)), 74, '    ')}`);
+  }
+
+  /**
+   * The peek line, printed **whether or not** the arms separated.
+   *
+   * A separable result read too early is still separable and still read too
+   * early. Collapsing the two would hide one of the facts, and it is always
+   * the inconvenient one that goes.
+   */
+  console.log();
+  if (result.stopping.honoured) {
+    console.log(`  ${c.dim(wrap(t.experiment.honoured(n(result.stopping.declared)), 74, '    '))}`);
+  } else {
+    const short = result.stopping.short === result.a.name ? result.a : result.b;
+    console.log(
+      `  ${c.yellow('!')} ${wrap(
+        t.experiment.peeked(short.name, n(result.stopping.declared), n(short.recorded)),
+        74,
+        '    ',
+      )}`,
+    );
+  }
+
+  if (result.marginal !== null) {
+    console.log();
+    console.log(
+      `  ${wrap(
+        result.marginal.usdPerExtraSuccess !== null
+          ? t.experiment.marginalDearer(
+              result.marginal.better,
+              formatUsd(result.marginal.usdPerExtraSuccess),
+            )
+          : t.experiment.marginalCheaper(result.marginal.better),
+        74,
+        '    ',
+      )}`,
+    );
+  }
+
+  console.log();
+  console.log(`  ${c.dim(wrap(t.experiment.neverPromotes(), 74, '    '))}`);
+  console.log();
+}
+
+/**
+ * `trazum quality <log> --label <name> --at <iso> [--gate]`
+ *
+ * The failure that actually matters: a prompt edit that quietly made the
+ * product worse. CI has been able to fail a build for tokens since 1.4 and for
+ * dollars since 1.21, and this has never been gateable — so every saving this
+ * tool has ever recommended went into a repository with its most important
+ * consequence unmeasured.
+ *
+ * **Named `quality` rather than `check --against-outcomes`, which is what the
+ * plan called for.** `check` reads *prompt files* and gates on tokens; it has
+ * never opened a usage log, and a command that takes either a prompt or a log
+ * depending on a flag is two commands wearing one name. The split-by-time this
+ * needs is also not a `check` idea — there is nothing in a prompt file with a
+ * timestamp on it.
+ */
+async function commandQuality(
+  args: Args,
+  config: TrazumConfig,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
+  const path = args.positional[0];
+  if (path === undefined) throw new Error(t.errors.missingInputFile());
+
+  const label = stringFlag(args, 'label');
+  if (label === undefined) throw new Error(t.quality.needsLabel());
+
+  const atRaw = stringFlag(args, 'at');
+  const atMs = atRaw === undefined ? Number.NaN : Date.parse(atRaw);
+  if (!Number.isFinite(atMs)) throw new Error(t.quality.needsAt());
+
+  /**
+   * Two profiles over the same file, split at the boundary — rather than one
+   * profile the caller has to slice.
+   *
+   * The alternative is asking somebody for two logs, which invites the mistake
+   * this whole module exists to avoid: two files gathered under conditions
+   * nobody wrote down.
+   */
+  const raw = await readUsageLog(path, t);
+  const sideOf = (since: number | undefined, until: number | undefined): GateSide => {
+    const report = profileUsage(raw, { catalogue: pricing, label, sinceMs: since, untilMs: until });
+    const slice = report.outcomeTallyByLabel.find((entry) => entry.label === label);
+    return {
+      arm: {
+        name: label,
+        totalUsd: report.total.totalUsd,
+        tally: slice?.tally ?? { byValue: [], recorded: 0, parsed: 0, unrecordedUsd: 0 },
+      },
+      calls: report.total.calls,
+      usdByModel: report.byModel.map((entry) => ({ model: entry.model, usd: entry.breakdown.totalUsd })),
+    };
+  };
+
+  const result = qualityGate(sideOf(undefined, atMs), sideOf(atMs, undefined), config.outcomes ?? null);
+  const pct = (value: number): string => `${(value * 100).toFixed(1)}%`;
+  const n = (value: number): string => value.toLocaleString(t.numberLocale);
+
+  console.log();
+  console.log(c.bold(t.quality.heading(label)));
+  console.log(`  ${c.dim(wrap(t.quality.notRandomised(), 74, '  '))}`);
+  console.log();
+  console.log(
+    `  ${t.quality.sides(
+      result.before.rate === null ? '—' : pct(result.before.rate),
+      result.after.rate === null ? '—' : pct(result.after.rate),
+      n(result.outcomes.before),
+      n(result.outcomes.after),
+    )}`,
+  );
+  console.log();
+
+  if (result.verdict === 'dropped') {
+    const cost =
+      result.cost === null
+        ? ''
+        : result.cost.deltaUsdPerCall < 0
+          ? `saves ${formatUsd(-result.cost.deltaUsdPerCall)} a call`
+          : `costs ${formatUsd(result.cost.deltaUsdPerCall)} a call more`;
+    console.log(
+      `  ${c.red('✗')} ${wrap(
+        t.quality.dropped(
+          pct(result.before.rate ?? 0),
+          pct(result.after.rate ?? 0),
+          n(result.outcomes.before + result.outcomes.after),
+          cost,
+        ),
+        74,
+        '    ',
+      )}`,
+    );
+  } else if (result.verdict === 'held') {
+    console.log(
+      `  ${c.green('✓')} ${wrap(
+        t.quality.held(pct(result.before.rate ?? 0), pct(result.after.rate ?? 0), n(result.outcomes.before + result.outcomes.after)),
+        74,
+        '    ',
+      )}`,
+    );
+  } else {
+    const need =
+      result.unknown === 'too-few-before' ? n(result.outcomes.before) : n(result.outcomes.after);
+    console.log(`  ${c.yellow('?')} ${wrap(t.quality.cannotTell(result.unknown ?? '', need), 74, '    ')}`);
+  }
+
+  /**
+   * Confounders print on **every** verdict, not only on `cannot-tell`.
+   *
+   * A rate that held while the model changed underneath is not evidence that
+   * the prompt is fine either, and hiding the confounder on a green result is
+   * how a gate teaches people to trust it in exactly the case it should not be
+   * trusted.
+   */
+  if (result.confounders.length > 0) {
+    console.log();
+    console.log(`  ${c.bold(t.quality.confoundersHeading())}`);
+    for (const confounder of result.confounders) {
+      const detail =
+        confounder.kind === 'model-mix-moved'
+          ? `${pct(confounder.drift)} (${confounder.model})`
+          : confounder.kind === 'volume-moved'
+            ? `${n(confounder.beforeCalls)} → ${n(confounder.afterCalls)} calls`
+            : `${pct(confounder.before)} → ${pct(confounder.after)}`;
+      console.log(`    ${c.yellow('!')} ${wrap(t.quality.confounder(confounder.kind, detail), 70, '      ')}`);
+    }
+  }
+
+  console.log();
+  console.log(`  ${c.dim(wrap(t.quality.cannotSee(), 74, '  '))}`);
+
+  if (boolFlag(args, 'gate')) {
+    console.log();
+    if (result.verdict === 'dropped') {
+      console.log(`  ${c.red(t.quality.gateFailed())}`);
+      process.exitCode = 1;
+    } else if (result.verdict === 'cannot-tell') {
+      // Three outcomes, never two. `cannot tell` holds the claim open rather
+      // than exiting green, the posture `verify --gate` has had since 1.39.
+      console.log(`  ${c.yellow(t.quality.gateHeldOpen())}`);
+      process.exitCode = 2;
+    }
+  }
+  console.log();
+}
+
 function commandModels(t: CliMessages, pricing: PricingCatalogue): void {
   const n = (value: number): string => value.toLocaleString(t.numberLocale);
   const col = t.models.columns;
@@ -8524,6 +8809,12 @@ async function main(): Promise<void> {
     case 'models':
       commandModels(t, pricing);
       break;
+    case 'quality':
+      await commandQuality(args, config, pricing, t);
+      break;
+    case 'experiment':
+      await commandExperiment(args, config, pricing, t);
+      break;
     case 'ladder':
       await commandLadder(args, config, pricing, t);
       break;