@trazum/cli 1.56.0 → 1.56.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@trazum/cli",
3
- "version": "1.56.0",
3
+ "version": "1.56.1",
4
4
  "description": "Trazum CLI: find where your LLM bill goes, price every finding per month, and enforce token budgets in CI.",
5
5
  "license": "MIT",
6
6
  "author": "David Mu\u00f1oz Rey",
@@ -37,7 +37,7 @@
37
37
  "prepublishOnly": "npm run build && npm test"
38
38
  },
39
39
  "dependencies": {
40
- "@trazum/core": "1.56.0"
40
+ "@trazum/core": "1.56.1"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/node": "^26.2.0",
package/src/i18n/en.ts CHANGED
@@ -59,7 +59,7 @@ ${bold('USAGE')}
59
59
  trazum rollup <document...|dir> [--json]
60
60
  trazum pulse [--max-stale-hours <n>]
61
61
  trazum models
62
- trazum rules
62
+ trazum rules [--measure <dir>] [--level <safe|aggressive>]
63
63
  trazum gateway <provider> --on-cannot-tell <fail-open|fail-closed>
64
64
  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
65
65
  trazum quality <log> --label <name> --at <iso> [--gate]
@@ -136,6 +136,25 @@ ${bold('OPTIONS FOR pulse')}
136
136
  late against — and never on how far the measurements reach, which is a
137
137
  provider reporting on its own schedule rather than a job that failed.
138
138
 
139
+ ${bold('OPTIONS FOR rules')}
140
+ --measure <dir> Measure what each rule actually recovers over the
141
+ prompts in that directory, instead of listing what
142
+ each one does.
143
+ --level <safe|aggressive> Which rules to measure. Default: safe.
144
+ --json The measurement as data.
145
+
146
+ Without --measure it lists the rules and what each is for. With it, the
147
+ optimiser is run once per rule alone and once per rule removed, and both
148
+ figures are printed: where two rules find the same tokens they diverge, and
149
+ either one on its own would be wrong in a different direction.
150
+
151
+ The floor is separated out. The optimiser normalises whitespace whether or not
152
+ a rule is enabled, and crediting that to the rules is how a headline
153
+ percentage survives on a corpus where the rules recover nothing.
154
+
155
+ "Inert" is always said about the corpus. A rule that finds nothing in these
156
+ files has not been shown to find nothing anywhere.
157
+
139
158
  ${bold('OPTIONS FOR feedback')}
140
159
  (none)
141
160
 
@@ -1612,6 +1631,24 @@ ${bold('EXAMPLES')}
1612
1631
  rules: {
1613
1632
  title: () => 'Available rules',
1614
1633
  disableHint: () => ' Turn off the ones you do not want with --disable id1,id2',
1634
+ measureHeading: (root, prompts, level) =>
1635
+ `What each rule recovers in ${root} — ${plural(prompts, 'prompt')}, level ${level}`,
1636
+ measureTotals: (before, saved, floor) =>
1637
+ `${before} tokens before. The rules recover ${saved}; normalisation recovers ${floor} whether a rule is enabled or not, and that is not the rules' work.`,
1638
+ measureRow: (id, marginal, alone, prompts) =>
1639
+ `${id.padEnd(22)} lost if removed ${marginal.padStart(6)} alone ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
1640
+ measureOverlap: (sumOfAlone, saved) =>
1641
+ `The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap — two rules finding the same tokens — and it is stated rather than resolved into a total, because a single figure here is the one number that cannot be true.`,
1642
+ measureRedundant: (ids) =>
1643
+ `Every token these find, something else finds too, here: ${ids}. An overlap, not a defect — and not a reason to delete one without deciding which is the better to keep.`,
1644
+ measureFiredWithoutSaving: (ids) =>
1645
+ `These changed the prompt and recovered no tokens: ${ids}. Not the same as inert — they were exercised, and they are altering somebody's instruction for no measured benefit.`,
1646
+ measureInert: (ids) =>
1647
+ `These changed nothing in this corpus: ${ids}. That is a fact about these files, not about the rules: a rule that finds nothing here has not been shown to find nothing anywhere.`,
1648
+ measureBand: (source) =>
1649
+ source === 'heuristic'
1650
+ ? 'Counted with the built-in estimator, so every figure above carries its documented band. A rule whose yield is a handful of tokens is inside the noise.'
1651
+ : 'Counted with an external tokenizer.',
1615
1652
  },
1616
1653
 
1617
1654
 
package/src/i18n/es.ts CHANGED
@@ -46,7 +46,7 @@ ${bold('USO')}
46
46
  trazum rollup <documento...|dir> [--json]
47
47
  trazum pulse [--max-stale-hours <n>]
48
48
  trazum models
49
- trazum rules
49
+ trazum rules [--measure <dir>] [--level <safe|aggressive>]
50
50
  trazum gateway <provider> --on-cannot-tell <fail-open|fail-closed>
51
51
  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
52
52
  trazum quality <log> --label <name> --at <iso> [--gate]
@@ -124,6 +124,26 @@ ${bold('OPCIONES DE pulse')}
124
124
  contra la que ir tarde — ni por hasta dónde llegan las mediciones, que es un
125
125
  proveedor informando a su ritmo y no un trabajo que falló.
126
126
 
127
+ ${bold('OPCIONES DE rules')}
128
+ --measure <dir> Mide lo que recupera realmente cada regla sobre
129
+ los prompts de ese directorio, en vez de listar
130
+ qué hace cada una.
131
+ --level <safe|aggressive> Qué reglas medir. Por defecto: safe.
132
+ --json La medición como datos.
133
+
134
+ Sin --measure lista las reglas y para qué sirve cada una. Con él, el
135
+ optimizador se ejecuta una vez con cada regla sola y una vez con cada regla
136
+ quitada, y se imprimen ambas cifras: donde dos reglas encuentran los mismos
137
+ tokens divergen, y cualquiera de las dos por separado se equivocaría en una
138
+ dirección distinta.
139
+
140
+ El suelo se separa. El optimizador normaliza espacios haya reglas activas o
141
+ no, y atribuir eso a las reglas es como sobrevive un porcentaje de titular en
142
+ un corpus donde las reglas no recuperan nada.
143
+
144
+ "Inerte" se dice siempre sobre el corpus. Una regla que no encuentra nada en
145
+ estos ficheros no ha demostrado no encontrar nada en ninguna parte.
146
+
127
147
  ${bold('OPCIONES DE feedback')}
128
148
  (ninguna)
129
149
 
@@ -1620,6 +1640,24 @@ ${bold('EJEMPLOS')}
1620
1640
  rules: {
1621
1641
  title: () => 'Reglas disponibles',
1622
1642
  disableHint: () => ' Desactiva las que no quieras con --disable id1,id2',
1643
+ measureHeading: (root, prompts, level) =>
1644
+ `Qué recupera cada regla en ${root} — ${plural(prompts, 'prompt')}, nivel ${level}`,
1645
+ measureTotals: (before, saved, floor) =>
1646
+ `${before} tokens antes. Las reglas recuperan ${saved}; la normalización recupera ${floor} haya reglas activas o no, y eso no es obra de las reglas.`,
1647
+ measureRow: (id, marginal, alone, prompts) =>
1648
+ `${id.padEnd(22)} se pierde si se quita ${marginal.padStart(6)} sola ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
1649
+ measureOverlap: (sumOfAlone, saved) =>
1650
+ `Las reglas recuperan ${sumOfAlone} entre todas de una en una y ${saved} juntas. La diferencia es el solapamiento — dos reglas encontrando los mismos tokens — y se declara en vez de resolverse en un total, porque una sola cifra aquí es el único número que no puede ser cierto.`,
1651
+ measureRedundant: (ids) =>
1652
+ `Cada token que encuentran estas lo encuentra algo más, aquí: ${ids}. Un solapamiento, no un defecto — y no una razón para borrar una sin decidir cuál conviene conservar.`,
1653
+ measureFiredWithoutSaving: (ids) =>
1654
+ `Estas cambiaron el prompt y no recuperaron ningún token: ${ids}. No es lo mismo que inerte — sí se ejercitaron, y están alterando la instrucción de alguien sin beneficio medido.`,
1655
+ measureInert: (ids) =>
1656
+ `Estas no cambiaron nada en este corpus: ${ids}. Es un hecho sobre estos ficheros, no sobre las reglas: una regla que no encuentra nada aquí no ha demostrado no encontrar nada en ninguna parte.`,
1657
+ measureBand: (source) =>
1658
+ source === 'heuristic'
1659
+ ? 'Contado con el estimador incorporado, así que toda cifra de arriba lleva su banda documentada. Una regla cuyo rendimiento son unos pocos tokens está dentro del ruido.'
1660
+ : 'Contado con un tokenizador externo.',
1623
1661
  },
1624
1662
 
1625
1663
 
package/src/i18n/types.ts CHANGED
@@ -516,6 +516,14 @@ export interface CliMessages {
516
516
  rules: {
517
517
  title(): string;
518
518
  disableHint(): string;
519
+ measureHeading(root: string, prompts: number, level: string): string;
520
+ measureTotals(before: string, saved: string, floor: string): string;
521
+ measureRow(id: string, marginal: string, alone: string, prompts: number): string;
522
+ measureOverlap(sumOfAlone: string, saved: string): string;
523
+ measureRedundant(ids: string): string;
524
+ measureInert(ids: string): string;
525
+ measureFiredWithoutSaving(ids: string): string;
526
+ measureBand(source: string): string;
519
527
  };
520
528
 
521
529
  /** `trazum rank` — which prompts to fix first, and why. */
package/src/index.ts CHANGED
@@ -71,6 +71,7 @@ import {
71
71
  fleetRollup,
72
72
  rollUp,
73
73
  heartbeats,
74
+ ruleYield,
74
75
  labelCoverage,
75
76
  measuredUsage,
76
77
  gateMargin,
@@ -299,6 +300,7 @@ const VALUE_FLAGS = new Set([
299
300
  'max-growth-usd',
300
301
  'max-cache-loss-usd',
301
302
  'max-stale-hours',
303
+ 'measure',
302
304
  'max-day-usd',
303
305
  'max-session-usd',
304
306
  'csv-out',
@@ -608,7 +610,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
608
610
  commitment: ['floor', 'discount', 'months', 'pricing', 'pricing-live'],
609
611
  report: ['year', 'json', 'pricing', 'pricing-live'],
610
612
  where: [],
611
- rules: [],
613
+ rules: ['measure', 'level', 'json'],
612
614
  blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
613
615
  doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
614
616
  };
@@ -3626,6 +3628,104 @@ function commandRules(t: CliMessages, locale: Locale): void {
3626
3628
  }
3627
3629
  }
3628
3630
 
3631
+ /**
3632
+ * `trazum rules --measure <dir> [--level <safe|aggressive>] [--json]` — and
3633
+ * what does each one actually recover here?
3634
+ *
3635
+ * The README says plainly that the deterministic rules recover about one per
3636
+ * cent, and that is the fair complaint about this tool. It is also an
3637
+ * **aggregate**, and an aggregate is where a distribution goes to hide: the
3638
+ * same one per cent is consistent with every rule pulling its weight and with
3639
+ * two rules doing all of it beside five that have never changed a byte.
3640
+ *
3641
+ * This measures which, over prompts the reader actually has, by running the
3642
+ * optimiser with each rule alone and then with each rule removed. **Both
3643
+ * figures are printed and neither is reconciled into the other**, because
3644
+ * where two rules find the same tokens they diverge, and either one on its own
3645
+ * would be wrong in a different direction.
3646
+ *
3647
+ * **The floor is separated out.** The optimiser normalises whitespace whether
3648
+ * or not any rule is enabled, and crediting that to the rules is how a
3649
+ * headline percentage survives on a corpus where the rules recover nothing.
3650
+ * The first version of the module did exactly that and reported every rule as
3651
+ * redundant.
3652
+ *
3653
+ * "Inert" is always said **about the corpus**. A rule that finds nothing in
3654
+ * these files has not been shown to find nothing anywhere, and the difference
3655
+ * is the whole distance between "delete this rule" and "measure it on
3656
+ * something else".
3657
+ */
3658
+ async function commandRulesMeasure(
3659
+ args: Args,
3660
+ config: TrazumConfig,
3661
+ pricing: PricingCatalogue,
3662
+ t: CliMessages,
3663
+ ): Promise<void> {
3664
+ const root = stringFlag(args, 'measure')!;
3665
+ const level = (stringFlag(args, 'level') ?? 'safe') as RuleLevel;
3666
+ if (level !== 'safe' && level !== 'aggressive') throw new Error(t.errors.badLevel(level));
3667
+
3668
+ const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
3669
+ const { files, truncated } = await walkPrompts(root, { extensions });
3670
+ if (files.length === 0) throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
3671
+
3672
+ const prompts: string[] = [];
3673
+ for (const file of files) {
3674
+ // `walkPrompts` returns names relative to the root it was given, so the
3675
+ // root has to go back on. Reading them bare only worked from a run whose
3676
+ // cwd happened to be the root, which is the one case a first probe uses.
3677
+ const raw = await readFile(join(root, file), 'utf8');
3678
+ // A marked source file contributes its marked prompts; anything else
3679
+ // contributes itself. The same rule `check` and `doctor` already follow,
3680
+ // so the three commands measure the same text.
3681
+ if (hasMarker(raw)) prompts.push(...extractPrompts(raw).prompts.map((p) => p.text));
3682
+ else prompts.push(raw);
3683
+ }
3684
+
3685
+ const report = ruleYield(prompts, RULES.map((rule) => rule.id), { level, pricing });
3686
+
3687
+ if (boolFlag(args, 'json')) {
3688
+ console.log(JSON.stringify({ ...report, root, files: files.length, truncated }, null, 2));
3689
+ return;
3690
+ }
3691
+
3692
+ const n = (value: number): string => value.toLocaleString(t.numberLocale);
3693
+
3694
+ console.log();
3695
+ console.log(c.bold(t.rules.measureHeading(root, report.prompts, level)));
3696
+ console.log(` ${c.dim(t.rules.measureTotals(n(report.tokensBefore), n(report.tokensSaved), n(report.floor)))}`);
3697
+ console.log();
3698
+
3699
+ for (const rule of report.rules) {
3700
+ const line = t.rules.measureRow(rule.id, n(rule.marginal), n(rule.alone), rule.prompts);
3701
+ if (rule.marginal > 0) console.log(` ${line}`);
3702
+ else console.log(` ${c.dim(line)}`);
3703
+ }
3704
+
3705
+ console.log();
3706
+ // The overlap, stated as the gap rather than resolved into a total: a single
3707
+ // figure here is the one number that cannot be true.
3708
+ if (report.sumOfAlone !== report.tokensSaved) {
3709
+ console.log(` ${c.dim(wrap(t.rules.measureOverlap(n(report.sumOfAlone), n(report.tokensSaved)), 74, ' '))}`);
3710
+ }
3711
+ if (report.redundantHere.length > 0) {
3712
+ console.log(` ${c.dim(wrap(t.rules.measureRedundant(report.redundantHere.join(', ')), 74, ' '))}`);
3713
+ }
3714
+ // Fired-and-saved-nothing before never-fired: the first is a finding about
3715
+ // the rule, the second only about the corpus, and a reader who meets them the
3716
+ // other way round reads both as the same shrug.
3717
+ if (report.firedWithoutSavingHere.length > 0) {
3718
+ console.log(
3719
+ ` ${c.yellow(wrap(t.rules.measureFiredWithoutSaving(report.firedWithoutSavingHere.join(', ')), 74, ' '))}`,
3720
+ );
3721
+ }
3722
+ if (report.inertHere.length > 0) {
3723
+ console.log(` ${c.dim(wrap(t.rules.measureInert(report.inertHere.join(', ')), 74, ' '))}`);
3724
+ }
3725
+ console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
3726
+ console.log();
3727
+ }
3728
+
3629
3729
  async function readInput(source: string | undefined, t: CliMessages): Promise<string> {
3630
3730
  if (!source) {
3631
3731
  throw new Error(t.errors.missingInputFile());
@@ -9864,7 +9964,8 @@ async function main(): Promise<void> {
9864
9964
  await commandWhere(args, config, pricing, t);
9865
9965
  break;
9866
9966
  case 'rules':
9867
- commandRules(t, locale);
9967
+ if (stringFlag(args, 'measure') !== undefined) await commandRulesMeasure(args, config, pricing, t);
9968
+ else commandRules(t, locale);
9868
9969
  break;
9869
9970
  case 'doctor':
9870
9971
  await commandDoctor(args, config, pricing, t, locale);