@trazum/cli 1.56.0 → 1.56.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@trazum/cli",
3
- "version": "1.56.0",
3
+ "version": "1.56.2",
4
4
  "description": "Trazum CLI: find where your LLM bill goes, price every finding per month, and enforce token budgets in CI.",
5
5
  "license": "MIT",
6
6
  "author": "David Mu\u00f1oz Rey",
@@ -37,7 +37,7 @@
37
37
  "prepublishOnly": "npm run build && npm test"
38
38
  },
39
39
  "dependencies": {
40
- "@trazum/core": "1.56.0"
40
+ "@trazum/core": "1.56.2"
41
41
  },
42
42
  "devDependencies": {
43
43
  "@types/node": "^26.2.0",
package/src/i18n/en.ts CHANGED
@@ -59,7 +59,7 @@ ${bold('USAGE')}
59
59
  trazum rollup <document...|dir> [--json]
60
60
  trazum pulse [--max-stale-hours <n>]
61
61
  trazum models
62
- trazum rules
62
+ trazum rules [--measure <dir>] [--level <safe|aggressive>]
63
63
  trazum gateway <provider> --on-cannot-tell <fail-open|fail-closed>
64
64
  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
65
65
  trazum quality <log> --label <name> --at <iso> [--gate]
@@ -136,6 +136,25 @@ ${bold('OPTIONS FOR pulse')}
136
136
  late against — and never on how far the measurements reach, which is a
137
137
  provider reporting on its own schedule rather than a job that failed.
138
138
 
139
+ ${bold('OPTIONS FOR rules')}
140
+ --measure <dir> Measure what each rule actually recovers over the
141
+ prompts in that directory, instead of listing what
142
+ each one does.
143
+ --level <safe|aggressive> Which rules to measure. Default: safe.
144
+ --json The measurement as data.
145
+
146
+ Without --measure it lists the rules and what each is for. With it, the
147
+ optimiser is run once per rule alone and once per rule removed, and both
148
+ figures are printed: where two rules find the same tokens they diverge, and
149
+ either one on its own would be wrong in a different direction.
150
+
151
+ The floor is separated out. The optimiser normalises whitespace whether or not
152
+ a rule is enabled, and crediting that to the rules is how a headline
153
+ percentage survives on a corpus where the rules recover nothing.
154
+
155
+ "Inert" is always said about the corpus. A rule that finds nothing in these
156
+ files has not been shown to find nothing anywhere.
157
+
139
158
  ${bold('OPTIONS FOR feedback')}
140
159
  (none)
141
160
 
@@ -905,6 +924,9 @@ ${bold('EXAMPLES')}
905
924
  dictionaryCoverage: (languages) =>
906
925
  ` The phrase dictionaries cover ${languages}. A prompt in another language `
907
926
  + 'is not necessarily efficient — it may just be one Trazum cannot read yet.',
927
+ dictionaryUnreviewed: (languages) =>
928
+ ` Of those, ${languages} carry entries nobody here reads: written by the same `
929
+ + 'process that wrote the rules, never agreed by a speaker of the language.',
908
930
  levelAggressive: () => '[aggressive]',
909
931
  levelSafe: () => '[safe]',
910
932
  ruleHits: (hits, tokensSaved) => `(${hits}×, ~${tokensSaved} tokens)`,
@@ -1612,6 +1634,24 @@ ${bold('EXAMPLES')}
1612
1634
  rules: {
1613
1635
  title: () => 'Available rules',
1614
1636
  disableHint: () => ' Turn off the ones you do not want with --disable id1,id2',
1637
+ measureHeading: (root, prompts, level) =>
1638
+ `What each rule recovers in ${root} — ${plural(prompts, 'prompt')}, level ${level}`,
1639
+ measureTotals: (before, saved, floor) =>
1640
+ `${before} tokens before. The rules recover ${saved}; normalisation recovers ${floor} whether a rule is enabled or not, and that is not the rules' work.`,
1641
+ measureRow: (id, marginal, alone, prompts) =>
1642
+ `${id.padEnd(22)} lost if removed ${marginal.padStart(6)} alone ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
1643
+ measureOverlap: (sumOfAlone, saved) =>
1644
+ `The rules recover ${sumOfAlone} between them one at a time and ${saved} together. The gap is the overlap — two rules finding the same tokens — and it is stated rather than resolved into a total, because a single figure here is the one number that cannot be true.`,
1645
+ measureRedundant: (ids) =>
1646
+ `Every token these find, something else finds too, here: ${ids}. An overlap, not a defect — and not a reason to delete one without deciding which is the better to keep.`,
1647
+ measureFiredWithoutSaving: (ids) =>
1648
+ `These changed the prompt and recovered no tokens: ${ids}. Not the same as inert — they were exercised, and they are altering somebody's instruction for no measured benefit.`,
1649
+ measureInert: (ids) =>
1650
+ `These changed nothing in this corpus: ${ids}. That is a fact about these files, not about the rules: a rule that finds nothing here has not been shown to find nothing anywhere.`,
1651
+ measureBand: (source) =>
1652
+ source === 'heuristic'
1653
+ ? 'Counted with the built-in estimator, so every figure above carries its documented band. A rule whose yield is a handful of tokens is inside the noise.'
1654
+ : 'Counted with an external tokenizer.',
1615
1655
  },
1616
1656
 
1617
1657
 
package/src/i18n/es.ts CHANGED
@@ -46,7 +46,7 @@ ${bold('USO')}
46
46
  trazum rollup <documento...|dir> [--json]
47
47
  trazum pulse [--max-stale-hours <n>]
48
48
  trazum models
49
- trazum rules
49
+ trazum rules [--measure <dir>] [--level <safe|aggressive>]
50
50
  trazum gateway <provider> --on-cannot-tell <fail-open|fail-closed>
51
51
  trazum experiment <log> --a <label> --b <label> --min-outcomes <n>
52
52
  trazum quality <log> --label <name> --at <iso> [--gate]
@@ -124,6 +124,26 @@ ${bold('OPCIONES DE pulse')}
124
124
  contra la que ir tarde — ni por hasta dónde llegan las mediciones, que es un
125
125
  proveedor informando a su ritmo y no un trabajo que falló.
126
126
 
127
+ ${bold('OPCIONES DE rules')}
128
+ --measure <dir> Mide lo que recupera realmente cada regla sobre
129
+ los prompts de ese directorio, en vez de listar
130
+ qué hace cada una.
131
+ --level <safe|aggressive> Qué reglas medir. Por defecto: safe.
132
+ --json La medición como datos.
133
+
134
+ Sin --measure lista las reglas y para qué sirve cada una. Con él, el
135
+ optimizador se ejecuta una vez con cada regla sola y una vez con cada regla
136
+ quitada, y se imprimen ambas cifras: donde dos reglas encuentran los mismos
137
+ tokens divergen, y cualquiera de las dos por separado se equivocaría en una
138
+ dirección distinta.
139
+
140
+ El suelo se separa. El optimizador normaliza espacios haya reglas activas o
141
+ no, y atribuir eso a las reglas es como sobrevive un porcentaje de titular en
142
+ un corpus donde las reglas no recuperan nada.
143
+
144
+ "Inerte" se dice siempre sobre el corpus. Una regla que no encuentra nada en
145
+ estos ficheros no ha demostrado no encontrar nada en ninguna parte.
146
+
127
147
  ${bold('OPCIONES DE feedback')}
128
148
  (ninguna)
129
149
 
@@ -923,6 +943,10 @@ ${bold('EJEMPLOS')}
923
943
  ` Los diccionarios de frases cubren ${languages}. Un prompt en otro idioma `
924
944
  + 'no es necesariamente eficiente: puede ser simplemente uno que Trazum '
925
945
  + 'todavía no sabe leer.',
946
+ dictionaryUnreviewed: (languages) =>
947
+ ` De esos, ${languages} llevan entradas que nadie aquí lee: escritas por el `
948
+ + 'mismo proceso que escribió las reglas, nunca aprobadas por alguien que '
949
+ + 'hable el idioma.',
926
950
  levelAggressive: () => '[agresiva]',
927
951
  levelSafe: () => '[segura]',
928
952
  ruleHits: (hits, tokensSaved) => `(${hits}×, ~${tokensSaved} tokens)`,
@@ -1620,6 +1644,24 @@ ${bold('EJEMPLOS')}
1620
1644
  rules: {
1621
1645
  title: () => 'Reglas disponibles',
1622
1646
  disableHint: () => ' Desactiva las que no quieras con --disable id1,id2',
1647
+ measureHeading: (root, prompts, level) =>
1648
+ `Qué recupera cada regla en ${root} — ${plural(prompts, 'prompt')}, nivel ${level}`,
1649
+ measureTotals: (before, saved, floor) =>
1650
+ `${before} tokens antes. Las reglas recuperan ${saved}; la normalización recupera ${floor} haya reglas activas o no, y eso no es obra de las reglas.`,
1651
+ measureRow: (id, marginal, alone, prompts) =>
1652
+ `${id.padEnd(22)} se pierde si se quita ${marginal.padStart(6)} sola ${alone.padStart(6)} ${plural(prompts, 'prompt')}`,
1653
+ measureOverlap: (sumOfAlone, saved) =>
1654
+ `Las reglas recuperan ${sumOfAlone} entre todas de una en una y ${saved} juntas. La diferencia es el solapamiento — dos reglas encontrando los mismos tokens — y se declara en vez de resolverse en un total, porque una sola cifra aquí es el único número que no puede ser cierto.`,
1655
+ measureRedundant: (ids) =>
1656
+ `Cada token que encuentran estas lo encuentra algo más, aquí: ${ids}. Un solapamiento, no un defecto — y no una razón para borrar una sin decidir cuál conviene conservar.`,
1657
+ measureFiredWithoutSaving: (ids) =>
1658
+ `Estas cambiaron el prompt y no recuperaron ningún token: ${ids}. No es lo mismo que inerte — sí se ejercitaron, y están alterando la instrucción de alguien sin beneficio medido.`,
1659
+ measureInert: (ids) =>
1660
+ `Estas no cambiaron nada en este corpus: ${ids}. Es un hecho sobre estos ficheros, no sobre las reglas: una regla que no encuentra nada aquí no ha demostrado no encontrar nada en ninguna parte.`,
1661
+ measureBand: (source) =>
1662
+ source === 'heuristic'
1663
+ ? 'Contado con el estimador incorporado, así que toda cifra de arriba lleva su banda documentada. Una regla cuyo rendimiento son unos pocos tokens está dentro del ruido.'
1664
+ : 'Contado con un tokenizador externo.',
1623
1665
  },
1624
1666
 
1625
1667
 
package/src/i18n/types.ts CHANGED
@@ -92,6 +92,13 @@ export interface CliMessages {
92
92
  nothingToTrim(): string;
93
93
  /** The languages the phrase dictionaries cover, printed when nothing fired. */
94
94
  dictionaryCoverage(languages: string): string;
95
+ /**
96
+ * Of those, the ones nobody here has been shown to read.
97
+ *
98
+ * Its own line rather than a qualifier inside `dictionaryCoverage`, because
99
+ * the day somebody maintains all seven it is deleted rather than reworded.
100
+ */
101
+ dictionaryUnreviewed(languages: string): string;
95
102
  levelAggressive(): string;
96
103
  levelSafe(): string;
97
104
  ruleHits(hits: number, tokensSaved: number): string;
@@ -516,6 +523,14 @@ export interface CliMessages {
516
523
  rules: {
517
524
  title(): string;
518
525
  disableHint(): string;
526
+ measureHeading(root: string, prompts: number, level: string): string;
527
+ measureTotals(before: string, saved: string, floor: string): string;
528
+ measureRow(id: string, marginal: string, alone: string, prompts: number): string;
529
+ measureOverlap(sumOfAlone: string, saved: string): string;
530
+ measureRedundant(ids: string): string;
531
+ measureInert(ids: string): string;
532
+ measureFiredWithoutSaving(ids: string): string;
533
+ measureBand(source: string): string;
519
534
  };
520
535
 
521
536
  /** `trazum rank` — which prompts to fix first, and why. */
package/src/index.ts CHANGED
@@ -71,6 +71,7 @@ import {
71
71
  fleetRollup,
72
72
  rollUp,
73
73
  heartbeats,
74
+ ruleYield,
74
75
  labelCoverage,
75
76
  measuredUsage,
76
77
  gateMargin,
@@ -94,6 +95,7 @@ import {
94
95
  optimize,
95
96
  parseBaseline,
96
97
  PHRASE_LANGUAGES,
98
+ languagesWithStanding,
97
99
  plannedCalls,
98
100
  PRICING_LAST_REVIEWED,
99
101
  profilePrompt,
@@ -299,6 +301,7 @@ const VALUE_FLAGS = new Set([
299
301
  'max-growth-usd',
300
302
  'max-cache-loss-usd',
301
303
  'max-stale-hours',
304
+ 'measure',
302
305
  'max-day-usd',
303
306
  'max-session-usd',
304
307
  'csv-out',
@@ -608,7 +611,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
608
611
  commitment: ['floor', 'discount', 'months', 'pricing', 'pricing-live'],
609
612
  report: ['year', 'json', 'pricing', 'pricing-live'],
610
613
  where: [],
611
- rules: [],
614
+ rules: ['measure', 'level', 'json'],
612
615
  blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
613
616
  doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
614
617
  };
@@ -909,6 +912,14 @@ function printReport(
909
912
  // Which languages the dictionaries actually cover. Only here, because this
910
913
  // is the one branch where silence reads as "your prompt is already clean".
911
914
  console.log(c.dim(t.report.dictionaryCoverage(languageNames(PHRASE_LANGUAGES, t))));
915
+ // And which of them nobody here reads. Naming the seven and stopping there
916
+ // reads as seven equal dictionaries; five of them have never been agreed by
917
+ // a speaker of the language, and a reader deciding whether to trust an
918
+ // empty result deserves the difference.
919
+ const unreviewed = languagesWithStanding(PHRASE_LANGUAGES, 'unreviewed');
920
+ if (unreviewed.length > 0) {
921
+ console.log(c.dim(t.report.dictionaryUnreviewed(languageNames(unreviewed, t))));
922
+ }
912
923
  }
913
924
 
914
925
  if (result.llm) {
@@ -3626,6 +3637,104 @@ function commandRules(t: CliMessages, locale: Locale): void {
3626
3637
  }
3627
3638
  }
3628
3639
 
3640
+ /**
3641
+ * `trazum rules --measure <dir> [--level <safe|aggressive>] [--json]` — and
3642
+ * what does each one actually recover here?
3643
+ *
3644
+ * The README says plainly that the deterministic rules recover about one per
3645
+ * cent, and that is the fair complaint about this tool. It is also an
3646
+ * **aggregate**, and an aggregate is where a distribution goes to hide: the
3647
+ * same one per cent is consistent with every rule pulling its weight and with
3648
+ * two rules doing all of it beside five that have never changed a byte.
3649
+ *
3650
+ * This measures which, over prompts the reader actually has, by running the
3651
+ * optimiser with each rule alone and then with each rule removed. **Both
3652
+ * figures are printed and neither is reconciled into the other**, because
3653
+ * where two rules find the same tokens they diverge, and either one on its own
3654
+ * would be wrong in a different direction.
3655
+ *
3656
+ * **The floor is separated out.** The optimiser normalises whitespace whether
3657
+ * or not any rule is enabled, and crediting that to the rules is how a
3658
+ * headline percentage survives on a corpus where the rules recover nothing.
3659
+ * The first version of the module did exactly that and reported every rule as
3660
+ * redundant.
3661
+ *
3662
+ * "Inert" is always said **about the corpus**. A rule that finds nothing in
3663
+ * these files has not been shown to find nothing anywhere, and the difference
3664
+ * is the whole distance between "delete this rule" and "measure it on
3665
+ * something else".
3666
+ */
3667
+ async function commandRulesMeasure(
3668
+ args: Args,
3669
+ config: TrazumConfig,
3670
+ pricing: PricingCatalogue,
3671
+ t: CliMessages,
3672
+ ): Promise<void> {
3673
+ const root = stringFlag(args, 'measure')!;
3674
+ const level = (stringFlag(args, 'level') ?? 'safe') as RuleLevel;
3675
+ if (level !== 'safe' && level !== 'aggressive') throw new Error(t.errors.badLevel(level));
3676
+
3677
+ const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
3678
+ const { files, truncated } = await walkPrompts(root, { extensions });
3679
+ if (files.length === 0) throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
3680
+
3681
+ const prompts: string[] = [];
3682
+ for (const file of files) {
3683
+ // `walkPrompts` returns names relative to the root it was given, so the
3684
+ // root has to go back on. Reading them bare only worked from a run whose
3685
+ // cwd happened to be the root, which is the one case a first probe uses.
3686
+ const raw = await readFile(join(root, file), 'utf8');
3687
+ // A marked source file contributes its marked prompts; anything else
3688
+ // contributes itself. The same rule `check` and `doctor` already follow,
3689
+ // so the three commands measure the same text.
3690
+ if (hasMarker(raw)) prompts.push(...extractPrompts(raw).prompts.map((p) => p.text));
3691
+ else prompts.push(raw);
3692
+ }
3693
+
3694
+ const report = ruleYield(prompts, RULES.map((rule) => rule.id), { level, pricing });
3695
+
3696
+ if (boolFlag(args, 'json')) {
3697
+ console.log(JSON.stringify({ ...report, root, files: files.length, truncated }, null, 2));
3698
+ return;
3699
+ }
3700
+
3701
+ const n = (value: number): string => value.toLocaleString(t.numberLocale);
3702
+
3703
+ console.log();
3704
+ console.log(c.bold(t.rules.measureHeading(root, report.prompts, level)));
3705
+ console.log(` ${c.dim(t.rules.measureTotals(n(report.tokensBefore), n(report.tokensSaved), n(report.floor)))}`);
3706
+ console.log();
3707
+
3708
+ for (const rule of report.rules) {
3709
+ const line = t.rules.measureRow(rule.id, n(rule.marginal), n(rule.alone), rule.prompts);
3710
+ if (rule.marginal > 0) console.log(` ${line}`);
3711
+ else console.log(` ${c.dim(line)}`);
3712
+ }
3713
+
3714
+ console.log();
3715
+ // The overlap, stated as the gap rather than resolved into a total: a single
3716
+ // figure here is the one number that cannot be true.
3717
+ if (report.sumOfAlone !== report.tokensSaved) {
3718
+ console.log(` ${c.dim(wrap(t.rules.measureOverlap(n(report.sumOfAlone), n(report.tokensSaved)), 74, ' '))}`);
3719
+ }
3720
+ if (report.redundantHere.length > 0) {
3721
+ console.log(` ${c.dim(wrap(t.rules.measureRedundant(report.redundantHere.join(', ')), 74, ' '))}`);
3722
+ }
3723
+ // Fired-and-saved-nothing before never-fired: the first is a finding about
3724
+ // the rule, the second only about the corpus, and a reader who meets them the
3725
+ // other way round reads both as the same shrug.
3726
+ if (report.firedWithoutSavingHere.length > 0) {
3727
+ console.log(
3728
+ ` ${c.yellow(wrap(t.rules.measureFiredWithoutSaving(report.firedWithoutSavingHere.join(', ')), 74, ' '))}`,
3729
+ );
3730
+ }
3731
+ if (report.inertHere.length > 0) {
3732
+ console.log(` ${c.dim(wrap(t.rules.measureInert(report.inertHere.join(', ')), 74, ' '))}`);
3733
+ }
3734
+ console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
3735
+ console.log();
3736
+ }
3737
+
3629
3738
  async function readInput(source: string | undefined, t: CliMessages): Promise<string> {
3630
3739
  if (!source) {
3631
3740
  throw new Error(t.errors.missingInputFile());
@@ -9864,7 +9973,8 @@ async function main(): Promise<void> {
9864
9973
  await commandWhere(args, config, pricing, t);
9865
9974
  break;
9866
9975
  case 'rules':
9867
- commandRules(t, locale);
9976
+ if (stringFlag(args, 'measure') !== undefined) await commandRulesMeasure(args, config, pricing, t);
9977
+ else commandRules(t, locale);
9868
9978
  break;
9869
9979
  case 'doctor':
9870
9980
  await commandDoctor(args, config, pricing, t, locale);