@trazum/cli 1.56.0 → 1.56.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join, resolve as resolvePath } from 'node:path';
5
5
  import { fileURLToPath } from 'node:url';
6
6
  import { gunzipSync } from 'node:zlib';
7
- import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
7
+ import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
8
8
  import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
9
9
  import { dayOf, formatGap, median, spanDays } from './time.js';
10
10
  // Everything that reads the filesystem, on its own entry point so the web
@@ -70,6 +70,7 @@ const VALUE_FLAGS = new Set([
70
70
  'max-growth-usd',
71
71
  'max-cache-loss-usd',
72
72
  'max-stale-hours',
73
+ 'measure',
73
74
  'max-day-usd',
74
75
  'max-session-usd',
75
76
  'csv-out',
@@ -338,7 +339,7 @@ const COMMAND_FLAGS = {
338
339
  commitment: ['floor', 'discount', 'months', 'pricing', 'pricing-live'],
339
340
  report: ['year', 'json', 'pricing', 'pricing-live'],
340
341
  where: [],
341
- rules: [],
342
+ rules: ['measure', 'level', 'json'],
342
343
  blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
343
344
  doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
344
345
  };
@@ -2781,6 +2782,94 @@ function commandRules(t, locale) {
2781
2782
  console.log();
2782
2783
  }
2783
2784
  }
2785
+ /**
2786
+ * `trazum rules --measure <dir> [--level <safe|aggressive>] [--json]` — and
2787
+ * what does each one actually recover here?
2788
+ *
2789
+ * The README says plainly that the deterministic rules recover about one per
2790
+ * cent, and that is the fair complaint about this tool. It is also an
2791
+ * **aggregate**, and an aggregate is where a distribution goes to hide: the
2792
+ * same one per cent is consistent with every rule pulling its weight and with
2793
+ * two rules doing all of it beside five that have never changed a byte.
2794
+ *
2795
+ * This measures which, over prompts the reader actually has, by running the
2796
+ * optimiser with each rule alone and then with each rule removed. **Both
2797
+ * figures are printed and neither is reconciled into the other**, because
2798
+ * where two rules find the same tokens they diverge, and either one on its own
2799
+ * would be wrong in a different direction.
2800
+ *
2801
+ * **The floor is separated out.** The optimiser normalises whitespace whether
2802
+ * or not any rule is enabled, and crediting that to the rules is how a
2803
+ * headline percentage survives on a corpus where the rules recover nothing.
2804
+ * The first version of the module did exactly that and reported every rule as
2805
+ * redundant.
2806
+ *
2807
+ * "Inert" is always said **about the corpus**. A rule that finds nothing in
2808
+ * these files has not been shown to find nothing anywhere, and the difference
2809
+ * is the whole distance between "delete this rule" and "measure it on
2810
+ * something else".
2811
+ */
2812
+ async function commandRulesMeasure(args, config, pricing, t) {
2813
+ const root = stringFlag(args, 'measure');
2814
+ const level = (stringFlag(args, 'level') ?? 'safe');
2815
+ if (level !== 'safe' && level !== 'aggressive')
2816
+ throw new Error(t.errors.badLevel(level));
2817
+ const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
2818
+ const { files, truncated } = await walkPrompts(root, { extensions });
2819
+ if (files.length === 0)
2820
+ throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
2821
+ const prompts = [];
2822
+ for (const file of files) {
2823
+ // `walkPrompts` returns names relative to the root it was given, so the
2824
+ // root has to go back on. Reading them bare only worked from a run whose
2825
+ // cwd happened to be the root, which is the one case a first probe uses.
2826
+ const raw = await readFile(join(root, file), 'utf8');
2827
+ // A marked source file contributes its marked prompts; anything else
2828
+ // contributes itself. The same rule `check` and `doctor` already follow,
2829
+ // so the three commands measure the same text.
2830
+ if (hasMarker(raw))
2831
+ prompts.push(...extractPrompts(raw).prompts.map((p) => p.text));
2832
+ else
2833
+ prompts.push(raw);
2834
+ }
2835
+ const report = ruleYield(prompts, RULES.map((rule) => rule.id), { level, pricing });
2836
+ if (boolFlag(args, 'json')) {
2837
+ console.log(JSON.stringify({ ...report, root, files: files.length, truncated }, null, 2));
2838
+ return;
2839
+ }
2840
+ const n = (value) => value.toLocaleString(t.numberLocale);
2841
+ console.log();
2842
+ console.log(c.bold(t.rules.measureHeading(root, report.prompts, level)));
2843
+ console.log(` ${c.dim(t.rules.measureTotals(n(report.tokensBefore), n(report.tokensSaved), n(report.floor)))}`);
2844
+ console.log();
2845
+ for (const rule of report.rules) {
2846
+ const line = t.rules.measureRow(rule.id, n(rule.marginal), n(rule.alone), rule.prompts);
2847
+ if (rule.marginal > 0)
2848
+ console.log(` ${line}`);
2849
+ else
2850
+ console.log(` ${c.dim(line)}`);
2851
+ }
2852
+ console.log();
2853
+ // The overlap, stated as the gap rather than resolved into a total: a single
2854
+ // figure here is the one number that cannot be true.
2855
+ if (report.sumOfAlone !== report.tokensSaved) {
2856
+ console.log(` ${c.dim(wrap(t.rules.measureOverlap(n(report.sumOfAlone), n(report.tokensSaved)), 74, ' '))}`);
2857
+ }
2858
+ if (report.redundantHere.length > 0) {
2859
+ console.log(` ${c.dim(wrap(t.rules.measureRedundant(report.redundantHere.join(', ')), 74, ' '))}`);
2860
+ }
2861
+ // Fired-and-saved-nothing before never-fired: the first is a finding about
2862
+ // the rule, the second only about the corpus, and a reader who meets them the
2863
+ // other way round reads both as the same shrug.
2864
+ if (report.firedWithoutSavingHere.length > 0) {
2865
+ console.log(` ${c.yellow(wrap(t.rules.measureFiredWithoutSaving(report.firedWithoutSavingHere.join(', ')), 74, ' '))}`);
2866
+ }
2867
+ if (report.inertHere.length > 0) {
2868
+ console.log(` ${c.dim(wrap(t.rules.measureInert(report.inertHere.join(', ')), 74, ' '))}`);
2869
+ }
2870
+ console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
2871
+ console.log();
2872
+ }
2784
2873
  async function readInput(source, t) {
2785
2874
  if (!source) {
2786
2875
  throw new Error(t.errors.missingInputFile());
@@ -7870,7 +7959,10 @@ async function main() {
7870
7959
  await commandWhere(args, config, pricing, t);
7871
7960
  break;
7872
7961
  case 'rules':
7873
- commandRules(t, locale);
7962
+ if (stringFlag(args, 'measure') !== undefined)
7963
+ await commandRulesMeasure(args, config, pricing, t);
7964
+ else
7965
+ commandRules(t, locale);
7874
7966
  break;
7875
7967
  case 'doctor':
7876
7968
  await commandDoctor(args, config, pricing, t, locale);