@trazum/cli 1.56.0 → 1.56.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join, resolve as resolvePath } from 'node:path';
5
5
  import { fileURLToPath } from 'node:url';
6
6
  import { gunzipSync } from 'node:zlib';
7
- import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
7
+ import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, languagesWithStanding, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
8
8
  import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
9
9
  import { dayOf, formatGap, median, spanDays } from './time.js';
10
10
  // Everything that reads the filesystem, on its own entry point so the web
@@ -70,6 +70,7 @@ const VALUE_FLAGS = new Set([
70
70
  'max-growth-usd',
71
71
  'max-cache-loss-usd',
72
72
  'max-stale-hours',
73
+ 'measure',
73
74
  'max-day-usd',
74
75
  'max-session-usd',
75
76
  'csv-out',
@@ -338,7 +339,7 @@ const COMMAND_FLAGS = {
338
339
  commitment: ['floor', 'discount', 'months', 'pricing', 'pricing-live'],
339
340
  report: ['year', 'json', 'pricing', 'pricing-live'],
340
341
  where: [],
341
- rules: [],
342
+ rules: ['measure', 'level', 'json'],
342
343
  blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
343
344
  doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
344
345
  };
@@ -595,6 +596,14 @@ measured = null) {
595
596
  // Which languages the dictionaries actually cover. Only here, because this
596
597
  // is the one branch where silence reads as "your prompt is already clean".
597
598
  console.log(c.dim(t.report.dictionaryCoverage(languageNames(PHRASE_LANGUAGES, t))));
599
+ // And which of them nobody here reads. Naming the seven and stopping there
600
+ // reads as seven equal dictionaries; five of them have never been agreed by
601
+ // a speaker of the language, and a reader deciding whether to trust an
602
+ // empty result deserves the difference.
603
+ const unreviewed = languagesWithStanding(PHRASE_LANGUAGES, 'unreviewed');
604
+ if (unreviewed.length > 0) {
605
+ console.log(c.dim(t.report.dictionaryUnreviewed(languageNames(unreviewed, t))));
606
+ }
598
607
  }
599
608
  if (result.llm) {
600
609
  console.log();
@@ -2781,6 +2790,94 @@ function commandRules(t, locale) {
2781
2790
  console.log();
2782
2791
  }
2783
2792
  }
2793
+ /**
2794
+ * `trazum rules --measure <dir> [--level <safe|aggressive>] [--json]` — and
2795
+ * what does each one actually recover here?
2796
+ *
2797
+ * The README says plainly that the deterministic rules recover about one per
2798
+ * cent, and that is the fair complaint about this tool. It is also an
2799
+ * **aggregate**, and an aggregate is where a distribution goes to hide: the
2800
+ * same one per cent is consistent with every rule pulling its weight and with
2801
+ * two rules doing all of it beside five that have never changed a byte.
2802
+ *
2803
+ * This measures which, over prompts the reader actually has, by running the
2804
+ * optimiser with each rule alone and then with each rule removed. **Both
2805
+ * figures are printed and neither is reconciled into the other**, because
2806
+ * where two rules find the same tokens they diverge, and either one on its own
2807
+ * would be wrong in a different direction.
2808
+ *
2809
+ * **The floor is separated out.** The optimiser normalises whitespace whether
2810
+ * or not any rule is enabled, and crediting that to the rules is how a
2811
+ * headline percentage survives on a corpus where the rules recover nothing.
2812
+ * The first version of the module did exactly that and reported every rule as
2813
+ * redundant.
2814
+ *
2815
+ * "Inert" is always said **about the corpus**. A rule that finds nothing in
2816
+ * these files has not been shown to find nothing anywhere, and the difference
2817
+ * is the whole distance between "delete this rule" and "measure it on
2818
+ * something else".
2819
+ */
2820
+ async function commandRulesMeasure(args, config, pricing, t) {
2821
+ const root = stringFlag(args, 'measure');
2822
+ const level = (stringFlag(args, 'level') ?? 'safe');
2823
+ if (level !== 'safe' && level !== 'aggressive')
2824
+ throw new Error(t.errors.badLevel(level));
2825
+ const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
2826
+ const { files, truncated } = await walkPrompts(root, { extensions });
2827
+ if (files.length === 0)
2828
+ throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
2829
+ const prompts = [];
2830
+ for (const file of files) {
2831
+ // `walkPrompts` returns names relative to the root it was given, so the
2832
+ // root has to go back on. Reading them bare only worked from a run whose
2833
+ // cwd happened to be the root, which is the one case a first probe uses.
2834
+ const raw = await readFile(join(root, file), 'utf8');
2835
+ // A marked source file contributes its marked prompts; anything else
2836
+ // contributes itself. The same rule `check` and `doctor` already follow,
2837
+ // so the three commands measure the same text.
2838
+ if (hasMarker(raw))
2839
+ prompts.push(...extractPrompts(raw).prompts.map((p) => p.text));
2840
+ else
2841
+ prompts.push(raw);
2842
+ }
2843
+ const report = ruleYield(prompts, RULES.map((rule) => rule.id), { level, pricing });
2844
+ if (boolFlag(args, 'json')) {
2845
+ console.log(JSON.stringify({ ...report, root, files: files.length, truncated }, null, 2));
2846
+ return;
2847
+ }
2848
+ const n = (value) => value.toLocaleString(t.numberLocale);
2849
+ console.log();
2850
+ console.log(c.bold(t.rules.measureHeading(root, report.prompts, level)));
2851
+ console.log(` ${c.dim(t.rules.measureTotals(n(report.tokensBefore), n(report.tokensSaved), n(report.floor)))}`);
2852
+ console.log();
2853
+ for (const rule of report.rules) {
2854
+ const line = t.rules.measureRow(rule.id, n(rule.marginal), n(rule.alone), rule.prompts);
2855
+ if (rule.marginal > 0)
2856
+ console.log(` ${line}`);
2857
+ else
2858
+ console.log(` ${c.dim(line)}`);
2859
+ }
2860
+ console.log();
2861
+ // The overlap, stated as the gap rather than resolved into a total: a single
2862
+ // figure here is the one number that cannot be true.
2863
+ if (report.sumOfAlone !== report.tokensSaved) {
2864
+ console.log(` ${c.dim(wrap(t.rules.measureOverlap(n(report.sumOfAlone), n(report.tokensSaved)), 74, ' '))}`);
2865
+ }
2866
+ if (report.redundantHere.length > 0) {
2867
+ console.log(` ${c.dim(wrap(t.rules.measureRedundant(report.redundantHere.join(', ')), 74, ' '))}`);
2868
+ }
2869
+ // Fired-and-saved-nothing before never-fired: the first is a finding about
2870
+ // the rule, the second only about the corpus, and a reader who meets them the
2871
+ // other way round reads both as the same shrug.
2872
+ if (report.firedWithoutSavingHere.length > 0) {
2873
+ console.log(` ${c.yellow(wrap(t.rules.measureFiredWithoutSaving(report.firedWithoutSavingHere.join(', ')), 74, ' '))}`);
2874
+ }
2875
+ if (report.inertHere.length > 0) {
2876
+ console.log(` ${c.dim(wrap(t.rules.measureInert(report.inertHere.join(', ')), 74, ' '))}`);
2877
+ }
2878
+ console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
2879
+ console.log();
2880
+ }
2784
2881
  async function readInput(source, t) {
2785
2882
  if (!source) {
2786
2883
  throw new Error(t.errors.missingInputFile());
@@ -7870,7 +7967,10 @@ async function main() {
7870
7967
  await commandWhere(args, config, pricing, t);
7871
7968
  break;
7872
7969
  case 'rules':
7873
- commandRules(t, locale);
7970
+ if (stringFlag(args, 'measure') !== undefined)
7971
+ await commandRulesMeasure(args, config, pricing, t);
7972
+ else
7973
+ commandRules(t, locale);
7874
7974
  break;
7875
7975
  case 'doctor':
7876
7976
  await commandDoctor(args, config, pricing, t, locale);