@trazum/cli 1.56.0 → 1.56.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +30 -1
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/es.d.ts.map +1 -1
- package/dist/i18n/es.js +31 -1
- package/dist/i18n/es.js.map +1 -1
- package/dist/i18n/types.d.ts +8 -0
- package/dist/i18n/types.d.ts.map +1 -1
- package/dist/index.js +95 -3
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/src/i18n/en.ts +38 -1
- package/src/i18n/es.ts +39 -1
- package/src/i18n/types.ts +8 -0
- package/src/index.ts +103 -2
package/dist/index.js
CHANGED
|
@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
|
|
|
4
4
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { gunzipSync } from 'node:zlib';
|
|
7
|
-
import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
7
|
+
import { CONTRACT_NAMES, applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, bandGoverns, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, allocate, annualRecord, replayCommitment, coversTheTerm, runExperiment, qualityGate, semanticPassCost, verifySemanticProposals, SEMANTIC_SYSTEM_PROMPT, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, rollUp, heartbeats, ruleYield, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
|
|
8
8
|
import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
|
|
9
9
|
import { dayOf, formatGap, median, spanDays } from './time.js';
|
|
10
10
|
// Everything that reads the filesystem, on its own entry point so the web
|
|
@@ -70,6 +70,7 @@ const VALUE_FLAGS = new Set([
|
|
|
70
70
|
'max-growth-usd',
|
|
71
71
|
'max-cache-loss-usd',
|
|
72
72
|
'max-stale-hours',
|
|
73
|
+
'measure',
|
|
73
74
|
'max-day-usd',
|
|
74
75
|
'max-session-usd',
|
|
75
76
|
'csv-out',
|
|
@@ -338,7 +339,7 @@ const COMMAND_FLAGS = {
|
|
|
338
339
|
commitment: ['floor', 'discount', 'months', 'pricing', 'pricing-live'],
|
|
339
340
|
report: ['year', 'json', 'pricing', 'pricing-live'],
|
|
340
341
|
where: [],
|
|
341
|
-
rules: [],
|
|
342
|
+
rules: ['measure', 'level', 'json'],
|
|
342
343
|
blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
|
|
343
344
|
doctor: ['level', 'model', 'calls', 'output-tokens', 'batch', 'disable', 'prompt', 'otlp-out'],
|
|
344
345
|
};
|
|
@@ -2781,6 +2782,94 @@ function commandRules(t, locale) {
|
|
|
2781
2782
|
console.log();
|
|
2782
2783
|
}
|
|
2783
2784
|
}
|
|
2785
|
+
/**
|
|
2786
|
+
* `trazum rules --measure <dir> [--level <safe|aggressive>] [--json]` — and
|
|
2787
|
+
* what does each one actually recover here?
|
|
2788
|
+
*
|
|
2789
|
+
* The README says plainly that the deterministic rules recover about one per
|
|
2790
|
+
* cent, and that is the fair complaint about this tool. It is also an
|
|
2791
|
+
* **aggregate**, and an aggregate is where a distribution goes to hide: the
|
|
2792
|
+
* same one per cent is consistent with every rule pulling its weight and with
|
|
2793
|
+
* two rules doing all of it beside five that have never changed a byte.
|
|
2794
|
+
*
|
|
2795
|
+
* This measures which, over prompts the reader actually has, by running the
|
|
2796
|
+
* optimiser with each rule alone and then with each rule removed. **Both
|
|
2797
|
+
* figures are printed and neither is reconciled into the other**, because
|
|
2798
|
+
* where two rules find the same tokens they diverge, and either one on its own
|
|
2799
|
+
* would be wrong in a different direction.
|
|
2800
|
+
*
|
|
2801
|
+
* **The floor is separated out.** The optimiser normalises whitespace whether
|
|
2802
|
+
* or not any rule is enabled, and crediting that to the rules is how a
|
|
2803
|
+
* headline percentage survives on a corpus where the rules recover nothing.
|
|
2804
|
+
* The first version of the module did exactly that and reported every rule as
|
|
2805
|
+
* redundant.
|
|
2806
|
+
*
|
|
2807
|
+
* "Inert" is always said **about the corpus**. A rule that finds nothing in
|
|
2808
|
+
* these files has not been shown to find nothing anywhere, and the difference
|
|
2809
|
+
* is the whole distance between "delete this rule" and "measure it on
|
|
2810
|
+
* something else".
|
|
2811
|
+
*/
|
|
2812
|
+
async function commandRulesMeasure(args, config, pricing, t) {
|
|
2813
|
+
const root = stringFlag(args, 'measure');
|
|
2814
|
+
const level = (stringFlag(args, 'level') ?? 'safe');
|
|
2815
|
+
if (level !== 'safe' && level !== 'aggressive')
|
|
2816
|
+
throw new Error(t.errors.badLevel(level));
|
|
2817
|
+
const extensions = config.extensions ?? [...DEFAULT_EXTENSIONS, ...SOURCE_EXTENSIONS];
|
|
2818
|
+
const { files, truncated } = await walkPrompts(root, { extensions });
|
|
2819
|
+
if (files.length === 0)
|
|
2820
|
+
throw new Error(t.errors.noPromptsFound(root, extensions.join(' ')));
|
|
2821
|
+
const prompts = [];
|
|
2822
|
+
for (const file of files) {
|
|
2823
|
+
// `walkPrompts` returns names relative to the root it was given, so the
|
|
2824
|
+
// root has to go back on. Reading them bare only worked from a run whose
|
|
2825
|
+
// cwd happened to be the root, which is the one case a first probe uses.
|
|
2826
|
+
const raw = await readFile(join(root, file), 'utf8');
|
|
2827
|
+
// A marked source file contributes its marked prompts; anything else
|
|
2828
|
+
// contributes itself. The same rule `check` and `doctor` already follow,
|
|
2829
|
+
// so the three commands measure the same text.
|
|
2830
|
+
if (hasMarker(raw))
|
|
2831
|
+
prompts.push(...extractPrompts(raw).prompts.map((p) => p.text));
|
|
2832
|
+
else
|
|
2833
|
+
prompts.push(raw);
|
|
2834
|
+
}
|
|
2835
|
+
const report = ruleYield(prompts, RULES.map((rule) => rule.id), { level, pricing });
|
|
2836
|
+
if (boolFlag(args, 'json')) {
|
|
2837
|
+
console.log(JSON.stringify({ ...report, root, files: files.length, truncated }, null, 2));
|
|
2838
|
+
return;
|
|
2839
|
+
}
|
|
2840
|
+
const n = (value) => value.toLocaleString(t.numberLocale);
|
|
2841
|
+
console.log();
|
|
2842
|
+
console.log(c.bold(t.rules.measureHeading(root, report.prompts, level)));
|
|
2843
|
+
console.log(` ${c.dim(t.rules.measureTotals(n(report.tokensBefore), n(report.tokensSaved), n(report.floor)))}`);
|
|
2844
|
+
console.log();
|
|
2845
|
+
for (const rule of report.rules) {
|
|
2846
|
+
const line = t.rules.measureRow(rule.id, n(rule.marginal), n(rule.alone), rule.prompts);
|
|
2847
|
+
if (rule.marginal > 0)
|
|
2848
|
+
console.log(` ${line}`);
|
|
2849
|
+
else
|
|
2850
|
+
console.log(` ${c.dim(line)}`);
|
|
2851
|
+
}
|
|
2852
|
+
console.log();
|
|
2853
|
+
// The overlap, stated as the gap rather than resolved into a total: a single
|
|
2854
|
+
// figure here is the one number that cannot be true.
|
|
2855
|
+
if (report.sumOfAlone !== report.tokensSaved) {
|
|
2856
|
+
console.log(` ${c.dim(wrap(t.rules.measureOverlap(n(report.sumOfAlone), n(report.tokensSaved)), 74, ' '))}`);
|
|
2857
|
+
}
|
|
2858
|
+
if (report.redundantHere.length > 0) {
|
|
2859
|
+
console.log(` ${c.dim(wrap(t.rules.measureRedundant(report.redundantHere.join(', ')), 74, ' '))}`);
|
|
2860
|
+
}
|
|
2861
|
+
// Fired-and-saved-nothing before never-fired: the first is a finding about
|
|
2862
|
+
// the rule, the second only about the corpus, and a reader who meets them the
|
|
2863
|
+
// other way round reads both as the same shrug.
|
|
2864
|
+
if (report.firedWithoutSavingHere.length > 0) {
|
|
2865
|
+
console.log(` ${c.yellow(wrap(t.rules.measureFiredWithoutSaving(report.firedWithoutSavingHere.join(', ')), 74, ' '))}`);
|
|
2866
|
+
}
|
|
2867
|
+
if (report.inertHere.length > 0) {
|
|
2868
|
+
console.log(` ${c.dim(wrap(t.rules.measureInert(report.inertHere.join(', ')), 74, ' '))}`);
|
|
2869
|
+
}
|
|
2870
|
+
console.log(` ${c.dim(wrap(t.rules.measureBand(report.tokenSource), 74, ' '))}`);
|
|
2871
|
+
console.log();
|
|
2872
|
+
}
|
|
2784
2873
|
async function readInput(source, t) {
|
|
2785
2874
|
if (!source) {
|
|
2786
2875
|
throw new Error(t.errors.missingInputFile());
|
|
@@ -7870,7 +7959,10 @@ async function main() {
|
|
|
7870
7959
|
await commandWhere(args, config, pricing, t);
|
|
7871
7960
|
break;
|
|
7872
7961
|
case 'rules':
|
|
7873
|
-
|
|
7962
|
+
if (stringFlag(args, 'measure') !== undefined)
|
|
7963
|
+
await commandRulesMeasure(args, config, pricing, t);
|
|
7964
|
+
else
|
|
7965
|
+
commandRules(t, locale);
|
|
7874
7966
|
break;
|
|
7875
7967
|
case 'doctor':
|
|
7876
7968
|
await commandDoctor(args, config, pricing, t, locale);
|