@trazum/cli 1.50.7 → 1.50.8

package/dist/index.js

@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve as resolvePath } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, runExperiment, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, runExperiment, qualityGate, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -33,6 +33,7 @@ const c = {
 };
 const VALUE_FLAGS = new Set([
     'a',
+    'at',
     'b',
     'min-outcomes',
     'against',
@@ -324,6 +325,7 @@ const COMMAND_FLAGS = {
     gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
     ladder: ['pricing', 'pricing-live', 'since', 'until', 'label'],
     experiment: ['a', 'b', 'min-outcomes', 'pricing', 'pricing-live'],
+    quality: ['label', 'at', 'gate', 'pricing', 'pricing-live'],
     where: [],
     rules: [],
     blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -1854,6 +1856,116 @@ async function commandExperiment(args, config, pricing, t) {
     console.log(`  ${c.dim(wrap(t.experiment.neverPromotes(), 74, '    '))}`);
     console.log();
 }
+/**
+ * `trazum quality <log> --label <name> --at <iso> [--gate]`
+ *
+ * The failure that actually matters: a prompt edit that quietly made the
+ * product worse. CI has been able to fail a build for tokens since 1.4 and for
+ * dollars since 1.21, and this has never been gateable — so every saving this
+ * tool has ever recommended went into a repository with its most important
+ * consequence unmeasured.
+ *
+ * **Named `quality` rather than `check --against-outcomes`, which is what the
+ * plan called for.** `check` reads *prompt files* and gates on tokens; it has
+ * never opened a usage log, and a command that takes either a prompt or a log
+ * depending on a flag is two commands wearing one name. The split-by-time this
+ * needs is also not a `check` idea — there is nothing in a prompt file with a
+ * timestamp on it.
+ */
+async function commandQuality(args, config, pricing, t) {
+    const path = args.positional[0];
+    if (path === undefined)
+        throw new Error(t.errors.missingInputFile());
+    const label = stringFlag(args, 'label');
+    if (label === undefined)
+        throw new Error(t.quality.needsLabel());
+    const atRaw = stringFlag(args, 'at');
+    const atMs = atRaw === undefined ? Number.NaN : Date.parse(atRaw);
+    if (!Number.isFinite(atMs))
+        throw new Error(t.quality.needsAt());
+    /**
+     * Two profiles over the same file, split at the boundary — rather than one
+     * profile the caller has to slice.
+     *
+     * The alternative is asking somebody for two logs, which invites the mistake
+     * this whole module exists to avoid: two files gathered under conditions
+     * nobody wrote down.
+     */
+    const raw = await readUsageLog(path, t);
+    const sideOf = (since, until) => {
+        const report = profileUsage(raw, { catalogue: pricing, label, sinceMs: since, untilMs: until });
+        const slice = report.outcomeTallyByLabel.find((entry) => entry.label === label);
+        return {
+            arm: {
+                name: label,
+                totalUsd: report.total.totalUsd,
+                tally: slice?.tally ?? { byValue: [], recorded: 0, parsed: 0, unrecordedUsd: 0 },
+            },
+            calls: report.total.calls,
+            usdByModel: report.byModel.map((entry) => ({ model: entry.model, usd: entry.breakdown.totalUsd })),
+        };
+    };
+    const result = qualityGate(sideOf(undefined, atMs), sideOf(atMs, undefined), config.outcomes ?? null);
+    const pct = (value) => `${(value * 100).toFixed(1)}%`;
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    console.log();
+    console.log(c.bold(t.quality.heading(label)));
+    console.log(`  ${c.dim(wrap(t.quality.notRandomised(), 74, '  '))}`);
+    console.log();
+    console.log(`  ${t.quality.sides(result.before.rate === null ? '—' : pct(result.before.rate), result.after.rate === null ? '—' : pct(result.after.rate), n(result.outcomes.before), n(result.outcomes.after))}`);
+    console.log();
+    if (result.verdict === 'dropped') {
+        const cost = result.cost === null
+            ? ''
+            : result.cost.deltaUsdPerCall < 0
+                ? `saves ${formatUsd(-result.cost.deltaUsdPerCall)} a call`
+                : `costs ${formatUsd(result.cost.deltaUsdPerCall)} a call more`;
+        console.log(`  ${c.red('✗')} ${wrap(t.quality.dropped(pct(result.before.rate ?? 0), pct(result.after.rate ?? 0), n(result.outcomes.before + result.outcomes.after), cost), 74, '    ')}`);
+    }
+    else if (result.verdict === 'held') {
+        console.log(`  ${c.green('✓')} ${wrap(t.quality.held(pct(result.before.rate ?? 0), pct(result.after.rate ?? 0), n(result.outcomes.before + result.outcomes.after)), 74, '    ')}`);
+    }
+    else {
+        const need = result.unknown === 'too-few-before' ? n(result.outcomes.before) : n(result.outcomes.after);
+        console.log(`  ${c.yellow('?')} ${wrap(t.quality.cannotTell(result.unknown ?? '', need), 74, '    ')}`);
+    }
+    /**
+     * Confounders print on **every** verdict, not only on `cannot-tell`.
+     *
+     * A rate that held while the model changed underneath is not evidence that
+     * the prompt is fine either, and hiding the confounder on a green result is
+     * how a gate teaches people to trust it in exactly the case it should not be
+     * trusted.
+     */
+    if (result.confounders.length > 0) {
+        console.log();
+        console.log(`  ${c.bold(t.quality.confoundersHeading())}`);
+        for (const confounder of result.confounders) {
+            const detail = confounder.kind === 'model-mix-moved'
+                ? `${pct(confounder.drift)} (${confounder.model})`
+                : confounder.kind === 'volume-moved'
+                    ? `${n(confounder.beforeCalls)} → ${n(confounder.afterCalls)} calls`
+                    : `${pct(confounder.before)} → ${pct(confounder.after)}`;
+            console.log(`    ${c.yellow('!')} ${wrap(t.quality.confounder(confounder.kind, detail), 70, '      ')}`);
+        }
+    }
+    console.log();
+    console.log(`  ${c.dim(wrap(t.quality.cannotSee(), 74, '  '))}`);
+    if (boolFlag(args, 'gate')) {
+        console.log();
+        if (result.verdict === 'dropped') {
+            console.log(`  ${c.red(t.quality.gateFailed())}`);
+            process.exitCode = 1;
+        }
+        else if (result.verdict === 'cannot-tell') {
+            // Three outcomes, never two. `cannot tell` holds the claim open rather
+            // than exiting green, the posture `verify --gate` has had since 1.39.
+            console.log(`  ${c.yellow(t.quality.gateHeldOpen())}`);
+            process.exitCode = 2;
+        }
+    }
+    console.log();
+}
 function commandModels(t, pricing) {
     const n = (value) => value.toLocaleString(t.numberLocale);
     const col = t.models.columns;
@@ -6901,6 +7013,9 @@ async function main() {
         case 'models':
             commandModels(t, pricing);
             break;
+        case 'quality':
+            await commandQuality(args, config, pricing, t);
+            break;
         case 'experiment':
             await commandExperiment(args, config, pricing, t);
             break;