@trazum/cli 1.50.6 → 1.50.7

package/dist/index.js

@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve as resolvePath } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, runExperiment, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -32,6 +32,9 @@ const c = {
     cyan: (s) => (useColor ? `\u001b[36m${s}\u001b[39m` : s),
 };
 const VALUE_FLAGS = new Set([
+    'a',
+    'b',
+    'min-outcomes',
     'against',
     'contract',
     'on-cannot-tell',
@@ -320,6 +323,7 @@ const COMMAND_FLAGS = {
     feedback: [],
     gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
     ladder: ['pricing', 'pricing-live', 'since', 'until', 'label'],
+    experiment: ['a', 'b', 'min-outcomes', 'pricing', 'pricing-live'],
     where: [],
     rules: [],
     blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -1769,6 +1773,87 @@ async function commandLadder(args, config, pricing, t) {
     if (anyProblem)
         process.exitCode = 1;
 }
+/**
+ * `trazum experiment <log> --a <label> --b <label> --min-outcomes <n>`
+ *
+ * Two arms on real traffic, judged on recorded outcomes and cost together.
+ *
+ * `--min-outcomes` is required and that is the entire point of it. A stopping
+ * rule declared after looking at the numbers is not a stopping rule, and
+ * nothing here can stop somebody reading a result early — what it can do is
+ * make the early read **visible to whoever reads the result later**, which is
+ * the part that survives the afternoon.
+ */
+async function commandExperiment(args, config, pricing, t) {
+    const path = args.positional[0];
+    if (path === undefined)
+        throw new Error(t.errors.missingInputFile());
+    const aName = stringFlag(args, 'a');
+    const bName = stringFlag(args, 'b');
+    if (aName === undefined || bName === undefined)
+        throw new Error(t.experiment.needsTwo());
+    const minRaw = stringFlag(args, 'min-outcomes');
+    const minOutcomesPerArm = minRaw === undefined ? Number.NaN : Number(minRaw);
+    if (!Number.isInteger(minOutcomesPerArm) || minOutcomesPerArm < 1) {
+        throw new Error(t.experiment.needsRule());
+    }
+    const report = profileUsage(await readUsageLog(path, t), { catalogue: pricing });
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    const pct = (value) => `${(value * 100).toFixed(1)}%`;
+    const armOf = (label) => {
+        const slice = report.outcomeTallyByLabel.find((entry) => entry.label === label);
+        return {
+            name: label,
+            totalUsd: slice?.totalUsd ?? 0,
+            tally: slice?.tally ?? { byValue: [], recorded: 0, parsed: 0, unrecordedUsd: 0 },
+        };
+    };
+    const result = runExperiment({ arms: [aName, bName], minOutcomesPerArm }, { a: armOf(aName), b: armOf(bName) }, config.outcomes ?? null);
+    console.log();
+    console.log(c.bold(t.experiment.heading(aName, bName)));
+    console.log();
+    for (const side of [result.a, result.b]) {
+        console.log(`  ${t.experiment.arm(side.name, side.rate === null ? '—' : pct(side.rate), n(side.successes), n(side.recorded), side.interval === null ? '—' : `[${pct(side.interval.low)}, ${pct(side.interval.high)}]`)}`);
+    }
+    console.log();
+    if (result.separation === 'not-separable') {
+        console.log(`  ${c.dim('·')} ${wrap(t.experiment.notSeparable(result.notSeparable ?? '', result.outcomesNeededPerArm === null ? '—' : n(result.outcomesNeededPerArm)), 74, '    ')}`);
+    }
+    else {
+        const winner = result.separation === 'a-wins' ? result.a.name : result.b.name;
+        const d = result.difference;
+        // Reported as a magnitude: the sign is carried by which arm is named, and
+        // printing "-30.0% to -18.0%" beside "b wins" is two ways of saying the
+        // same thing that a reader has to reconcile.
+        const lo = Math.min(Math.abs(d.low), Math.abs(d.high));
+        const hi = Math.max(Math.abs(d.low), Math.abs(d.high));
+        console.log(`  ${c.green('✓')} ${wrap(t.experiment.wins(winner, pct(lo), pct(hi)), 74, '    ')}`);
+    }
+    /**
+     * The peek line, printed **whether or not** the arms separated.
+     *
+     * A separable result read too early is still separable and still read too
+     * early. Collapsing the two would hide one of the facts, and it is always
+     * the inconvenient one that goes.
+     */
+    console.log();
+    if (result.stopping.honoured) {
+        console.log(`  ${c.dim(wrap(t.experiment.honoured(n(result.stopping.declared)), 74, '    '))}`);
+    }
+    else {
+        const short = result.stopping.short === result.a.name ? result.a : result.b;
+        console.log(`  ${c.yellow('!')} ${wrap(t.experiment.peeked(short.name, n(result.stopping.declared), n(short.recorded)), 74, '    ')}`);
+    }
+    if (result.marginal !== null) {
+        console.log();
+        console.log(`  ${wrap(result.marginal.usdPerExtraSuccess !== null
+            ? t.experiment.marginalDearer(result.marginal.better, formatUsd(result.marginal.usdPerExtraSuccess))
+            : t.experiment.marginalCheaper(result.marginal.better), 74, '    ')}`);
+    }
+    console.log();
+    console.log(`  ${c.dim(wrap(t.experiment.neverPromotes(), 74, '    '))}`);
+    console.log();
+}
 function commandModels(t, pricing) {
     const n = (value) => value.toLocaleString(t.numberLocale);
     const col = t.models.columns;
@@ -6816,6 +6901,9 @@ async function main() {
         case 'models':
             commandModels(t, pricing);
             break;
+        case 'experiment':
+            await commandExperiment(args, config, pricing, t);
+            break;
         case 'ladder':
             await commandLadder(args, config, pricing, t);
             break;