@trazum/cli 1.50.4 → 1.50.6

package/dist/index.js

@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve as resolvePath } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, outcomeReport, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, BREAK_EVEN_BAND, ladderPosition, validateLadder, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -319,6 +319,7 @@ const COMMAND_FLAGS = {
     conform: ['contract', 'json'],
     feedback: [],
     gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
+    ladder: ['pricing', 'pricing-live', 'since', 'until', 'label'],
     where: [],
     rules: [],
     blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -1665,6 +1666,109 @@ async function commandGateway(args, config, configDir, pricing, t) {
     console.log(`  ${c.dim(wrap(t.gateway.policy(policyFlag), 74, '    '))}`);
     console.log();
 }
+/**
+ * `trazum ladder <log>` — is the ladder saving money, or is it a bill?
+ *
+ * The one number this command exists to print is the **break-even escalation
+ * rate**. "We route to the cheap model first" describes a policy that saves
+ * money and a policy that costs money equally well; only the rate separates
+ * them, and nobody works it out in their head because the shape of the
+ * arithmetic is not obvious — an escalation pays twice, since the cheap
+ * attempt is not refunded.
+ */
+async function commandLadder(args, config, pricing, t) {
+    const path = args.positional[0];
+    if (path === undefined) {
+        throw new Error(t.errors.missingInputFile());
+    }
+    const report = profileUsage(await readUsageLog(path, t), { catalogue: pricing });
+    const ladders = config.ladders ?? {};
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    const pct = (value) => `${(value * 100).toFixed(1)}%`;
+    console.log();
+    console.log(c.bold(t.ladder.heading()));
+    if (Object.keys(ladders).length === 0) {
+        console.log(`  ${c.dim(wrap(t.ladder.noLadders(), 74, '  '))}`);
+        console.log();
+        return;
+    }
+    console.log(`  ${c.dim(wrap(t.ladder.theDoubleSpend(), 74, '  '))}`);
+    console.log();
+    const vocabulary = config.outcomes ?? null;
+    let anyProblem = false;
+    for (const [label, policy] of Object.entries(ladders)) {
+        /**
+         * Validated before it is measured, and loudly.
+         *
+         * A ladder that escalates on a value declared a *success* pays twice for
+         * work that already worked, on every call, while looking exactly like a
+         * cost-saving measure in the config. Printing its measured position first
+         * would bury that under a number.
+         */
+        const problems = validateLadder(policy, vocabulary, pricing);
+        if (problems.length > 0) {
+            anyProblem = true;
+            console.log(`  ${c.red('✗')} ${c.bold(t.ladder.problemsHeading(label))}`);
+            for (const problem of problems) {
+                const detail = 'value' in problem
+                    ? problem.value
+                    : 'model' in problem
+                        ? problem.model
+                        : String(problem.tiers);
+                console.log(`      ${wrap(t.ladder.problem(problem.kind, detail), 70, '      ')}`);
+            }
+            console.log();
+            continue;
+        }
+        const slice = report.outcomeTallyByLabel.find((entry) => entry.label === label);
+        const breakdown = report.byLabel.find((entry) => entry.label === label);
+        /**
+         * The shape of the work comes from the measured calls, so the break-even
+         * rate is priced against what this workload actually sends rather than
+         * against a token count somebody guessed at.
+         */
+        const calls = breakdown?.breakdown.calls ?? 0;
+        const shape = breakdown === undefined || calls === 0
+            ? { inputTokens: 0, outputTokens: 0 }
+            : {
+                inputTokens: Math.round((breakdown.breakdown.inputTokens +
+                    breakdown.breakdown.cacheReadTokens +
+                    breakdown.breakdown.cacheWriteTokens) /
+                    calls),
+                outputTokens: Math.round(breakdown.breakdown.outputTokens / calls),
+            };
+        const empty = { byValue: [], recorded: 0, parsed: 0, unrecordedUsd: 0 };
+        const position = ladderPosition(policy, slice?.tally ?? empty, shape, vocabulary, pricing);
+        console.log(`  ${c.bold(t.ladder.workload(label))}  ${c.dim(policy.tiers.join(' → '))}`);
+        console.log(`    ${c.dim(t.ladder.arithmetic(formatUsd(position.arithmetic.cheapUsd), formatUsd(position.arithmetic.dearUsd), position.arithmetic.breakEvenRate === null ? '—' : pct(position.arithmetic.breakEvenRate)))}`);
+        if (position.verdict === 'cannot-tell') {
+            console.log(`    ${c.yellow('?')} ${wrap(t.ladder.cannotTell(position.unknown ?? '', n(position.calls)), 70, '      ')}`);
+        }
+        else {
+            console.log(`    ${t.ladder.measured(pct(position.measuredRate ?? 0), n(position.escalations), n(position.calls))}`);
+            const delta = formatUsd(Math.abs(position.deltaUsdPerCall ?? 0));
+            if (position.verdict === 'saving') {
+                console.log(`    ${c.green('✓')} ${wrap(t.ladder.saving(delta), 70, '      ')}`);
+            }
+            else if (position.verdict === 'costing') {
+                console.log(`    ${c.red('✗')} ${wrap(t.ladder.costing(delta), 70, '      ')}`);
+            }
+            else {
+                console.log(`    ${c.dim('·')} ${wrap(t.ladder.atBreakEven(pct(BREAK_EVEN_BAND)), 70, '      ')}`);
+            }
+        }
+        console.log();
+    }
+    console.log(`  ${c.dim(wrap(t.ladder.notExecuted(), 74, '  '))}`);
+    console.log();
+    /**
+     * A misconfigured ladder fails the command, because it is the one finding
+     * here that is wrong *now* rather than a measurement somebody should look
+     * at. Everything else exits 0: this is a survey, like `doctor`.
+     */
+    if (anyProblem)
+        process.exitCode = 1;
+}
 function commandModels(t, pricing) {
     const n = (value) => value.toLocaleString(t.numberLocale);
     const col = t.models.columns;
@@ -5268,6 +5372,54 @@ async function commandProfile(args, config, configDir, pricing, t) {
             if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
                 console.log(`  ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, '    ')}`);
             }
+            /**
+             * What an outcome costs, per workload — the finding a total cannot make.
+             *
+             * Printed only when at least one workload has enough recorded outcomes
+             * to say something, and every row that cannot state a figure says which
+             * of the five reasons applies rather than showing a blank.
+             */
+            const ranking = rankPerOutcome(report.outcomeTallyByLabel.map((entry) => ({
+                key: entry.label,
+                calls: entry.calls,
+                totalUsd: entry.totalUsd,
+                tally: entry.tally,
+            })), config.outcomes ?? null);
+            if (ranking.byCall.length > 0) {
+                console.log();
+                console.log(c.bold(t.profile.perOutcomeHeading()));
+                const pcol = t.profile.perOutcomeColumns;
+                const prows = ranking.byCall.map((slice) => ({
+                    key: slice.key,
+                    perCall: formatUsd(slice.usdPerCall),
+                    perOutcome: slice.per.usdPerSuccess !== null
+                        ? formatUsd(slice.per.usdPerSuccess)
+                        : t.profile.perOutcomeWithheld(slice.per.withheld ?? 'no-vocabulary', n(slice.per.successes), pct(slice.per.coverage)),
+                    recorded: pct(slice.per.coverage),
+                }));
+                const pw = {
+                    key: Math.max(...prows.map((r) => r.key.length), pcol.workload.length),
+                    perCall: Math.max(...prows.map((r) => r.perCall.length), pcol.perCall.length),
+                    perOutcome: Math.max(...prows.map((r) => r.perOutcome.length), pcol.perOutcome.length),
+                    recorded: Math.max(...prows.map((r) => r.recorded.length), pcol.recorded.length),
+                };
+                console.log(c.dim(`  ${pcol.workload.padEnd(pw.key)}  ${pcol.perCall.padStart(pw.perCall)}  ` +
+                    `${pcol.perOutcome.padStart(pw.perOutcome)}  ${pcol.recorded.padStart(pw.recorded)}`));
+                for (const row of prows) {
+                    console.log(`  ${row.key.padEnd(pw.key)}  ${row.perCall.padStart(pw.perCall)}  ` +
+                        `${row.perOutcome.padStart(pw.perOutcome)}  ${c.dim(row.recorded.padStart(pw.recorded))}`);
+                }
+                console.log();
+                console.log(`  ${c.dim(wrap(t.profile.perOutcomeNumerator(), 74, '    '))}`);
+                // The disagreement between the two orders, which is itself the finding.
+                if (ranking.disagreements.length > 0) {
+                    console.log();
+                    console.log(`  ${c.dim(wrap(t.profile.perOutcomeBothOrders(), 74, '    '))}`);
+                    for (const d of ranking.disagreements) {
+                        console.log(`  ${c.yellow('→')} ${wrap(t.profile.perOutcomeDisagreement(d.slice.key, n(d.callRank + 1), n(d.outcomeRank + 1)), 74, '    ')}`);
+                    }
+                }
+            }
             if (outcomes.undeclared.length > 0) {
                 console.log(`  ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, '    ')}`);
             }
@@ -6664,6 +6816,9 @@ async function main() {
         case 'models':
             commandModels(t, pricing);
             break;
+        case 'ladder':
+            await commandLadder(args, config, pricing, t);
+            break;
         case 'gateway':
             await commandGateway(args, config, configDir, pricing, t);
             break;