@trazum/cli 1.50.3 → 1.50.5

package/dist/index.js

@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve as resolvePath } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, outcomeReport, rankPerOutcome, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -3587,6 +3587,7 @@ async function commandProfile(args, config, configDir, pricing, t) {
         can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
         can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
         can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
+        can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
         can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
         // "No cache traffic" is not a missing field: the split can only exist on
         // records that wrote, and a log that never wrote has nothing to record.
@@ -5209,6 +5210,117 @@ async function commandProfile(args, config, configDir, pricing, t) {
      * section at all, because a paragraph of things that are fine is the
      * paragraph readers learn to skip.
      */
+    /**
+     * Outcomes — the counterpart, where somebody recorded one.
+     *
+     * Printed above the coverage section rather than below it, because when this
+     * section is present it is the most valuable thing on the page: every other
+     * figure in this report is a cost, and this is the only one that says what
+     * the money bought.
+     *
+     * Silent when nothing recorded an outcome. The coverage section below
+     * already names the missing field and what it would unlock, and printing an
+     * empty Outcomes heading above it would be the same sentence twice.
+     */
+    {
+        const outcomes = outcomeReport(report.outcomeTally, config.outcomes ?? null);
+        if (outcomes.coverage.recorded > 0) {
+            console.log();
+            console.log(c.bold(t.profile.outcomeHeading()));
+            const col = t.profile.outcomeColumns;
+            const rows = [...outcomes.slices, ...outcomes.undeclared].map((slice) => ({
+                value: slice.value,
+                verdict: slice.verdict === 'success'
+                    ? t.profile.verdictSuccess()
+                    : slice.verdict === 'undeclared'
+                        ? t.profile.verdictUndeclared()
+                        : t.profile.verdictOther(),
+                calls: n(slice.calls),
+                spend: formatUsd(slice.usd),
+            }));
+            const w = {
+                value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
+                verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
+                calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
+                spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
+            };
+            console.log(c.dim(`  ${col.outcome.padEnd(w.value)}  ${''.padEnd(w.verdict)}  ` +
+                `${col.calls.padStart(w.calls)}  ${col.spend.padStart(w.spend)}`));
+            for (const row of rows) {
+                const tint = row.verdict === t.profile.verdictUndeclared()
+                    ? c.yellow
+                    : row.verdict === t.profile.verdictSuccess()
+                        ? c.green
+                        : c.dim;
+                console.log(`  ${row.value.padEnd(w.value)}  ${tint(row.verdict.padEnd(w.verdict))}  ` +
+                    `${row.calls.padStart(w.calls)}  ${row.spend.padStart(w.spend)}`);
+            }
+            console.log();
+            if (outcomes.successShareOfRecordedUsd !== null) {
+                const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
+                console.log(`  ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, '    ')}`);
+            }
+            else if (outcomes.noRate !== null) {
+                console.log(`  ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, '    '))}`);
+            }
+            // What the rate does not cover, every time it is printed. A rate over a
+            // twelfth of the bill is a rate about a twelfth of the bill.
+            if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, '    ')}`);
+            }
+            /**
+             * What an outcome costs, per workload — the finding a total cannot make.
+             *
+             * Printed only when at least one workload has enough recorded outcomes
+             * to say something, and every row that cannot state a figure says which
+             * of the five reasons applies rather than showing a blank.
+             */
+            const ranking = rankPerOutcome(report.outcomeTallyByLabel.map((entry) => ({
+                key: entry.label,
+                calls: entry.calls,
+                totalUsd: entry.totalUsd,
+                tally: entry.tally,
+            })), config.outcomes ?? null);
+            if (ranking.byCall.length > 0) {
+                console.log();
+                console.log(c.bold(t.profile.perOutcomeHeading()));
+                const pcol = t.profile.perOutcomeColumns;
+                const prows = ranking.byCall.map((slice) => ({
+                    key: slice.key,
+                    perCall: formatUsd(slice.usdPerCall),
+                    perOutcome: slice.per.usdPerSuccess !== null
+                        ? formatUsd(slice.per.usdPerSuccess)
+                        : t.profile.perOutcomeWithheld(slice.per.withheld ?? 'no-vocabulary', n(slice.per.successes), pct(slice.per.coverage)),
+                    recorded: pct(slice.per.coverage),
+                }));
+                const pw = {
+                    key: Math.max(...prows.map((r) => r.key.length), pcol.workload.length),
+                    perCall: Math.max(...prows.map((r) => r.perCall.length), pcol.perCall.length),
+                    perOutcome: Math.max(...prows.map((r) => r.perOutcome.length), pcol.perOutcome.length),
+                    recorded: Math.max(...prows.map((r) => r.recorded.length), pcol.recorded.length),
+                };
+                console.log(c.dim(`  ${pcol.workload.padEnd(pw.key)}  ${pcol.perCall.padStart(pw.perCall)}  ` +
+                    `${pcol.perOutcome.padStart(pw.perOutcome)}  ${pcol.recorded.padStart(pw.recorded)}`));
+                for (const row of prows) {
+                    console.log(`  ${row.key.padEnd(pw.key)}  ${row.perCall.padStart(pw.perCall)}  ` +
+                        `${row.perOutcome.padStart(pw.perOutcome)}  ${c.dim(row.recorded.padStart(pw.recorded))}`);
+                }
+                console.log();
+                console.log(`  ${c.dim(wrap(t.profile.perOutcomeNumerator(), 74, '    '))}`);
+                // The disagreement between the two orders, which is itself the finding.
+                if (ranking.disagreements.length > 0) {
+                    console.log();
+                    console.log(`  ${c.dim(wrap(t.profile.perOutcomeBothOrders(), 74, '    '))}`);
+                    for (const d of ranking.disagreements) {
+                        console.log(`  ${c.yellow('→')} ${wrap(t.profile.perOutcomeDisagreement(d.slice.key, n(d.callRank + 1), n(d.outcomeRank + 1)), 74, '    ')}`);
+                    }
+                }
+            }
+            if (outcomes.undeclared.length > 0) {
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, '    ')}`);
+            }
+        }
+    }
     const coverage = report.fieldCoverage;
     if (coverage.parsed > 0) {
         const missing = [];
@@ -5219,6 +5331,14 @@ async function commandProfile(args, config, configDir, pricing, t) {
         if (coverage.session < coverage.parsed) {
             missing.push(t.profile.needsSession(partial(coverage.session)));
         }
+        /**
+         * Listed first among the missing when it is missing entirely, because it
+         * is the one field that changes what every other figure here *means*. The
+         * rest sharpen a cost; this one gives it a counterpart.
+         */
+        if (coverage.outcome < coverage.parsed) {
+            missing.push(t.profile.needsOutcome(partial(coverage.outcome)));
+        }
         if (coverage.ts < coverage.parsed) {
             missing.push(t.profile.needsTs(partial(coverage.ts)));
         }