@trazum/cli 1.36.0 → 1.38.0

package/dist/index.js

@@ -2,7 +2,7 @@
 import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { join, resolve as resolvePath } from 'node:path';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, buildPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -26,6 +26,7 @@ const c = {
 const VALUE_FLAGS = new Set([
     'against',
     'from-log',
+    'min-usd',
     // `route` takes a path here, and the flag is deliberately not `--prompt`:
     // everywhere else in this tool `--prompt` names a marked prompt *inside* a
     // source file, and reusing it for a path would be a trap laid for the reader.
@@ -159,6 +160,12 @@ function levelFlag(args, config, t) {
  * model id. It beats the default because reading the code is better than
  * assuming, and loses to config because being told is better than reading.
  */
+/**
+ * The file names a usage log answers to, shared by every command that reads a
+ * directory of them. One list, because two commands disagreeing on what counts
+ * as a log would be the same directory billing differently by verb.
+ */
+const LOG_EXTENSIONS = ['.jsonl', '.ndjson', '.log', '.json'];
 /**
  * One usage log, gzip included, shared by every command that reads one.
  *
@@ -278,7 +285,8 @@ const COMMAND_FLAGS = {
     ],
     check: ['max-tokens', 'level', 'exact-tokens', 'markdown-out', 'baseline'],
     baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
-    profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary'],
+    profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
+    plan: ['json', 'out', 'markdown-out', 'min-usd', 'pricing', 'pricing-live'],
     route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
     eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
     prune: ['cases', 'concurrency', 'json', 'yes'],
@@ -1636,6 +1644,130 @@ function isoDate() {
  * metered API calls somebody was actually billed for — the bill exists wherever
  * Trazum happens to be running, so the host has no bearing on it.
  */
+/**
+ * `trazum plan <log>` — not a list of findings, a ranked plan of what to do.
+ *
+ * The composition (route and batch on one slice never summed) happens in
+ * core's `buildPlan`; this command owns the I/O and the rendering. The plan
+ * saves as a dated JSON file on request, which is what makes verifying it
+ * against a later log possible at all — a prediction nobody wrote down is a
+ * prediction nobody can be held to.
+ */
+async function commandPlan(args, pricing, t) {
+    const path = args.positional[0];
+    if (path === undefined)
+        throw new Error(t.plan.noTarget());
+    const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
+    const READABLE = [...LOG_EXTENSIONS, ...GZ];
+    const target = await stat(path).catch(() => null);
+    let files = [path];
+    if (target?.isDirectory()) {
+        const entries = await readdir(path, { withFileTypes: true });
+        files = entries
+            .filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
+            .map((entry) => join(path, entry.name))
+            .sort((a, b) => a.localeCompare(b));
+        if (files.length === 0)
+            throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
+    }
+    const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
+    const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
+    const report = profileUsage(raw, { catalogue: pricing });
+    if (report.total.calls === 0)
+        throw new Error(t.plan.nothingPriced());
+    const levers = billLevers(report, { catalogue: pricing });
+    const plan = buildPlan(report, levers, pricing.lastReviewed);
+    const minUsd = typeof args.flags.get('min-usd') === 'string' ? numberFlag(args, 'min-usd', 0, t) : 0;
+    const actions = plan.actions.filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) >= minUsd);
+    const filtered = plan.actions.length - actions.length;
+    const droppedUsd = plan.actions
+        .filter((a) => (a.savingUsd ?? a.stakeUsd ?? 0) < minUsd)
+        .reduce((sum, a) => sum + (a.savingUsd ?? a.stakeUsd ?? 0), 0);
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    /**
+     * The document's totals cover the actions the document holds — a filtered
+     * plan whose totals still counted the filtered actions would be a file
+     * that contradicts itself, and 1.39's verify would hold it to money it
+     * cannot see. What --min-usd dropped is stated with its worth, never
+     * silently.
+     */
+    const stamped = {
+        ...plan,
+        actions,
+        projectedSavingUsd: actions.reduce((sum, a) => sum + (a.savingUsd ?? 0), 0),
+        measuredStakeUsd: actions.reduce((sum, a) => sum + (a.stakeUsd ?? 0), 0),
+        createdAt: new Date().toISOString(),
+    };
+    const outPath = stringFlag(args, 'out');
+    if (outPath !== undefined) {
+        await writeFile(outPath, `${JSON.stringify(stamped, null, 2)}\n`);
+    }
+    await writeMarkdown(args, () => {
+        const lines = [];
+        lines.push(`## ${t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))}`);
+        lines.push('');
+        lines.push(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)));
+        if (plan.span === null) {
+            lines.push('');
+            lines.push(`_${t.plan.noClock()}_`);
+        }
+        for (const action of actions) {
+            const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
+            const money = action.savingUsd !== null
+                ? t.plan.projected(formatUsd(action.savingUsd))
+                : t.plan.staked(formatUsd(action.stakeUsd ?? 0));
+            lines.push('');
+            lines.push(`### ${t.plan.action(action.kind, name, action.model)} — ${money}`);
+            if (action.detail.routeTo !== undefined)
+                lines.push(`- ${t.plan.routeTo(action.detail.routeTo.displayName)}`);
+            for (const assumption of action.assumes)
+                lines.push(`- ${t.plan.assume(assumption)}`);
+            if (action.check !== null)
+                lines.push(`- ${t.plan.check(action.check)}`);
+        }
+        if (filtered > 0) {
+            lines.push('');
+            lines.push(`_${t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd))}_`);
+        }
+        lines.push('');
+        lines.push(`_${t.plan.footer()}_`);
+        return lines.join('\n');
+    });
+    if (boolFlag(args, 'json')) {
+        console.log(JSON.stringify(stamped, null, 2));
+        return;
+    }
+    console.log(c.bold(t.plan.heading(n(actions.length), formatUsd(plan.totalUsd))));
+    console.log(`  ${wrap(t.plan.totals(formatUsd(stamped.projectedSavingUsd), formatUsd(stamped.measuredStakeUsd)), 74, '  ')}`);
+    if (plan.span === null) {
+        console.log(`  ${c.dim(wrap(t.plan.noClock(), 74, '  '))}`);
+    }
+    for (const action of actions) {
+        const name = action.label === UNLABELLED ? t.profile.unlabelled() : action.label;
+        const money = action.savingUsd !== null
+            ? t.plan.projected(formatUsd(action.savingUsd))
+            : t.plan.staked(formatUsd(action.stakeUsd ?? 0));
+        console.log();
+        console.log(`  ${c.green('→')} ${c.bold(t.plan.action(action.kind, name, action.model))}  ${money}`);
+        if (action.detail.routeTo !== undefined) {
+            console.log(`    ${c.dim(t.plan.routeTo(action.detail.routeTo.displayName))}`);
+        }
+        for (const assumption of action.assumes) {
+            console.log(`    ${c.yellow('?')} ${c.dim(wrap(t.plan.assume(assumption), 72, '      '))}`);
+        }
+        if (action.check !== null) {
+            console.log(`    ${c.dim(wrap(t.plan.check(action.check), 72, '      '))}`);
+        }
+    }
+    if (filtered > 0) {
+        console.log();
+        console.log(`  ${c.dim(wrap(t.plan.filtered(n(filtered), formatUsd(minUsd), formatUsd(droppedUsd)), 74, '  '))}`);
+    }
+    console.log();
+    console.log(`  ${c.dim(wrap(t.plan.footer(), 74, '  '))}`);
+    if (outPath !== undefined)
+        console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
+}
 async function commandProfile(args, config, pricing, t) {
     const path = args.positional[0];
     if (path === undefined) {
@@ -1658,7 +1790,6 @@ async function commandProfile(args, config, pricing, t) {
      * directory holding nothing readable is an error naming what it looked for,
      * not an empty report.
      */
-    const LOG_EXTENSIONS = ['.jsonl', '.ndjson', '.log', '.json'];
     /**
      * The same names, gzipped — which is what a rotated log actually looks like
      * a day after it rotates.
@@ -1675,10 +1806,17 @@ async function commandProfile(args, config, pricing, t) {
     const target = await stat(path).catch(() => null);
     let logFiles = [path];
     if (target?.isDirectory()) {
-        const entries = await readdir(path, { withFileTypes: true });
+        /**
+         * Recursive under `--by-source`, flat otherwise. The fleet's whole point
+         * is one directory per service, so the walk must descend; the flat mode
+         * keeps its long-standing behaviour because a directory of rotated logs
+         * with an unrelated subfolder should not quietly absorb it.
+         */
+        const bySourceMode = boolFlag(args, 'by-source');
+        const entries = await readdir(path, { withFileTypes: true, recursive: bySourceMode });
         logFiles = entries
             .filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
-            .map((entry) => join(path, entry.name))
+            .map((entry) => join(entry.parentPath ?? path, entry.name))
             .sort((a, b) => a.localeCompare(b));
         if (logFiles.length === 0) {
             throw new Error(t.profile.noLogsInDirectory(path, READABLE.join(', ')));
@@ -1801,6 +1939,109 @@ async function commandProfile(args, config, pricing, t) {
     }
     const n = (value) => value.toLocaleString(t.numberLocale);
     const pct = (share) => `${(share * 100).toFixed(1)}%`;
+    /**
+     * `--by-source`: one report per service, plus the rollup — the fleet.
+     *
+     * A merged bill is right for one service and wrong for twelve: it hides
+     * which service the money comes from, per-service budgets cannot exist,
+     * and the findings a comparison between services could make are invisible.
+     * Files are assigned to sources by the most specific matching glob from the
+     * config's `sources` block; a file matching no source is named loudly,
+     * because a log that silently joined no report is spend missing from every
+     * bill.
+     */
+    if (boolFlag(args, 'by-source')) {
+        const sourceDefs = config.sources;
+        if (sourceDefs === undefined || Object.keys(sourceDefs).length === 0) {
+            throw new Error(t.profile.bySourceNeedsConfig());
+        }
+        const { bySource, unmatched } = assignSources(logFiles, sourceDefs);
+        if (bySource.size === 0) {
+            throw new Error(t.profile.bySourceNothingMatched(Object.keys(sourceDefs).join(', ')));
+        }
+        const textByFile = new Map(logFiles.map((file, i) => [file, logTexts[i]]));
+        const fleetSources = [];
+        const cacheDeltas = new Map();
+        for (const [name, files] of [...bySource.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+            const text = files
+                .map((file) => textByFile.get(file))
+                .map((chunk) => (chunk.endsWith('\n') ? chunk : `${chunk}\n`))
+                .join('');
+            const sourceReport = profileUsage(text, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
+            fleetSources.push({ name, report: sourceReport });
+            cacheDeltas.set(name, cacheEconomics(sourceReport.total).deltaUsd);
+        }
+        const aggregate = profileUsage(raw, { catalogue: pricing, label: onlyLabel, sinceMs, untilMs });
+        const rollup = fleetRollup(fleetSources, {
+            cacheDeltas,
+            aggregateCacheDelta: cacheEconomics(aggregate.total).deltaUsd,
+        });
+        if (boolFlag(args, 'json')) {
+            console.log(JSON.stringify({
+                schemaVersion: 1,
+                bySource: fleetSources.map((source) => ({ name: source.name, report: source.report })),
+                rollup: {
+                    totalUsd: rollup.totalUsd,
+                    calls: rollup.calls,
+                    sources: rollup.sources,
+                    worst: rollup.worst,
+                    mismatchedSpans: rollup.mismatchedSpans,
+                    splitBrains: rollup.splitBrains,
+                    cacheUnderwater: rollup.cacheUnderwater,
+                    unmatchedFiles: unmatched,
+                },
+            }, (key, value) => (value instanceof Map ? undefined : value), 2));
+        }
+        else {
+            console.log(c.bold(t.profile.fleetHeading(n(rollup.sources.length), formatUsd(rollup.totalUsd), t.profile.calls(rollup.calls))));
+            for (const row of rollup.sources) {
+                const span = row.spanDays === null ? t.profile.fleetNoClock() : t.profile.fleetSpan(row.spanDays.toFixed(1));
+                console.log(`  ${t.profile.fleetRow(row.name, formatUsd(row.usd), pct(row.share), t.profile.calls(row.calls), span)}`);
+            }
+            if (rollup.worst !== null && rollup.sources.length > 1) {
+                console.log();
+                console.log(`  ${c.yellow('!')} ${c.bold(wrap(t.profile.fleetWorst(rollup.worst.name, formatUsd(rollup.worst.usd), pct(rollup.worst.share)), 74, '    '))}`);
+            }
+            if (rollup.mismatchedSpans) {
+                console.log(`  ${c.dim(wrap(t.profile.fleetMismatchedSpans(), 74, '  '))}`);
+            }
+            for (const split of rollup.splitBrains.slice(0, 3)) {
+                console.log();
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.fleetSplitBrain(split.label, split.sources.map((v) => `${v.name} → ${v.model} (${formatUsd(v.usd)})`).join(', ')), 74, '    ')}`);
+            }
+            for (const under of rollup.cacheUnderwater.slice(0, 3)) {
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.fleetCacheUnderwater(under.name, formatUsd(under.deltaUsd)), 74, '    ')}`);
+            }
+            for (const file of unmatched) {
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.fleetUnmatched(file), 74, '    ')}`);
+            }
+            console.log();
+            console.log(`  ${c.dim(wrap(t.profile.fleetFooter(), 74, '  '))}`);
+        }
+        /**
+         * The per-source gates. Each budget judges its own service and the run
+         * fails naming the service — a total that hides which source crossed its
+         * line is the rendering this mode exists to end. Waivable per source
+         * through `bySource:<name>`, under the same expiry discipline.
+         */
+        const bySourceBudgets = config.spend?.bySource ?? {};
+        for (const [name, limit] of Object.entries(bySourceBudgets)) {
+            const found = fleetSources.find((source) => source.name === name);
+            if (found === undefined) {
+                console.error(c.dim(t.profile.fleetBudgetMissing(name)));
+                continue;
+            }
+            const usd = found.report.total.totalUsd;
+            if (usd > limit) {
+                console.error(c.red(t.profile.fleetBudgetFailed(name, formatUsd(usd), formatUsd(limit))));
+                process.exitCode = 1;
+            }
+            else {
+                console.error(c.dim(t.profile.fleetBudgetOk(name, formatUsd(usd), formatUsd(limit))));
+            }
+        }
+        return;
+    }
     /**
      * `--dry-run`: what this log could and could not answer, and no bill.
      *
@@ -4691,6 +4932,9 @@ async function main() {
         case 'profile':
             await commandProfile(args, config, pricing, t);
             break;
+        case 'plan':
+            await commandPlan(args, pricing, t);
+            break;
         case 'route':
             await commandRoute(args, pricing, t);
             break;