@trazum/cli 1.38.0 → 1.40.0

package/dist/index.js

@@ -2,7 +2,7 @@
 import { readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { join, resolve as resolvePath } from 'node:path';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, buildPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, buildHistory, buildPlan, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, detectFromSource, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -287,6 +287,8 @@ const COMMAND_FLAGS = {
     baseline: ['model', 'calls', 'output-tokens', 'cache-hit-rate', 'batch', 'exact-tokens', 'out', 'o'],
     profile: ['json', 'pricing', 'pricing-live', 'against', 'what-if', 'markdown-out', 'csv-out', 'csv-shape', 'max-usd', 'max-growth-usd', 'max-cache-loss-usd', 'max-day-usd', 'max-session-usd', 'label', 'since', 'until', 'dry-run', 'markdown-summary', 'by-source'],
     plan: ['json', 'out', 'markdown-out', 'min-usd', 'pricing', 'pricing-live'],
+    verify: ['against', 'gate', 'json', 'markdown-out', 'pricing', 'pricing-live'],
+    history: ['json', 'markdown-out'],
     route: ['prompt-file', 'cases', 'label', 'concurrency', 'json', 'yes', 'pricing', 'pricing-live'],
     eval: ['cases', 'level', 'concurrency', 'export', 'out', 'o', 'model'],
     prune: ['cases', 'concurrency', 'json', 'yes'],
@@ -1644,6 +1646,224 @@ function isoDate() {
  * metered API calls somebody was actually billed for — the bill exists wherever
  * Trazum happens to be running, so the host has no bearing on it.
  */
+/**
+ * `trazum history <dir>` — many reports over many periods, as one series.
+ *
+ * Derived from *stored* `--json` documents, never re-parsed logs: a team can
+ * keep a year of reports and throw the raw logs away, which is what the
+ * privacy story requires anyway. Shapes are named — a climb, a decay, the
+ * same action planned twice — and no series, however long, becomes a
+ * forecast.
+ */
+async function commandHistory(args, t) {
+    const path = args.positional[0];
+    if (path === undefined)
+        throw new Error(t.history.noTarget());
+    const target = await stat(path).catch(() => null);
+    if (!target?.isDirectory())
+        throw new Error(t.history.noTarget());
+    const entries = await readdir(path, { withFileTypes: true });
+    const files = entries
+        .filter((entry) => entry.isFile() && entry.name.endsWith('.json'))
+        .map((entry) => join(path, entry.name))
+        .sort((a, b) => a.localeCompare(b));
+    const reports = [];
+    const plans = [];
+    const unrecognized = [];
+    for (const file of files) {
+        let parsed;
+        try {
+            parsed = JSON.parse(await readFile(file, 'utf8'));
+        }
+        catch {
+            unrecognized.push(file);
+            continue;
+        }
+        const report = storedReportFrom(file, parsed);
+        if (report !== null) {
+            reports.push(report);
+            continue;
+        }
+        const maybePlan = parsed;
+        if (maybePlan?.schemaVersion === 1 && Array.isArray(maybePlan.actions)) {
+            plans.push(maybePlan);
+            continue;
+        }
+        unrecognized.push(file);
+    }
+    const history = buildHistory(reports, plans);
+    if (history.periods.length < 3) {
+        throw new Error(t.history.needsThree(String(history.periods.length)));
+    }
+    const stamped = { ...history, unrecognizedFiles: unrecognized };
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    const day = (ms) => new Date(ms).toISOString().slice(0, 10);
+    const pct = (value) => `${(value * 100).toFixed(1)}%`;
+    const runLine = (run) => {
+        if (run.kind === 'label-spend-climbing') {
+            const name = run.subject === UNLABELLED ? t.profile.unlabelled() : run.subject;
+            return t.history.runLabel(name, n(run.periods), run.sinceName, formatUsd(run.from), formatUsd(run.to));
+        }
+        if (run.kind === 'model-share-climbing') {
+            return t.history.runModel(run.subject, n(run.periods), run.sinceName, pct(run.from), pct(run.to));
+        }
+        return t.history.runCache(n(run.periods), run.sinceName, pct(run.from), pct(run.to));
+    };
+    const lines = (md) => {
+        const out = [];
+        const first = history.periods[0];
+        const last = history.periods[history.periods.length - 1];
+        const heading = t.history.heading(n(history.periods.length), day(first.fromMs), day(last.toMs));
+        out.push(md ? `## ${heading}` : heading);
+        for (const period of history.periods) {
+            const row = t.history.periodRow(period.name, formatUsd(period.totalUsd), n(period.calls), ((period.toMs - period.fromMs) / 86_400_000).toFixed(1));
+            out.push(md ? `- ${row}` : `  ${row}`);
+        }
+        if (history.runs.length > 0)
+            out.push('');
+        for (const run of history.runs) {
+            out.push(md ? `- ${runLine(run)}` : `  ! ${runLine(run)}`);
+        }
+        if (history.repeatedPlanActions.length > 0)
+            out.push('');
+        for (const repeat of history.repeatedPlanActions) {
+            const name = repeat.label === UNLABELLED ? t.profile.unlabelled() : repeat.label;
+            const row = t.history.repeated(repeat.kind, name, repeat.model, n(repeat.appearances), repeat.firstPlanned?.slice(0, 10) ?? null, repeat.lastPlanned?.slice(0, 10) ?? null);
+            out.push(md ? `- ${row}` : `  ! ${row}`);
+        }
+        for (const name of history.undatedReports) {
+            out.push(md ? `- ${t.history.undated(name)}` : `  ${t.history.undated(name)}`);
+        }
+        for (const name of unrecognized) {
+            out.push(md ? `- ${t.history.unrecognized(name)}` : `  ${t.history.unrecognized(name)}`);
+        }
+        out.push('');
+        out.push(md ? `_${t.history.footer()}_` : `  ${t.history.footer()}`);
+        return out;
+    };
+    await writeMarkdown(args, () => lines(true).join('\n'));
+    if (boolFlag(args, 'json')) {
+        console.log(JSON.stringify(stamped, null, 2));
+        return;
+    }
+    const [head, ...rest] = lines(false);
+    console.log(c.bold(head));
+    for (const row of rest)
+        console.log(row === '' ? '' : wrap(row, 76, '    '));
+}
+/**
+ * `trazum verify <plan.json> --against <newer.jsonl|dir>` — did it work?
+ *
+ * The plan predicted; this holds the prediction to the log that came after
+ * it. Three outcomes and never two — arrived, did not arrive, cannot be told
+ * — because "cannot be told" rendered as "arrived" is how every other tool
+ * congratulates a team for a workload that merely vanished. With `--gate`,
+ * a broken promise is a failing exit code: a different and more useful gate
+ * than "spend went up".
+ */
+async function commandVerify(args, pricing, t) {
+    const planPath = args.positional[0];
+    if (planPath === undefined)
+        throw new Error(t.verify.noTarget());
+    const againstPath = stringFlag(args, 'against');
+    if (againstPath === undefined)
+        throw new Error(t.verify.needsAgainst());
+    let plan;
+    try {
+        const parsed = JSON.parse(await readFile(planPath, 'utf8'));
+        if (parsed?.schemaVersion !== 1 || !Array.isArray(parsed.actions)) {
+            throw new Error(t.verify.badPlan(planPath));
+        }
+        plan = parsed;
+    }
+    catch (error) {
+        if (error instanceof SyntaxError)
+            throw new Error(t.verify.badPlan(planPath));
+        throw error;
+    }
+    const GZ = LOG_EXTENSIONS.map((ext) => `${ext}.gz`);
+    const READABLE = [...LOG_EXTENSIONS, ...GZ];
+    const target = await stat(againstPath).catch(() => null);
+    let files = [againstPath];
+    if (target?.isDirectory()) {
+        const entries = await readdir(againstPath, { withFileTypes: true });
+        files = entries
+            .filter((entry) => entry.isFile() && READABLE.some((ext) => entry.name.endsWith(ext)))
+            .map((entry) => join(againstPath, entry.name))
+            .sort((a, b) => a.localeCompare(b));
+        if (files.length === 0)
+            throw new Error(t.profile.noLogsInDirectory(againstPath, READABLE.join(', ')));
+    }
+    const texts = await Promise.all(files.map((file) => readUsageLog(file, t)));
+    const raw = texts.map((text) => (text.endsWith('\n') ? text : `${text}\n`)).join('');
+    const report = profileUsage(raw, { catalogue: pricing });
+    const verification = verifyPlan(plan, report, { currentPricingLastReviewed: pricing.lastReviewed });
+    const gate = boolFlag(args, 'gate');
+    const n = (value) => value.toLocaleString(t.numberLocale);
+    const lines = (md) => {
+        const out = [];
+        const actionLine = (v) => {
+            const name = v.action.label === UNLABELLED ? t.profile.unlabelled() : v.action.label;
+            const rows = [];
+            rows.push(t.verify.action(v.action.kind, name, v.action.model, v.outcome));
+            if (v.outcome === 'cannot-tell' && v.reason !== null)
+                rows.push(t.verify.reason(v.reason));
+            if (v.action.kind === 'route' || v.action.kind === 'route+batch') {
+                if (v.outcome !== 'cannot-tell') {
+                    rows.push(t.verify.routeObserved(String(v.observed.dearestModel ?? ''), formatUsd(Number(v.observed.onTargetUsd ?? 0)), formatUsd(Number(v.observed.onOldModelUsd ?? 0))));
+                }
+                if (v.action.kind === 'route+batch' && v.outcome !== 'cannot-tell')
+                    rows.push(t.verify.batchUnobservable());
+            }
+            if (v.action.kind === 'fix-truncation' && v.outcome === 'not-arrived') {
+                rows.push(t.verify.truncationObserved(formatUsd(Number(v.observed.retryBillUsd ?? 0))));
+            }
+            if (v.action.kind === 'fix-caching' && v.outcome !== 'cannot-tell') {
+                rows.push(t.verify.cacheObserved(formatUsd(Number(v.observed.deltaUsd ?? 0)), v.outcome));
+            }
+            if (v.attribution?.calls !== undefined) {
+                rows.push(t.verify.attribution(n(Math.round(v.attribution.calls.before)), n(Math.round(v.attribution.calls.after)), n(Math.round(v.attribution.outputPerCallTokens?.before ?? 0)), n(Math.round(v.attribution.outputPerCallTokens?.after ?? 0))));
+            }
+            return rows;
+        };
+        const heading = t.verify.heading(n(verification.actions.length), verification.planCreatedAt === null ? null : verification.planCreatedAt.slice(0, 10));
+        out.push(md ? `## ${heading}` : heading);
+        out.push(t.verify.counts(n(verification.arrived), n(verification.notArrived), n(verification.cannotTell)));
+        if (verification.pricesChanged) {
+            out.push(t.verify.pricesChanged(verification.planPricing, verification.currentPricing));
+        }
+        for (const v of verification.actions) {
+            out.push('');
+            const [head, ...rest] = actionLine(v);
+            out.push(md ? `### ${head}` : `→ ${head}`);
+            for (const row of rest)
+                out.push(md ? `- ${row}` : `  · ${row}`);
+        }
+        out.push('');
+        out.push(t.verify.footer());
+        return out;
+    };
+    await writeMarkdown(args, () => lines(true).join('\n'));
+    if (boolFlag(args, 'json')) {
+        console.log(JSON.stringify(verification, null, 2));
+    }
+    else {
+        const [head, ...rest] = lines(false);
+        console.log(c.bold(head));
+        for (const row of rest) {
+            console.log(row === '' ? '' : `  ${wrap(row, 74, '    ')}`);
+        }
+    }
+    if (gate) {
+        if (verification.gateFailures > 0) {
+            console.error(c.red(t.verify.gateFailed(n(verification.gateFailures), n(verification.actions.length))));
+            process.exitCode = 1;
+        }
+        else {
+            console.log(c.green(t.verify.gateOk()));
+        }
+    }
+}
 /**
  * `trazum plan <log>` — not a list of findings, a ranked plan of what to do.
  *
@@ -4935,6 +5155,12 @@ async function main() {
         case 'plan':
             await commandPlan(args, pricing, t);
             break;
+        case 'verify':
+            await commandVerify(args, pricing, t);
+            break;
+        case 'history':
+            await commandHistory(args, t);
+            break;
         case 'route':
             await commandRoute(args, pricing, t);
             break;