@trazum/cli 1.50.2 → 1.50.4

package/dist/index.js

@@ -4,7 +4,7 @@ import { open, readdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve as resolvePath } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { gunzipSync } from 'node:zlib';
-import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
+import { applyRewrites, BASELINE_FILENAME, BASELINE_VERSION, breaches, cacheableMinimum, analyzeCachePrefix, billLevers, bucketedCacheEconomics, bucketedProfile, buildHistory, buildPlan, connectorFor, CONNECTORS, normalizeAnthropicUsage, normalizeOpenAIUsage, bucketsFromRecords, evaluateWatch, firedKey, pruneRecords, recordsFromBuckets, storeInventory, storedReportFrom, verifyPlan, cacheEconomics, cacheHitRate, contextPressure, comparePrompts, compareToBaseline, computeSavings, countTokensAnthropic, DEFAULT_USAGE, budgetPositions, conform, outcomeReport, FAILURE_POLICIES, detectFromSource, matchLocale, parsePlanDocument, waiverDay, waiverHistory, proposeInit, MIN_RATE_DAYS, parseConfig, coverageDrift, driversBetween, explainGateFailure, assignSources, fleetRollup, labelCoverage, measuredUsage, gateMargin, GATE_MARGIN_TIGHT, estimateTokens, evaluate, extractPrompts, findExamples, formatBaseline, formatSignedUsd, formatUsd, getMessages, getModel, hasMarker, LOCALES, MAX_BASELINE_BYTES, moneyIsComparable, mostSpecificMatch, nearestName, optimize, parseBaseline, PHRASE_LANGUAGES, plannedCalls, profilePrompt, profileToCsv, profileUsage, promptId, providerFromEnv, pruneExamples, refineWithLlm, rejectionText, reorderForCache, repriceProfile, reviewAgeDays, reviewExamples, RULES, sharedPrefixes, sharesOf, SOURCE_EXTENSIONS, suggestRewrites, toOtlpMetrics, toPromptfoo, TTL_1H_MS, UNLABELLED, withExactTokenCounts, } from '@trazum/core';
 import { cacheDir, cacheStats, cachingProvider, clearCache } from './suggest-cache.js';
 import { dayOf, formatGap, median, spanDays } from './time.js';
 // Everything that reads the filesystem, on its own entry point so the web
@@ -15,6 +15,7 @@ import { fetchProviderUsage, findCredential } from './connect.js';
 import { STORE_DIR, appendRecords, readStore, rewriteStore } from './store-fs.js';
 import { WAIVER_LOG, appendWaiverUse, readWaiverLog } from './waiver-log.js';
 import { DEFAULT_PORT, buildServer, listen } from './serve.js';
+import { DEFAULT_GATEWAY_PORT, UPSTREAMS, buildGateway, listenGateway, } from './gateway-server.js';
 import { WATCH_STATE_VERSION, checkWebhook, postWebhook, readWatchState, writeWatchState, } from './watch-run.js';
 import { LOCALE_ENV_VARS, detectLocale, getCliMessages } from './i18n/index.js';
 import { MAX_SUMMARY_CHARS, fitWithin, renderBlameMarkdown, renderCheckMarkdown, renderDiffMarkdown, renderRankMarkdown, renderProfileMarkdown, } from './markdown.js';
@@ -33,6 +34,7 @@ const c = {
 const VALUE_FLAGS = new Set([
     'against',
     'contract',
+    'on-cannot-tell',
     'from-log',
     'min-usd',
     'payload',
@@ -316,6 +318,7 @@ const COMMAND_FLAGS = {
     init: ['dry-run', 'yes', 'json', 'pricing', 'pricing-live'],
     conform: ['contract', 'json'],
     feedback: [],
+    gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
     where: [],
     rules: [],
     blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -1575,6 +1578,93 @@ function commandFeedback(t) {
     console.log(`  ${url}`);
     console.log();
 }
+/**
+ * `trazum gateway <provider>` — in the path, and refusing rather than advising.
+ *
+ * The last thing this product could not do. `serve` answers a question an
+ * implementation may ignore; a connector reports the runaway after it ran.
+ * Standing between the caller and the provider fixes both — usage is measured
+ * from the provider's own response as it comes back, and a refusal is a
+ * refusal.
+ *
+ * **The failure policy is required.** `--on-cannot-tell fail-open` keeps the
+ * product working and lets the bill run; `fail-closed` stops the bill and takes
+ * the product down with it. Both are defensible and there is deliberately no
+ * default: a proxy that picks silently has made the most consequential decision
+ * in somebody's architecture on their behalf, at install time, without saying
+ * so.
+ *
+ * **Substitution is off unless it is written down.** `spend.substitute` in the
+ * config, with the operator's own reason, and every substituted call is marked
+ * so no later report treats it as the call the caller made.
+ */
+async function commandGateway(args, config, configDir, pricing, t) {
+    const provider = args.positional[0];
+    if (provider === undefined || UPSTREAMS[provider] === undefined) {
+        throw new Error(t.gateway.badProvider(provider ?? '', Object.keys(UPSTREAMS).join(', ')));
+    }
+    /**
+     * No default, and the error says why rather than just what.
+     *
+     * The one flag in this product that refuses to guess on the reader's behalf,
+     * because the two answers differ in which failure they accept and nobody but
+     * the operator knows which their product can survive.
+     */
+    const policyFlag = stringFlag(args, 'on-cannot-tell');
+    if (policyFlag === undefined || !FAILURE_POLICIES.includes(policyFlag)) {
+        throw new Error(t.gateway.needsPolicy(FAILURE_POLICIES.join(', ')));
+    }
+    const { resolved } = await readStore(configDir);
+    const budget = budgetPositions(resolved.records, config.spend, { catalogue: pricing });
+    const position = budget.positions[0] ?? null;
+    /**
+     * Read once at start, like `serve`'s.
+     *
+     * A file read in the request path would put Trazum's own latency between a
+     * caller and their provider on every call, which is a cost this product
+     * would otherwise be reporting on somebody else. The staleness is real, so a
+     * refusal carries `asOfMs` and says what it rested on.
+     */
+    const standing = position === null || position.coverage === 'none'
+        ? null
+        : {
+            limitUsd: position.limitUsd,
+            consumedUsd: position.consumedUsd,
+            provenance: 'measured',
+            asOfMs: Date.now(),
+        };
+    const measured = { calls: 0, usd: 0 };
+    const server = buildGateway({
+        provider,
+        catalogue: pricing,
+        policy: {
+            onCannotTell: policyFlag,
+            ...(config.spend?.substitute === undefined ? {} : { substitute: config.spend.substitute }),
+        },
+        standing: () => standing,
+        record: (call) => {
+            measured.calls += 1;
+            console.error(c.dim(t.gateway.measured(call.model, call.label, call.inputTokens, call.outputTokens, call.substituted)));
+        },
+        note: (line) => {
+            console.error(c.yellow(`  ${line}`));
+        },
+    });
+    const socket = stringFlag(args, 'socket');
+    const portRaw = stringFlag(args, 'port');
+    const port = portRaw === undefined ? DEFAULT_GATEWAY_PORT : Number(portRaw);
+    if (socket === undefined && (!Number.isInteger(port) || port < 0 || port > 65_535)) {
+        throw new Error(t.serve.badPort(String(portRaw)));
+    }
+    const where = await listenGateway(server, socket !== undefined ? { socket } : { port });
+    console.log(c.bold(t.gateway.listening(where, provider)));
+    console.log(`  ${c.dim(wrap(t.gateway.pointYourSdk(where), 74, '    '))}`);
+    console.log(`  ${c.dim(wrap(t.gateway.credential(), 74, '    '))}`);
+    console.log(`  ${c.dim(wrap(t.gateway.neverSubstitutes(), 74, '    '))}`);
+    console.log(`  ${c.dim(wrap(standing === null ? t.gateway.noStanding() : t.gateway.standing(formatUsd(standing.consumedUsd), formatUsd(standing.limitUsd)), 74, '    '))}`);
+    console.log(`  ${c.dim(wrap(t.gateway.policy(policyFlag), 74, '    '))}`);
+    console.log();
+}
 function commandModels(t, pricing) {
     const n = (value) => value.toLocaleString(t.numberLocale);
     const col = t.models.columns;
@@ -2761,7 +2851,7 @@ async function commandConnect(args, pricing, t) {
  * same action planned twice — and no series, however long, becomes a
  * forecast.
  */
-async function commandHistory(args, config, pricing, t) {
+async function commandHistory(args, config, configDir, pricing, t) {
     /**
      * `--store` builds the series from measured spend already on disk.
      *
@@ -2873,7 +2963,7 @@ async function commandHistory(args, config, pricing, t) {
      * the waiver record belongs to the repository whose gates fired, and the
      * stored reports may have come from anywhere.
      */
-    const waivers = await readWaiverLog('.');
+    const waivers = await readWaiverLog(configDir);
     const waiverReport = waiverHistory(waivers.uses, config.waive ?? []);
     const stamped = { ...history, unrecognizedFiles: unrecognized, waivers: waiverReport };
     const n = (value) => value.toLocaleString(t.numberLocale);
@@ -3225,7 +3315,7 @@ async function commandPlan(args, pricing, t) {
     if (outPath !== undefined)
         console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
 }
-async function commandProfile(args, config, pricing, t) {
+async function commandProfile(args, config, configDir, pricing, t) {
     const path = args.positional[0];
     if (path === undefined) {
         console.log();
@@ -3497,6 +3587,7 @@ async function commandProfile(args, config, pricing, t) {
         can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
         can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
         can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
+        can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
         can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
         // "No cache traffic" is not a missing field: the split can only exist on
         // records that wrote, and a log that never wrote has nothing to record.
@@ -3984,7 +4075,7 @@ async function commandProfile(args, config, pricing, t) {
         if (waiverUses.length === 0)
             return;
         for (const use of waiverUses) {
-            const failed = await appendWaiverUse('.', use);
+            const failed = await appendWaiverUse(configDir, use);
             if (failed !== null) {
                 console.error(c.dim(t.profile.waiveNotRecorded(WAIVER_LOG, failed)));
                 return;
@@ -5119,6 +5210,69 @@ async function commandProfile(args, config, pricing, t) {
      * section at all, because a paragraph of things that are fine is the
      * paragraph readers learn to skip.
      */
+    /**
+     * Outcomes — the counterpart, where somebody recorded one.
+     *
+     * Printed above the coverage section rather than below it, because when this
+     * section is present it is the most valuable thing on the page: every other
+     * figure in this report is a cost, and this is the only one that says what
+     * the money bought.
+     *
+     * Silent when nothing recorded an outcome. The coverage section below
+     * already names the missing field and what it would unlock, and printing an
+     * empty Outcomes heading above it would be the same sentence twice.
+     */
+    {
+        const outcomes = outcomeReport(report.outcomeTally, config.outcomes ?? null);
+        if (outcomes.coverage.recorded > 0) {
+            console.log();
+            console.log(c.bold(t.profile.outcomeHeading()));
+            const col = t.profile.outcomeColumns;
+            const rows = [...outcomes.slices, ...outcomes.undeclared].map((slice) => ({
+                value: slice.value,
+                verdict: slice.verdict === 'success'
+                    ? t.profile.verdictSuccess()
+                    : slice.verdict === 'undeclared'
+                        ? t.profile.verdictUndeclared()
+                        : t.profile.verdictOther(),
+                calls: n(slice.calls),
+                spend: formatUsd(slice.usd),
+            }));
+            const w = {
+                value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
+                verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
+                calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
+                spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
+            };
+            console.log(c.dim(`  ${col.outcome.padEnd(w.value)}  ${''.padEnd(w.verdict)}  ` +
+                `${col.calls.padStart(w.calls)}  ${col.spend.padStart(w.spend)}`));
+            for (const row of rows) {
+                const tint = row.verdict === t.profile.verdictUndeclared()
+                    ? c.yellow
+                    : row.verdict === t.profile.verdictSuccess()
+                        ? c.green
+                        : c.dim;
+                console.log(`  ${row.value.padEnd(w.value)}  ${tint(row.verdict.padEnd(w.verdict))}  ` +
+                    `${row.calls.padStart(w.calls)}  ${row.spend.padStart(w.spend)}`);
+            }
+            console.log();
+            if (outcomes.successShareOfRecordedUsd !== null) {
+                const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
+                console.log(`  ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, '    ')}`);
+            }
+            else if (outcomes.noRate !== null) {
+                console.log(`  ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, '    '))}`);
+            }
+            // What the rate does not cover, every time it is printed. A rate over a
+            // twelfth of the bill is a rate about a twelfth of the bill.
+            if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.outcomeUnrecorded(pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd), formatUsd(outcomes.coverage.unrecordedUsd)), 74, '    ')}`);
+            }
+            if (outcomes.undeclared.length > 0) {
+                console.log(`  ${c.yellow('!')} ${wrap(t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')), 74, '    ')}`);
+            }
+        }
+    }
     const coverage = report.fieldCoverage;
     if (coverage.parsed > 0) {
         const missing = [];
@@ -5129,6 +5283,14 @@ async function commandProfile(args, config, pricing, t) {
         if (coverage.session < coverage.parsed) {
             missing.push(t.profile.needsSession(partial(coverage.session)));
         }
+        /**
+         * Listed first among the missing when it is missing entirely, because it
+         * is the one field that changes what every other figure here *means*. The
+         * rest sharpen a cost; this one gives it a counterpart.
+         */
+        if (coverage.outcome < coverage.parsed) {
+            missing.push(t.profile.needsOutcome(partial(coverage.outcome)));
+        }
         if (coverage.ts < coverage.parsed) {
             missing.push(t.profile.needsTs(partial(coverage.ts)));
         }
@@ -6431,6 +6593,22 @@ async function main() {
         };
     }
     const { config } = loaded;
+    /**
+     * Where the waiver record lives: **beside the config that declared it**.
+     *
+     * It used to be the process's working directory, which is a different place
+     * whenever somebody runs `trazum profile ../logs/x.jsonl --config ../repo/
+     * trazum.config.json` — and that is not hypothetical. This repository's own
+     * test suite did exactly that from `packages/cli`, so sixty records of a
+     * fixture's decisions accumulated in a package directory and one of them was
+     * committed to `main`, where it sat for two releases.
+     *
+     * A waiver is a decision a *repository* made. The record of using it belongs
+     * with the file that made it, not with wherever the terminal happened to be.
+     * No config means no waivers, so there is nothing to write and `.` is never
+     * reached.
+     */
+    const configDir = loaded.path === null ? '.' : dirname(loaded.path);
     const pricing = await pricingFor(args, loaded, t);
     // The config only gets to choose the locale when nothing more explicit did.
     if (config.locale && !stringFlag(args, 'locale')) {
@@ -6448,7 +6626,7 @@ async function main() {
             await commandBaseline(args, config, pricing, t, locale);
             break;
         case 'profile':
-            await commandProfile(args, config, pricing, t);
+            await commandProfile(args, config, configDir, pricing, t);
             break;
         case 'plan':
             await commandPlan(args, pricing, t);
@@ -6457,7 +6635,7 @@ async function main() {
             await commandVerify(args, pricing, t);
             break;
         case 'history':
-            await commandHistory(args, config, pricing, t);
+            await commandHistory(args, config, configDir, pricing, t);
             break;
         case 'connect':
             await commandConnect(args, pricing, t);
@@ -6486,6 +6664,9 @@ async function main() {
         case 'models':
             commandModels(t, pricing);
             break;
+        case 'gateway':
+            await commandGateway(args, config, configDir, pricing, t);
+            break;
         case 'feedback':
             commandFeedback(t);
             break;