@trazum/cli 1.50.2 → 1.50.4

package/src/index.ts

@@ -39,6 +39,8 @@ import {
   DEFAULT_USAGE,
   budgetPositions,
   conform,
+  outcomeReport,
+  FAILURE_POLICIES,
   detectFromSource,
   matchLocale,
   parsePlanDocument,
@@ -133,6 +135,8 @@ import type {
 import type {
   BudgetReport,
   ContractName,
+  FailurePolicy,
+  GatewayStanding,
   UsageProfileReport,
   WaiverUse,
   InitDecline,
@@ -179,6 +183,12 @@ import { fetchProviderUsage, findCredential } from './connect.js';
 import { STORE_DIR, appendRecords, readStore, rewriteStore } from './store-fs.js';
 import { WAIVER_LOG, appendWaiverUse, readWaiverLog } from './waiver-log.js';
 import { DEFAULT_PORT, buildServer, listen } from './serve.js';
+import {
+  DEFAULT_GATEWAY_PORT,
+  UPSTREAMS,
+  buildGateway,
+  listenGateway,
+} from './gateway-server.js';
 import {
   WATCH_STATE_VERSION,
   checkWebhook,
@@ -233,6 +243,7 @@ interface Args {
 const VALUE_FLAGS = new Set([
   'against',
   'contract',
+  'on-cannot-tell',
   'from-log',
   'min-usd',
   'payload',
@@ -557,6 +568,7 @@ const COMMAND_FLAGS: Record<string, string[]> = {
   init: ['dry-run', 'yes', 'json', 'pricing', 'pricing-live'],
   conform: ['contract', 'json'],
   feedback: [],
+  gateway: ['on-cannot-tell', 'port', 'socket', 'pricing', 'pricing-live'],
   where: [],
   rules: [],
   blame: ['limit', 'model', 'calls', 'output-tokens', 'batch', 'prompt', 'markdown-out'],
@@ -2064,6 +2076,119 @@ function commandFeedback(t: CliMessages): void {
   console.log();
 }
 
+/**
+ * `trazum gateway <provider>` — in the path, and refusing rather than advising.
+ *
+ * The last thing this product could not do. `serve` answers a question an
+ * implementation may ignore; a connector reports the runaway after it ran.
+ * Standing between the caller and the provider fixes both — usage is measured
+ * from the provider's own response as it comes back, and a refusal is a
+ * refusal.
+ *
+ * **The failure policy is required.** `--on-cannot-tell fail-open` keeps the
+ * product working and lets the bill run; `fail-closed` stops the bill and takes
+ * the product down with it. Both are defensible and there is deliberately no
+ * default: a proxy that picks silently has made the most consequential decision
+ * in somebody's architecture on their behalf, at install time, without saying
+ * so.
+ *
+ * **Substitution is off unless it is written down.** `spend.substitute` in the
+ * config, with the operator's own reason, and every substituted call is marked
+ * so no later report treats it as the call the caller made.
+ */
+async function commandGateway(
+  args: Args,
+  config: TrazumConfig,
+  configDir: string,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
+  const provider = args.positional[0];
+  if (provider === undefined || UPSTREAMS[provider] === undefined) {
+    throw new Error(t.gateway.badProvider(provider ?? '', Object.keys(UPSTREAMS).join(', ')));
+  }
+
+  /**
+   * No default, and the error says why rather than just what.
+   *
+   * The one flag in this product that refuses to guess on the reader's behalf,
+   * because the two answers differ in which failure they accept and nobody but
+   * the operator knows which their product can survive.
+   */
+  const policyFlag = stringFlag(args, 'on-cannot-tell');
+  if (policyFlag === undefined || !FAILURE_POLICIES.includes(policyFlag as FailurePolicy)) {
+    throw new Error(t.gateway.needsPolicy(FAILURE_POLICIES.join(', ')));
+  }
+
+  const { resolved } = await readStore(configDir);
+  const budget = budgetPositions(resolved.records, config.spend, { catalogue: pricing });
+  const position = budget.positions[0] ?? null;
+
+  /**
+   * Read once at start, like `serve`'s.
+   *
+   * A file read in the request path would put Trazum's own latency between a
+   * caller and their provider on every call, which is a cost this product
+   * would otherwise be reporting on somebody else. The staleness is real, so a
+   * refusal carries `asOfMs` and says what it rested on.
+   */
+  const standing: GatewayStanding | null =
+    position === null || position.coverage === 'none'
+      ? null
+      : {
+          limitUsd: position.limitUsd,
+          consumedUsd: position.consumedUsd,
+          provenance: 'measured',
+          asOfMs: Date.now(),
+        };
+
+  const measured: { calls: number; usd: number } = { calls: 0, usd: 0 };
+  const server = buildGateway({
+    provider,
+    catalogue: pricing,
+    policy: {
+      onCannotTell: policyFlag as FailurePolicy,
+      ...(config.spend?.substitute === undefined ? {} : { substitute: config.spend.substitute }),
+    },
+    standing: () => standing,
+    record: (call) => {
+      measured.calls += 1;
+      console.error(
+        c.dim(
+          t.gateway.measured(
+            call.model,
+            call.label,
+            call.inputTokens,
+            call.outputTokens,
+            call.substituted,
+          ),
+        ),
+      );
+    },
+    note: (line) => {
+      console.error(c.yellow(`  ${line}`));
+    },
+  });
+
+  const socket = stringFlag(args, 'socket');
+  const portRaw = stringFlag(args, 'port');
+  const port = portRaw === undefined ? DEFAULT_GATEWAY_PORT : Number(portRaw);
+  if (socket === undefined && (!Number.isInteger(port) || port < 0 || port > 65_535)) {
+    throw new Error(t.serve.badPort(String(portRaw)));
+  }
+
+  const where = await listenGateway(server, socket !== undefined ? { socket } : { port });
+  console.log(c.bold(t.gateway.listening(where, provider)));
+  console.log(`  ${c.dim(wrap(t.gateway.pointYourSdk(where), 74, '    '))}`);
+  console.log(`  ${c.dim(wrap(t.gateway.credential(), 74, '    '))}`);
+  console.log(`  ${c.dim(wrap(t.gateway.neverSubstitutes(), 74, '    '))}`);
+  console.log(
+    `  ${c.dim(wrap(standing === null ? t.gateway.noStanding() : t.gateway.standing(formatUsd(standing.consumedUsd), formatUsd(standing.limitUsd)), 74, '    '))}`,
+  );
+  console.log(`  ${c.dim(wrap(t.gateway.policy(policyFlag), 74, '    '))}`);
+  console.log();
+}
+
 function commandModels(t: CliMessages, pricing: PricingCatalogue): void {
   const n = (value: number): string => value.toLocaleString(t.numberLocale);
   const col = t.models.columns;
@@ -3640,6 +3765,7 @@ async function commandConnect(
 async function commandHistory(
   args: Args,
   config: TrazumConfig,
+  configDir: string,
   pricing: PricingCatalogue,
   t: CliMessages,
 ): Promise<void> {
@@ -3759,7 +3885,7 @@ async function commandHistory(
    * the waiver record belongs to the repository whose gates fired, and the
    * stored reports may have come from anywhere.
    */
-  const waivers = await readWaiverLog('.');
+  const waivers = await readWaiverLog(configDir);
   const waiverReport = waiverHistory(waivers.uses, config.waive ?? []);
 
   const stamped = { ...history, unrecognizedFiles: unrecognized, waivers: waiverReport };
@@ -4161,7 +4287,13 @@ async function commandPlan(
   if (outPath !== undefined) console.log(c.dim(wrap(t.plan.wrote(outPath), 74, '')));
 }
 
-async function commandProfile(args: Args, config: TrazumConfig, pricing: PricingCatalogue, t: CliMessages): Promise<void> {
+async function commandProfile(
+  args: Args,
+  config: TrazumConfig,
+  configDir: string,
+  pricing: PricingCatalogue,
+  t: CliMessages,
+): Promise<void> {
   const path = args.positional[0];
   if (path === undefined) {
     console.log();
@@ -4454,6 +4586,7 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
     can(cov.label > 0, t.profile.dryRunLabels(share(cov.label)));
     can(cov.ts > 0, t.profile.dryRunClock(share(cov.ts)));
     can(cov.session > 0, t.profile.dryRunSessions(share(cov.session)));
+    can(cov.outcome > 0, t.profile.dryRunOutcomes(share(cov.outcome)));
     can(cov.stopReason > 0, t.profile.dryRunStopReason(share(cov.stopReason)));
     // "No cache traffic" is not a missing field: the split can only exist on
     // records that wrote, and a log that never wrote has nothing to record.
@@ -4985,7 +5118,7 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
   const recordWaiverUses = async (): Promise<void> => {
     if (waiverUses.length === 0) return;
     for (const use of waiverUses) {
-      const failed = await appendWaiverUse('.', use);
+      const failed = await appendWaiverUse(configDir, use);
       if (failed !== null) {
         console.error(c.dim(t.profile.waiveNotRecorded(WAIVER_LOG, failed)));
         return;
@@ -6285,6 +6418,97 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
    * section at all, because a paragraph of things that are fine is the
    * paragraph readers learn to skip.
    */
+  /**
+   * Outcomes — the counterpart, where somebody recorded one.
+   *
+   * Printed above the coverage section rather than below it, because when this
+   * section is present it is the most valuable thing on the page: every other
+   * figure in this report is a cost, and this is the only one that says what
+   * the money bought.
+   *
+   * Silent when nothing recorded an outcome. The coverage section below
+   * already names the missing field and what it would unlock, and printing an
+   * empty Outcomes heading above it would be the same sentence twice.
+   */
+  {
+    const outcomes = outcomeReport(report.outcomeTally, config.outcomes ?? null);
+    if (outcomes.coverage.recorded > 0) {
+      console.log();
+      console.log(c.bold(t.profile.outcomeHeading()));
+
+      const col = t.profile.outcomeColumns;
+      const rows = [...outcomes.slices, ...outcomes.undeclared].map((slice) => ({
+        value: slice.value,
+        verdict:
+          slice.verdict === 'success'
+            ? t.profile.verdictSuccess()
+            : slice.verdict === 'undeclared'
+              ? t.profile.verdictUndeclared()
+              : t.profile.verdictOther(),
+        calls: n(slice.calls),
+        spend: formatUsd(slice.usd),
+      }));
+      const w = {
+        value: Math.max(...rows.map((r) => r.value.length), col.outcome.length),
+        verdict: Math.max(...rows.map((r) => r.verdict.length), 0),
+        calls: Math.max(...rows.map((r) => r.calls.length), col.calls.length),
+        spend: Math.max(...rows.map((r) => r.spend.length), col.spend.length),
+      };
+      console.log(
+        c.dim(
+          `  ${col.outcome.padEnd(w.value)}  ${''.padEnd(w.verdict)}  ` +
+            `${col.calls.padStart(w.calls)}  ${col.spend.padStart(w.spend)}`,
+        ),
+      );
+      for (const row of rows) {
+        const tint =
+          row.verdict === t.profile.verdictUndeclared()
+            ? c.yellow
+            : row.verdict === t.profile.verdictSuccess()
+              ? c.green
+              : c.dim;
+        console.log(
+          `  ${row.value.padEnd(w.value)}  ${tint(row.verdict.padEnd(w.verdict))}  ` +
+            `${row.calls.padStart(w.calls)}  ${row.spend.padStart(w.spend)}`,
+        );
+      }
+
+      console.log();
+      if (outcomes.successShareOfRecordedUsd !== null) {
+        const declaredUsd = outcomes.slices.reduce((sum, slice) => sum + slice.usd, 0);
+        console.log(
+          `  ${wrap(t.profile.outcomeRate(pct(outcomes.successShareOfRecordedUsd), formatUsd(declaredUsd)), 74, '    ')}`,
+        );
+      } else if (outcomes.noRate !== null) {
+        console.log(`  ${c.dim(wrap(t.profile.outcomeNoRate(outcomes.noRate), 74, '    '))}`);
+      }
+
+      // What the rate does not cover, every time it is printed. A rate over a
+      // twelfth of the bill is a rate about a twelfth of the bill.
+      if (outcomes.coverage.unrecordedUsd > 0 && report.total.totalUsd > 0) {
+        console.log(
+          `  ${c.yellow('!')} ${wrap(
+            t.profile.outcomeUnrecorded(
+              pct(outcomes.coverage.unrecordedUsd / report.total.totalUsd),
+              formatUsd(outcomes.coverage.unrecordedUsd),
+            ),
+            74,
+            '    ',
+          )}`,
+        );
+      }
+      if (outcomes.undeclared.length > 0) {
+        console.log(
+          `  ${c.yellow('!')} ${wrap(
+            t.profile.outcomeUndeclared(outcomes.undeclared.map((s) => s.value).join(', ')),
+            74,
+            '    ',
+          )}`,
+        );
+      }
+    }
+  }
+
   const coverage = report.fieldCoverage;
   if (coverage.parsed > 0) {
     const missing: string[] = [];
@@ -6295,6 +6519,14 @@ async function commandProfile(args: Args, config: TrazumConfig, pricing: Pricing
     if (coverage.session < coverage.parsed) {
       missing.push(t.profile.needsSession(partial(coverage.session)));
     }
+    /**
+     * Listed first among the missing when it is missing entirely, because it
+     * is the one field that changes what every other figure here *means*. The
+     * rest sharpen a cost; this one gives it a counterpart.
+     */
+    if (coverage.outcome < coverage.parsed) {
+      missing.push(t.profile.needsOutcome(partial(coverage.outcome)));
+    }
     if (coverage.ts < coverage.parsed) {
       missing.push(t.profile.needsTs(partial(coverage.ts)));
     }
@@ -8014,6 +8246,22 @@ async function main(): Promise<void> {
     };
   }
   const { config } = loaded;
+  /**
+   * Where the waiver record lives: **beside the config that declared it**.
+   *
+   * It used to be the process's working directory, which is a different place
+   * whenever somebody runs `trazum profile ../logs/x.jsonl --config ../repo/
+   * trazum.config.json` — and that is not hypothetical. This repository's own
+   * test suite did exactly that from `packages/cli`, so sixty records of a
+   * fixture's decisions accumulated in a package directory and one of them was
+   * committed to `main`, where it sat for two releases.
+   *
+   * A waiver is a decision a *repository* made. The record of using it belongs
+   * with the file that made it, not with wherever the terminal happened to be.
+   * No config means no waivers, so there is nothing to write and `.` is never
+   * reached.
+   */
+  const configDir = loaded.path === null ? '.' : dirname(loaded.path);
   const pricing = await pricingFor(args, loaded, t);
 
   // The config only gets to choose the locale when nothing more explicit did.
@@ -8033,7 +8281,7 @@ async function main(): Promise<void> {
       await commandBaseline(args, config, pricing, t, locale);
       break;
     case 'profile':
-      await commandProfile(args, config, pricing, t);
+      await commandProfile(args, config, configDir, pricing, t);
       break;
     case 'plan':
       await commandPlan(args, pricing, t);
@@ -8042,7 +8290,7 @@ async function main(): Promise<void> {
       await commandVerify(args, pricing, t);
       break;
     case 'history':
-      await commandHistory(args, config, pricing, t);
+      await commandHistory(args, config, configDir, pricing, t);
       break;
     case 'connect':
       await commandConnect(args, pricing, t);
@@ -8071,6 +8319,9 @@ async function main(): Promise<void> {
     case 'models':
       commandModels(t, pricing);
       break;
+    case 'gateway':
+      await commandGateway(args, config, configDir, pricing, t);
+      break;
     case 'feedback':
       commandFeedback(t);
       break;