@trazum/core 1.26.0 → 1.28.0

package/src/truncation-retry.ts

@@ -0,0 +1,175 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+import type { PricingCatalogue } from './pricing.js';
+import type { UsageRecord } from './usage.js';
+
+/**
+ * The retry bill of truncation.
+ *
+ * ## The half of the cost the truncation line could not see
+ *
+ * `truncatedOutputUsd` prices the answers cut off at `max_tokens` — paid in
+ * full, and the attempt bought nothing. The sentence next to it has always
+ * said "frequently retried, billed again", and that second half was an
+ * assertion rather than a measurement: nothing counted the retries.
+ *
+ * This does. A truncated answer followed **within a couple of minutes by
+ * another call in the same conversation** is the shape a retry has — the
+ * user or the harness asked again, usually with a raised ceiling — and both
+ * sides of that pair are in the log. The first call's full price is money
+ * that bought a cut-off answer; the follow-up is the same question billed a
+ * second time, at conversation prices, with the history re-sent.
+ *
+ * ## What it compares, and why only that
+ *
+ * Each call is checked against the one immediately before it in the same
+ * session — the bounded-memory design `repeats.ts` uses, for the same
+ * reasons — and only when the previous call was truncated and the gap is
+ * non-negative and inside the window. Two minutes by default, wider than the
+ * repeat window: a human rephrasing after a cut-off answer takes longer than
+ * a harness retrying a timeout.
+ *
+ * ## What it refuses to conclude
+ *
+ * It cannot see content, so it cannot tell a retry from a user changing the
+ * subject right after a truncated answer. The pair is a pattern, stated as
+ * one. A single pair is not reported — one retry is an anecdote — and the
+ * per-slice denominator (`truncatedCalls` that *could* be checked) travels
+ * with the count, so "3 of 40" and "3 of 3" read as differently as they are.
+ */
+
+/** Truncated answers followed by another call in the same conversation. */
+export interface TruncationRetry {
+  label: string;
+  model: string;
+  modelName: string;
+  /** Truncated calls that were followed up inside the window. */
+  retried: number;
+  /** Truncated calls in this slice that carried a session and a clock. */
+  truncatedCalls: number;
+  /** The full price of the truncated attempts that were followed up. */
+  wastedUsd: number;
+  /** The full price of the follow-up calls. */
+  retryUsd: number;
+  /** The window the follow-up had to fall inside, in milliseconds. */
+  withinMs: number;
+}
+
+export interface TruncationRetryOptions {
+  catalogue: PricingCatalogue;
+  on?: Date;
+  /**
+   * How close the follow-up has to be. Two minutes by default — a human
+   * rephrasing after a cut-off answer takes longer than a harness retrying,
+   * and past a couple of minutes the next call is a next question.
+   */
+  withinMs?: number;
+  /** Slices below this many retried pairs are dropped. Default 2. */
+  minRetried?: number;
+}
+
+export interface TruncationRetryTracker {
+  add(record: UsageRecord): void;
+  finish(): TruncationRetry[];
+}
+
+interface Slice {
+  retried: number;
+  truncatedCalls: number;
+  wastedUsd: number;
+  retryUsd: number;
+}
+
+/** An accumulator, fed in the pass a profile already makes. */
+export function createTruncationRetryTracker(options: TruncationRetryOptions): TruncationRetryTracker {
+  const { catalogue, on = new Date(), withinMs = 120_000, minRetried = 2 } = options;
+  const slices = new Map<string, Slice>();
+  /** The previous call of each session: whether it truncated, when, its price, and whose slice it was. */
+  const previous = new Map<string, { truncated: boolean; ts: number; usd: number; key: string }>();
+
+  const priceOf = (record: UsageRecord): number | null => {
+    const model = catalogue.byId.get(record.model);
+    if (!model) return null;
+    const { inputPerMTok, outputPerMTok } = effectivePricing(model, on);
+    const rates = multipliersFor(model);
+    const per = (count: number, rate: number): number => (count / 1_000_000) * rate;
+    return (
+      per(record.inputTokens, inputPerMTok) +
+      per(record.cacheReadTokens, inputPerMTok * rates.cacheRead) +
+      per(record.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
+      per(record.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h) +
+      per(record.outputTokens, outputPerMTok)
+    );
+  };
+
+  const add = (record: UsageRecord): void => {
+    if (record.session === null || record.ts === null) return;
+    const usd = priceOf(record);
+    // An unpriced model has no dollars anywhere else in the report; a retry
+    // bill stated for it would be a figure with nothing behind it.
+    if (usd === null) return;
+
+    const key = `${record.label ?? UNLABELLED}\n${record.model}`;
+    const before = previous.get(record.session);
+    previous.set(record.session, { truncated: record.truncated === true, ts: record.ts, usd, key });
+
+    if (record.truncated === true) {
+      let slice = slices.get(key);
+      if (!slice) {
+        slice = { retried: 0, truncatedCalls: 0, wastedUsd: 0, retryUsd: 0 };
+        slices.set(key, slice);
+      }
+      slice.truncatedCalls += 1;
+    }
+
+    if (before === undefined || !before.truncated) return;
+    const gap = record.ts - before.ts;
+    // Out of order is not a retry, and neither is tomorrow's next question.
+    if (gap < 0 || gap >= withinMs) return;
+
+    // Attributed to the slice of the *truncated* call: that is where the
+    // ceiling that caused this lives, and where the fix is applied.
+    let slice = slices.get(before.key);
+    if (!slice) {
+      slice = { retried: 0, truncatedCalls: 0, wastedUsd: 0, retryUsd: 0 };
+      slices.set(before.key, slice);
+    }
+    slice.retried += 1;
+    slice.wastedUsd += before.usd;
+    slice.retryUsd += usd;
+  };
+
+  const finish = (): TruncationRetry[] => {
+    const out: TruncationRetry[] = [];
+    for (const [key, slice] of slices) {
+      if (slice.retried < minRetried) continue;
+      const split = key.indexOf('\n');
+      const modelId = key.slice(split + 1);
+      const model = catalogue.byId.get(modelId);
+      if (!model) continue;
+      out.push({
+        label: key.slice(0, split),
+        model: modelId,
+        modelName: model.displayName,
+        retried: slice.retried,
+        truncatedCalls: slice.truncatedCalls,
+        wastedUsd: slice.wastedUsd,
+        retryUsd: slice.retryUsd,
+        withinMs,
+      });
+    }
+    return out.sort((a, b) => b.wastedUsd + b.retryUsd - (a.wastedUsd + a.retryUsd));
+  };
+
+  return { add, finish };
+}
+
+/** The same measurement over a list of records, for a caller holding one. */
+export function truncationRetries(
+  records: readonly UsageRecord[],
+  options: TruncationRetryOptions,
+): TruncationRetry[] {
+  const tracker = createTruncationRetryTracker(options);
+  for (const record of records) tracker.add(record);
+  return tracker.finish();
+}

package/src/usage.ts

@@ -3,6 +3,7 @@ import { createConversationTracker } from './conversation.js';
 import { createOutputShapeTracker } from './output-shape.js';
 import { createInputShapeTracker } from './input-shape.js';
 import { createRepeatsTracker } from './repeats.js';
+import { createTruncationRetryTracker } from './truncation-retry.js';
 import { createTtlFitTracker } from './ttl-fit.js';
 import { createSessionLedgerTracker } from './session-ledger.js';
 import { createSessionCostTracker } from './session-cost.js';
@@ -13,6 +14,7 @@ import type { ConversationGrowth } from './conversation.js';
 import type { OutputShape } from './output-shape.js';
 import type { InputShape } from './input-shape.js';
 import type { RepeatedTurns } from './repeats.js';
+import type { TruncationRetry } from './truncation-retry.js';
 import type { PricingCatalogue } from './pricing.js';
 
 /**
@@ -313,6 +315,15 @@ export interface UsageProfileReport {
    * carries neither, which is a different statement from "none happened".
    */
   repeatedTurns: RepeatedTurns[];
+  /**
+   * Truncated answers followed within two minutes by another call in the
+   * same conversation — the "frequently retried, billed again" half of the
+   * truncation finding, measured instead of asserted. The wasted attempt's
+   * full price and the follow-up's travel together, with the checkable
+   * denominator. Needs a session, a clock and a stop reason; a pattern,
+   * never a certainty — the log cannot see content.
+   */
+  truncationRetries: TruncationRetry[];
   /**
    * The period the log covers, when its records carry a clock, over every
    * parsed record — priced and unpriced alike, because when a call happened is
@@ -346,6 +357,12 @@ export interface UsageProfileReport {
     /** The label that spent the most this day, or null when nothing had one. */
     topLabel: string | null;
     topLabelUsd: number;
+    /**
+     * The day's spend per model, largest first — the series `modelMixDrift`
+     * summarises, exposed whole so a spreadsheet or a chart can draw the
+     * migration day by day instead of in two halves.
+     */
+    byModel: Array<{ model: string; usd: number; calls: number }>;
   }>;
   /**
    * Lines that are exact duplicates of an earlier line, and what they added
@@ -417,6 +434,35 @@ export interface UsageProfileReport {
    * would be guessing.
    */
   spendByHour: Array<{ hour: number; usd: number; calls: number }>;
+  /**
+   * How the model mix moved across this log's own span — the drift `--against`
+   * can only see with a second log.
+   *
+   * A bill can grow with no workload growing: traffic quietly migrating from
+   * the cheap model to the expensive one, a deploy that flipped a default, a
+   * fallback that became the main path. Day totals cannot show it (both
+   * models land in the same number) and per-model totals cannot either (a
+   * total has no direction). So this splits the log's days into two halves,
+   * chronologically, and states each model's **share of the priced spend** in
+   * each half, exactly.
+   *
+   * `null` — not empty — when the log has fewer than four days with priced,
+   * dated spend: two points per half is the least a "half" can honestly
+   * claim, and a drift computed over one day against one day would be
+   * weather presented as climate. The renderings decide what counts as
+   * "moved"; the data states the shares and stops. No forecast: where the
+   * mix goes next is not in the log.
+   */
+  modelMixDrift: {
+    /** Days in each half. `firstDays + lastDays` = every day with priced spend. */
+    firstDays: number;
+    lastDays: number;
+    /** Priced spend in each half, so a share can be turned back into money. */
+    firstUsd: number;
+    lastUsd: number;
+    /** Per model, every model either half saw. Shares are of that half's spend. */
+    models: Array<{ model: string; firstShare: number; lastShare: number; firstUsd: number; lastUsd: number }>;
+  } | null;
   /**
    * Whether each slice's cache TTL fits how fast its turns arrive — the
    * mechanism behind a losing cache verdict, and the one place an overlong TTL
@@ -876,6 +922,7 @@ export function profileUsage(text: string, options: UsageProfileOptions): UsageP
   const output = createOutputShapeTracker({ catalogue, on });
   const input = createInputShapeTracker({ catalogue, on });
   const repeats = createRepeatsTracker({ catalogue, on });
+  const truncRetries = createTruncationRetryTracker({ catalogue, on });
   const ttlFit = createTtlFitTracker({ catalogue, on });
   const ledger = createSessionLedgerTracker({ catalogue, on });
   const sessionCosts = createSessionCostTracker({ catalogue, on });
@@ -892,7 +939,7 @@ export function profileUsage(text: string, options: UsageProfileOptions): UsageP
   let spanTo = -Infinity;
   let spanCalls = 0;
   /** Per UTC day: spend, calls, and spend per label. Bounded by days × labels. */
-  const days = new Map<string, { usd: number; calls: number; byLabel: Map<string, number> }>();
+  const days = new Map<string, { usd: number; calls: number; byLabel: Map<string, number>; byModel: Map<string, { usd: number; calls: number }> }>();
   /** Per hour of the UTC day. Bounded by twenty-four entries, whatever the log. */
   const hours = new Map<number, { usd: number; calls: number }>();
 
@@ -952,6 +999,7 @@ export function profileUsage(text: string, options: UsageProfileOptions): UsageP
     output.add(record);
     input.add(record);
     repeats.add(record);
+    truncRetries.add(record);
     ttlFit.add(record);
     ledger.add(record);
     sessionCosts.add(record);
@@ -991,13 +1039,17 @@ export function profileUsage(text: string, options: UsageProfileOptions): UsageP
       const usd = total.totalUsd - usdBefore;
       let entry = days.get(day);
       if (!entry) {
-        entry = { usd: 0, calls: 0, byLabel: new Map() };
+        entry = { usd: 0, calls: 0, byLabel: new Map(), byModel: new Map() };
         days.set(day, entry);
       }
       entry.usd += usd;
       entry.calls += 1;
       const labelKey = record.label ?? UNLABELLED;
       entry.byLabel.set(labelKey, (entry.byLabel.get(labelKey) ?? 0) + usd);
+      const modelCell = entry.byModel.get(record.model) ?? { usd: 0, calls: 0 };
+      modelCell.usd += usd;
+      modelCell.calls += 1;
+      entry.byModel.set(record.model, modelCell);
 
       // The same exact per-record dollar, bucketed by hour of the UTC day.
       const hour = new Date(record.ts).getUTCHours();
@@ -1050,6 +1102,7 @@ export function profileUsage(text: string, options: UsageProfileOptions): UsageP
     outputShapes: output.finish(total.totalUsd),
     inputShapes: input.finish(total.totalUsd),
     repeatedTurns: repeats.finish(),
+    truncationRetries: truncRetries.finish(),
     span: spanCalls > 0 ? { fromMs: spanFrom, toMs: spanTo, calls: spanCalls } : null,
     spendByDay: [...days.entries()]
       .sort((a, b) => a[0].localeCompare(b[0]))
@@ -1062,10 +1115,56 @@ export function profileUsage(text: string, options: UsageProfileOptions): UsageP
             topLabelUsd = usd;
           }
         }
-        return { day, usd: entry.usd, calls: entry.calls, topLabel, topLabelUsd };
+        return {
+          day,
+          usd: entry.usd,
+          calls: entry.calls,
+          topLabel,
+          topLabelUsd,
+          byModel: [...entry.byModel.entries()]
+            .map(([model, cell]) => ({ model, usd: cell.usd, calls: cell.calls }))
+            .sort((a, b) => b.usd - a.usd),
+        };
       }),
     duplicateLines: duplicates,
     fieldCoverage: coverage,
+    modelMixDrift: (() => {
+      const ordered = [...days.entries()].sort((a, b) => a[0].localeCompare(b[0]));
+      if (ordered.length < 4) return null;
+      const mid = Math.floor(ordered.length / 2);
+      const halves = [ordered.slice(0, mid), ordered.slice(mid)] as const;
+      const totals = halves.map((half) => half.reduce((sum, [, e]) => sum + e.usd, 0));
+      // A half with no priced spend has no shares to state — division by zero
+      // is not a drift, and neither is a mix over zero dollars.
+      if (totals[0]! <= 0 || totals[1]! <= 0) return null;
+      const perModel = new Map<string, { first: number; last: number }>();
+      halves.forEach((half, index) => {
+        for (const [, entry] of half) {
+          for (const [model, dayCell] of entry.byModel) {
+            const cell = perModel.get(model) ?? { first: 0, last: 0 };
+            if (index === 0) cell.first += dayCell.usd;
+            else cell.last += dayCell.usd;
+            perModel.set(model, cell);
+          }
+        }
+      });
+      return {
+        firstDays: halves[0].length,
+        lastDays: halves[1].length,
+        firstUsd: totals[0]!,
+        lastUsd: totals[1]!,
+        models: [...perModel.entries()]
+          .map(([model, cell]) => ({
+            model,
+            firstShare: cell.first / totals[0]!,
+            lastShare: cell.last / totals[1]!,
+            firstUsd: cell.first,
+            lastUsd: cell.last,
+          }))
+          // The biggest movement first — the order a reader would act in.
+          .sort((a, b) => Math.abs(b.lastShare - b.firstShare) - Math.abs(a.lastShare - a.firstShare)),
+      };
+    })(),
     spendByHour: [...hours.entries()]
       .sort((a, b) => a[0] - b[0])
       .map(([hour, entry]) => ({ hour, usd: entry.usd, calls: entry.calls })),