@trazum/core 1.10.0 → 1.26.0

package/src/repeats.ts

@@ -0,0 +1,166 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+import type { PricingCatalogue } from './pricing.js';
+import type { UsageRecord } from './usage.js';
+
+/**
+ * The same request, sent again a moment later.
+ *
+ * ## The failure mode nothing else here can see
+ *
+ * A conversation's input **grows with every turn** — that is the finding
+ * `conversations` exists for. So two consecutive calls in one conversation
+ * carrying *exactly* the same input size, seconds apart, is the shape of
+ * something going wrong rather than something working: a retry after a
+ * timeout, an agent step repeating because a tool call failed, a loop that
+ * re-sends the whole context and gets nowhere.
+ *
+ * That is expensive in a way a total hides completely. The retried call is
+ * billed in full, and on an agent workload the input is the bill.
+ * `duplicateLines` catches the same line recorded twice; this catches two
+ * *different* calls that sent the same thing.
+ *
+ * ## What it compares, and why only that
+ *
+ * Each call is compared to **the one immediately before it in the same
+ * session**, and only when the log carries both a session and a clock. Not to
+ * every earlier call: a workload that legitimately sends a fixed-size prompt
+ * would light up under a looser rule, and holding every size a session has
+ * ever sent would grow without bound on a log measured in megabytes. One
+ * previous call per session is bounded and is where a retry actually sits.
+ *
+ * The gap must be **non-negative and under the window** (a minute by default).
+ * Non-negative because a log is not guaranteed to be in time order, and a
+ * negative gap means the two records arrived out of order rather than that a
+ * call repeated.
+ *
+ * ## What it refuses to conclude
+ *
+ * It cannot see content, so it cannot tell a retry from two genuinely
+ * identical requests a second apart. It reports the count and the money and
+ * stops — the same rule `duplicateLines` follows. Every rendering says the
+ * pattern is *usually* a retry or a loop, never that it is one.
+ */
+
+/** Consecutive same-size calls in one conversation, per slice. */
+export interface RepeatedTurns {
+  label: string;
+  model: string;
+  modelName: string;
+  /** How many calls repeated the previous call's input size inside the window. */
+  repeats: number;
+  /** Calls in this slice that could be checked at all — the denominator. */
+  checkedCalls: number;
+  /** What those repeated calls cost, in full. */
+  usd: number;
+  /** The window they had to fall inside, in milliseconds. */
+  withinMs: number;
+}
+
+export interface RepeatsOptions {
+  catalogue: PricingCatalogue;
+  on?: Date;
+  /**
+   * How close two calls have to be. A minute by default: long enough to cover
+   * a timeout and a retry, short enough that an ordinary next turn — which
+   * needs a human or a tool to produce it — rarely lands inside it.
+   */
+  withinMs?: number;
+  /** Slices below this many repeats are dropped. Default 2. */
+  minRepeats?: number;
+}
+
+export interface RepeatsTracker {
+  add(record: UsageRecord): void;
+  finish(): RepeatedTurns[];
+}
+
+interface Slice {
+  repeats: number;
+  checkedCalls: number;
+  usd: number;
+}
+
+/** An accumulator, fed in the pass a profile already makes. */
+export function createRepeatsTracker(options: RepeatsOptions): RepeatsTracker {
+  const { catalogue, on = new Date(), withinMs = 60_000, minRepeats = 2 } = options;
+  const slices = new Map<string, Slice>();
+  /** The previous call of each session: its input size and when it happened. */
+  const previous = new Map<string, { tokens: number; ts: number }>();
+
+  const add = (record: UsageRecord): void => {
+    const model = catalogue.byId.get(record.model);
+    // An unpriced model has no dollars anywhere else; naming money here that
+    // no total contains would be a figure with nothing behind it.
+    if (!model) return;
+    if (record.session === null || record.ts === null) return;
+
+    const tokens =
+      record.inputTokens +
+      record.cacheReadTokens +
+      record.cacheWrite5mTokens +
+      record.cacheWrite1hTokens;
+    if (tokens <= 0) return;
+
+    const key = `${record.label ?? UNLABELLED}\n${record.model}`;
+    let slice = slices.get(key);
+    if (!slice) {
+      slice = { repeats: 0, checkedCalls: 0, usd: 0 };
+      slices.set(key, slice);
+    }
+    slice.checkedCalls += 1;
+
+    const before = previous.get(record.session);
+    previous.set(record.session, { tokens, ts: record.ts });
+    if (before === undefined) return;
+
+    const gap = record.ts - before.ts;
+    // Out of order is not a repeat, and neither is a call an hour later.
+    if (gap < 0 || gap >= withinMs) return;
+    if (before.tokens !== tokens) return;
+
+    const { inputPerMTok, outputPerMTok } = effectivePricing(model, on);
+    const rates = multipliersFor(model);
+    const per = (count: number, rate: number): number => (count / 1_000_000) * rate;
+    slice.repeats += 1;
+    slice.usd +=
+      per(record.inputTokens, inputPerMTok) +
+      per(record.cacheReadTokens, inputPerMTok * rates.cacheRead) +
+      per(record.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
+      per(record.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h) +
+      per(record.outputTokens, outputPerMTok);
+  };
+
+  const finish = (): RepeatedTurns[] => {
+    const out: RepeatedTurns[] = [];
+    for (const [key, slice] of slices) {
+      if (slice.repeats < minRepeats) continue;
+      const split = key.indexOf('\n');
+      const modelId = key.slice(split + 1);
+      const model = catalogue.byId.get(modelId);
+      if (!model) continue;
+      out.push({
+        label: key.slice(0, split),
+        model: modelId,
+        modelName: model.displayName,
+        repeats: slice.repeats,
+        checkedCalls: slice.checkedCalls,
+        usd: slice.usd,
+        withinMs,
+      });
+    }
+    return out.sort((a, b) => b.usd - a.usd);
+  };
+
+  return { add, finish };
+}
+
+/** The same measurement over a list of records, for a caller holding one. */
+export function repeatedTurns(
+  records: readonly UsageRecord[],
+  options: RepeatsOptions,
+): RepeatedTurns[] {
+  const tracker = createRepeatsTracker(options);
+  for (const record of records) tracker.add(record);
+  return tracker.finish();
+}

package/src/reprice.ts

@@ -0,0 +1,227 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import type { PricingCatalogue } from './pricing.js';
+import type { ModelPricing } from './types.js';
+import type { UsageBreakdown, UsageProfileReport } from './usage.js';
+
+/**
+ * The same tokens at another model's rates.
+ *
+ * ## Why this is allowed to exist
+ *
+ * `usage.ts` refuses to report a saving, and the reason is that a saving
+ * requires imagining a prompt nobody wrote. This does not. Every token here
+ * was actually billed; the only thing being changed is the rate card it is
+ * multiplied by, which is the same move `cacheEconomics` makes and is
+ * arithmetic rather than a guess.
+ *
+ * What it answers is the question a bill provokes and a total cannot settle:
+ * `classify` spent $4,000 on a frontier model — what would those exact calls
+ * have cost on the small one? That is a number, and it is the number a routing
+ * decision is argued over.
+ *
+ * ## The four things it refuses
+ *
+ * **It says nothing about whether the answers would be as good.** This is
+ * multiplication. Whether the cheap model can do the work is a question about
+ * the work, and Trazum has never seen the prompts — by design, there is nowhere
+ * in a usage record to put one. Every rendering states this next to the figure,
+ * because a dollar number with no caveat attached reads as a recommendation.
+ *
+ * **It refuses to price traffic that would not fit.** A cheaper model with a
+ * smaller context window does not make a 400k-token call cheaper; it makes it
+ * impossible. Slices holding a call larger than the target's window are pulled
+ * out into `overContext` and their money is excluded from every total here —
+ * counting an impossible call's price difference as a saving is exactly the
+ * flattering direction this repository refuses. `maxCallInputTokens` is the
+ * maximum rather than an average for the same reason: one call over the ceiling
+ * is a failed call, and a mean hides it.
+ *
+ * **It excludes what is already there.** Calls billed on the target model
+ * reprice to themselves, and folding them in would pad both totals with money
+ * that cannot move — a $10,000 bill of which $9,900 is already on the cheap
+ * model would report a 1% difference and read as "not worth doing". They are
+ * counted in `alreadyOnTarget` instead, so the reader sees the shape.
+ *
+ * **It assumes the token counts survive the move, and says so.** A different
+ * model tokenizes differently, and one that is worse at the task may answer at
+ * greater length or be retried. The counts are the ones that were billed, not a
+ * prediction; that makes this an exact restatement of the past at another price
+ * and an approximation of the future. Stated, not hidden — `sameTokensAssumed`
+ * exists so no rendering can forget to say it.
+ *
+ * ## Why the write TTLs are kept apart
+ *
+ * A cache write costs 1.25x input at five minutes and 2x at an hour, and that
+ * ratio is not a constant across providers. Repricing a combined write total
+ * would mean picking one of the two rates for tokens that were billed at both,
+ * so `UsageBreakdown` carries the split and this reads it. Writes whose TTL the
+ * log never stated are in the 5-minute bucket, the same assumption
+ * `cacheWriteUsdIfAssumed1h` prices — `assumedWriteTtlCalls` is carried through
+ * so a reader knows the comparison inherits it.
+ */
+
+/** One label-and-model slice, as billed and as it would have been billed. */
+export interface RepricedSlice {
+  label: string;
+  /** The model these calls were actually billed on. */
+  model: string;
+  calls: number;
+  /** What the log says this slice cost. */
+  currentUsd: number;
+  /** What the same tokens cost at the target's rates. */
+  targetUsd: number;
+  /** `targetUsd - currentUsd`. Negative is cheaper on the target. */
+  deltaUsd: number;
+  /** The slice's largest single call, cache reads and writes included. */
+  maxCallInputTokens: number;
+}
+
+/** A slice holding a call the target model could not have accepted. */
+export interface OverContextSlice {
+  label: string;
+  model: string;
+  calls: number;
+  currentUsd: number;
+  /** The call that does not fit, in tokens. */
+  maxCallInputTokens: number;
+}
+
+export interface RepriceReport {
+  target: {
+    id: string;
+    displayName: string;
+    /** The ceiling `overContext` was judged against. */
+    contextWindow: number;
+  };
+  /**
+   * Slices that could move, largest saving first. Ties break on the larger
+   * bill, so the row somebody would act on comes first.
+   */
+  slices: RepricedSlice[];
+  /** Totals over `slices` only — the money that would actually change hands. */
+  currentUsd: number;
+  targetUsd: number;
+  deltaUsd: number;
+  /**
+   * Slices excluded because at least one of their calls is larger than the
+   * target's context window. Their money is in none of the totals above.
+   */
+  overContext: OverContextSlice[];
+  /** Calls already billed on the target, and what they cost. Not repriced. */
+  alreadyOnTarget: { calls: number; usd: number };
+  /**
+   * Repriced calls whose write TTL the log did not state. Non-zero means both
+   * sides of the comparison rest on the cheaper assumption.
+   */
+  assumedWriteTtlCalls: number;
+  /**
+   * Models in the log the catalogue could not price, and how many calls they
+   * made. Their tokens could be priced on the target, but the difference
+   * cannot be — there is no current figure to subtract from. Named rather
+   * than dropped, so a comparison covering half a bill cannot look complete.
+   */
+  unpricedModels: string[];
+  unpricedCalls: number;
+  /** Always true. A field, not a comment, so a rendering can print it. */
+  sameTokensAssumed: true;
+}
+
+/**
+ * What a set of token counts costs on one model, at the rates in force on a
+ * date.
+ *
+ * The same arithmetic `profileUsage` does per call, over an aggregate — which
+ * is only sound because every token class is priced independently of the
+ * others and of how many calls produced them.
+ */
+export function priceTokensOn(
+  breakdown: Pick<
+    UsageBreakdown,
+    'inputTokens' | 'cacheReadTokens' | 'cacheWrite5mTokens' | 'cacheWrite1hTokens' | 'outputTokens'
+  >,
+  model: ModelPricing,
+  on: Date = new Date(),
+): number {
+  const { inputPerMTok, outputPerMTok } = effectivePricing(model, on);
+  const rates = multipliersFor(model);
+  const per = (tokens: number, rate: number): number => (tokens / 1_000_000) * rate;
+  return (
+    per(breakdown.inputTokens, inputPerMTok) +
+    per(breakdown.cacheReadTokens, inputPerMTok * rates.cacheRead) +
+    per(breakdown.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
+    per(breakdown.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h) +
+    per(breakdown.outputTokens, outputPerMTok)
+  );
+}
+
+/**
+ * Reprices a profile's label-and-model slices onto one target model.
+ *
+ * Returns `null` when the catalogue does not know the target: a comparison
+ * against a price nobody has is worse than no comparison, and the caller is
+ * better placed to say so in its own words than this is to invent a zero.
+ */
+export function repriceProfile(
+  report: UsageProfileReport,
+  targetId: string,
+  catalogue: PricingCatalogue,
+  on: Date = new Date(),
+): RepriceReport | null {
+  const target = catalogue.byId.get(targetId);
+  if (!target) return null;
+
+  const slices: RepricedSlice[] = [];
+  const overContext: OverContextSlice[] = [];
+  const alreadyOnTarget = { calls: 0, usd: 0 };
+  let assumedWriteTtlCalls = 0;
+
+  for (const slice of report.byLabelAndModel) {
+    const { breakdown } = slice;
+    if (slice.model === target.id) {
+      alreadyOnTarget.calls += breakdown.calls;
+      alreadyOnTarget.usd += breakdown.totalUsd;
+      continue;
+    }
+    if (breakdown.maxCallInputTokens > target.contextWindow) {
+      overContext.push({
+        label: slice.label,
+        model: slice.model,
+        calls: breakdown.calls,
+        currentUsd: breakdown.totalUsd,
+        maxCallInputTokens: breakdown.maxCallInputTokens,
+      });
+      continue;
+    }
+    const targetUsd = priceTokensOn(breakdown, target, on);
+    assumedWriteTtlCalls += breakdown.assumedWriteTtlCalls;
+    slices.push({
+      label: slice.label,
+      model: slice.model,
+      calls: breakdown.calls,
+      currentUsd: breakdown.totalUsd,
+      targetUsd,
+      deltaUsd: targetUsd - breakdown.totalUsd,
+      maxCallInputTokens: breakdown.maxCallInputTokens,
+    });
+  }
+
+  slices.sort((a, b) => a.deltaUsd - b.deltaUsd || b.currentUsd - a.currentUsd);
+  overContext.sort((a, b) => b.currentUsd - a.currentUsd);
+
+  const currentUsd = slices.reduce((sum, s) => sum + s.currentUsd, 0);
+  const targetUsd = slices.reduce((sum, s) => sum + s.targetUsd, 0);
+
+  return {
+    target: { id: target.id, displayName: target.displayName, contextWindow: target.contextWindow },
+    slices,
+    currentUsd,
+    targetUsd,
+    deltaUsd: targetUsd - currentUsd,
+    overContext,
+    alreadyOnTarget,
+    assumedWriteTtlCalls,
+    unpricedModels: report.unpricedModels,
+    unpricedCalls: report.unpriced.calls,
+    sameTokensAssumed: true,
+  };
+}

package/src/session-cost.ts

@@ -0,0 +1,170 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+import type { PricingCatalogue } from './pricing.js';
+import type { UsageRecord } from './usage.js';
+
+/**
+ * What one conversation costs.
+ *
+ * ## The question a total cannot answer
+ *
+ * "Support cost $4,000 last month" does not say whether that is forty thousand
+ * cheap conversations or four hundred expensive ones, and every decision made
+ * on top of it needs the answer: what to charge per seat, where to put a quota,
+ * whether one runaway agent loop is eating the budget. The bill has the data —
+ * the log groups by `session` already — and nothing was reporting it.
+ *
+ * ## Median and p95, not mean
+ *
+ * A mean conversation cost is the total divided by the session count, which is
+ * the total again wearing a hat: one 400-turn agent loop drags it up and hides
+ * the ordinary case. The **median** is the conversation in the middle — what a
+ * typical one costs — and the **p95** is the one a quota has to survive. The
+ * gap between them is the finding: `$0.02 median, $1.80 p95` is a workload with
+ * a tail worth hunting; `$0.40 median, $0.55 p95` is a workload that is simply
+ * expensive, and no amount of tail-hunting will fix it.
+ *
+ * Every figure is **exact** — the provider's own billed counts, summed per
+ * conversation at each model's published rates. No counterfactual, no estimate.
+ *
+ * ## What it refuses to claim
+ *
+ * A conversation that started before this log or continues after it is counted
+ * only for the turns recorded here, so its cost is a floor. That is stated
+ * rather than corrected: guessing at unseen turns would be exactly the kind of
+ * invention this package exists to end. Session keys group turns and never
+ * leave this module, as everywhere the field is touched.
+ */
+
+export interface SessionCostShape {
+  label: string;
+  model: string;
+  modelName: string;
+  /** Conversations measured. Never which ones. */
+  sessions: number;
+  calls: number;
+  /** What those conversations cost in total — exact, billed. */
+  totalUsd: number;
+  /** The conversation in the middle. */
+  medianUsd: number;
+  /** The conversation a quota has to survive: 95th percentile, by nearest rank. */
+  p95Usd: number;
+  /** The single most expensive conversation in the slice. */
+  maxUsd: number;
+  /** Turns in the median conversation, for scale. */
+  medianTurns: number;
+}
+
+export interface SessionCostOptions {
+  catalogue: PricingCatalogue;
+  on?: Date;
+  /**
+   * Slices with fewer conversations than this are dropped: a median over three
+   * sessions is not a median, it is one of the three, and a p95 over them is
+   * the maximum wearing a percentile's name. Default 5.
+   */
+  minSessions?: number;
+}
+
+export interface SessionCostTracker {
+  add(record: UsageRecord): void;
+  finish(): SessionCostShape[];
+}
+
+/** Every billed dollar of one call, at its own model's rates. */
+function costOf(record: UsageRecord, catalogue: PricingCatalogue, on: Date): number | null {
+  const model = catalogue.byId.get(record.model);
+  if (!model) return null;
+  const { inputPerMTok, outputPerMTok } = effectivePricing(model, on);
+  const rates = multipliersFor(model);
+  const per = (tokens: number, rate: number): number => (tokens / 1_000_000) * rate;
+  return (
+    per(record.inputTokens, inputPerMTok) +
+    per(record.cacheReadTokens, inputPerMTok * rates.cacheRead) +
+    per(record.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
+    per(record.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h) +
+    per(record.outputTokens, outputPerMTok)
+  );
+}
+
+const median = (sorted: number[]): number => {
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
+};
+
+/**
+ * The 95th percentile by **nearest rank**: the smallest value at or above which
+ * 95% of the conversations sit. Interpolating between two conversations would
+ * report a cost no conversation had, and this figure exists to be compared
+ * against a real quota.
+ */
+const p95 = (sorted: number[]): number =>
+  sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)]!;
+
+export function createSessionCostTracker(options: SessionCostOptions): SessionCostTracker {
+  const { catalogue, on = new Date(), minSessions = 5 } = options;
+  const slices = new Map<string, Map<string, { usd: number; turns: number }>>();
+
+  const add = (record: UsageRecord): void => {
+    if (record.session === null) return;
+    const cost = costOf(record, catalogue, on);
+    // An unpriced model contributes no dollars anywhere else either.
+    if (cost === null) return;
+
+    const sliceKey = `${record.label ?? UNLABELLED}\n${record.model}`;
+    let sessions = slices.get(sliceKey);
+    if (!sessions) {
+      sessions = new Map();
+      slices.set(sliceKey, sessions);
+    }
+    const existing = sessions.get(record.session);
+    if (existing) {
+      existing.usd += cost;
+      existing.turns += 1;
+    } else {
+      sessions.set(record.session, { usd: cost, turns: 1 });
+    }
+  };
+
+  const finish = (): SessionCostShape[] => {
+    const out: SessionCostShape[] = [];
+
+    for (const [sliceKey, sessions] of slices) {
+      if (sessions.size < minSessions) continue;
+      const split = sliceKey.indexOf('\n');
+      const modelId = sliceKey.slice(split + 1);
+      const model = catalogue.byId.get(modelId);
+      if (!model) continue;
+
+      const costs = [...sessions.values()].map((s) => s.usd).sort((a, b) => a - b);
+      const turns = [...sessions.values()].map((s) => s.turns).sort((a, b) => a - b);
+      out.push({
+        label: sliceKey.slice(0, split),
+        model: modelId,
+        modelName: model.displayName,
+        sessions: sessions.size,
+        calls: turns.reduce((sum, t) => sum + t, 0),
+        totalUsd: costs.reduce((sum, c) => sum + c, 0),
+        medianUsd: median(costs),
+        p95Usd: p95(costs),
+        maxUsd: costs[costs.length - 1]!,
+        medianTurns: median(turns),
+      });
+    }
+
+    // The most money first — the order somebody would act in.
+    return out.sort((a, b) => b.totalUsd - a.totalUsd || a.label.localeCompare(b.label));
+  };
+
+  return { add, finish };
+}
+
+/** The same measurement over a list, for a caller holding one already. */
+export function sessionCostShapes(
+  records: readonly UsageRecord[],
+  options: SessionCostOptions,
+): SessionCostShape[] {
+  const tracker = createSessionCostTracker(options);
+  for (const record of records) tracker.add(record);
+  return tracker.finish();
+}