@trazum/core 1.50.5 → 1.50.7

package/src/experiment.ts

@@ -0,0 +1,275 @@
+/**
+ * Two arms on real traffic, and the three things that stop it being theatre.
+ *
+ * `eval` compares two prompts on cases somebody wrote; `route` compares two
+ * models on the same. Both measure agreement in a laboratory. The traffic is
+ * the only place the real question gets answered — and the moment a comparison
+ * runs on real traffic, three failures become available that a laboratory does
+ * not have.
+ *
+ * ## 1. A winner where there is none
+ *
+ * Two arms always produce two numbers, and one of them is always larger. An
+ * A/B report that names a winner from that is a coin flip with a dashboard.
+ * The verdict here is **three-valued**, the way `verify`'s has been since
+ * 1.39: A wins, B wins, or **not separable on this traffic** — and the third
+ * one comes with the number of outcomes per arm that *would* separate them, so
+ * "run it longer" is a quantified instruction rather than a shrug.
+ *
+ * ## 2. Peeking
+ *
+ * A test stopped on the first afternoon it looked good is not a test. The
+ * stopping rule is declared **before** the experiment starts, and the report
+ * says whether it was honoured. It cannot enforce that — nobody can stop
+ * somebody reading a number early — but it can make an early stop *visible*
+ * to whoever reads the result later, which is the part that matters.
+ *
+ * ## 3. Quality and cost judged apart, then together
+ *
+ * The interesting arm is almost never better *and* cheaper. It is better and
+ * dearer, and the decision needs one figure nobody computes: **what an extra
+ * success costs**. That is the marginal figure — the difference in spend over
+ * the difference in successes — and it is the number a product decision
+ * actually turns on, printed rather than left as an exercise.
+ *
+ * ## The statistics are shown, not asserted
+ *
+ * Wilson score intervals per arm and Newcombe's interval on the difference,
+ * because both behave at the small samples this will actually see. The
+ * intervals are returned, not just the verdict: a reader who disagrees with
+ * the threshold can see the numbers it was applied to, which is the same
+ * discipline `eval` established by running the original twice before judging
+ * anything against it.
+ */
+
+import { judgeOutcome } from './outcome.js';
+import type { OutcomeTally, OutcomeVocabulary } from './outcome.js';
+
+/** z for a two-sided 95% interval, and for 80% power. */
+const Z_95 = 1.959963984540054;
+const Z_POWER_80 = 0.8416212335729143;
+
+/**
+ * The declaration, made before the experiment runs.
+ *
+ * Its whole purpose is to exist *earlier* than the result. A stopping rule
+ * invented after looking at the numbers is not a stopping rule.
+ */
+export interface ExperimentDeclaration {
+  /** Two arms. More would need a multiple-comparison correction nobody asked for. */
+  arms: [string, string];
+  /**
+   * Outcomes each arm must record before the result may be read.
+   *
+   * Declared as a count rather than a duration, because a duration is a proxy
+   * for a count and the proxy breaks the week traffic doubles.
+   */
+  minOutcomesPerArm: number;
+}
+
+/** What one arm actually did. */
+export interface ExperimentArm {
+  name: string;
+  tally: OutcomeTally;
+  /** Everything this arm spent, recorded outcome or not. */
+  totalUsd: number;
+}
+
+export interface ArmResult {
+  name: string;
+  successes: number;
+  /** Calls carrying a *declared* outcome — the denominator. */
+  recorded: number;
+  /** Successes over recorded, or null when nothing was recorded. */
+  rate: number | null;
+  /** Wilson score interval on that rate, or null. */
+  interval: { low: number; high: number } | null;
+  /** Spend on calls carrying a declared outcome. */
+  recordedUsd: number;
+}
+
+export type Separation = 'a-wins' | 'b-wins' | 'not-separable';
+
+export type NotSeparableReason =
+  | 'interval-includes-zero'
+  | 'no-difference-observed'
+  | 'nothing-recorded';
+
+export interface Marginal {
+  /**
+   * What one extra success costs, going from the worse arm to the better one.
+   *
+   * `(dearer spend − cheaper spend) / (more successes − fewer successes)`,
+   * both per call so arms of different sizes compare. Null when the better arm
+   * is also the cheaper one, because then nothing is being bought and a
+   * "cost per extra success" would be a negative number people would quote.
+   */
+  usdPerExtraSuccess: number | null;
+  /** The arm that resolved more, by rate. */
+  better: string;
+  /** Whether that arm also costs more per call. */
+  dearer: boolean;
+}
+
+export interface ExperimentResult {
+  a: ArmResult;
+  b: ArmResult;
+  separation: Separation;
+  /** Why, when not separable. A refusal never arrives bare. */
+  notSeparable: NotSeparableReason | null;
+  /** Newcombe's interval on (rate A − rate B), or null. */
+  difference: { point: number; low: number; high: number } | null;
+  /**
+   * Outcomes **per arm** that would separate the observed difference, or null.
+   *
+   * Null when the arms recorded the same rate: no sample size separates a
+   * difference of zero, and returning a very large number would read as "keep
+   * going" when the honest answer is "there is nothing here to find".
+   */
+  outcomesNeededPerArm: number | null;
+  stopping: {
+    declared: number;
+    /** Whether both arms cleared the declared minimum. */
+    honoured: boolean;
+    /** The arm that has not, when one has not. */
+    short: string | null;
+  };
+  marginal: Marginal | null;
+}
+
+/** Wilson score interval — behaves at the sample sizes this will actually see. */
+function wilson(successes: number, n: number, z = Z_95): { low: number; high: number } | null {
+  if (n <= 0) return null;
+  const p = successes / n;
+  const denominator = 1 + (z * z) / n;
+  const centre = (p + (z * z) / (2 * n)) / denominator;
+  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denominator;
+  return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) };
+}
+
+function resultOf(arm: ExperimentArm, vocabulary: OutcomeVocabulary | null): ArmResult {
+  const declared = vocabulary ?? { values: [], success: [] };
+  let successes = 0;
+  let recorded = 0;
+  let recordedUsd = 0;
+  for (const entry of arm.tally.byValue) {
+    // Undeclared values are out of both halves, as everywhere since 1.50.4: a
+    // typo in an exporter must not decide an experiment.
+    if (judgeOutcome(entry.value, declared) === 'undeclared') continue;
+    recorded += entry.calls;
+    recordedUsd += entry.usd;
+    if (declared.success.includes(entry.value)) successes += entry.calls;
+  }
+  return {
+    name: arm.name,
+    successes,
+    recorded,
+    rate: recorded > 0 ? successes / recorded : null,
+    interval: wilson(successes, recorded),
+    recordedUsd,
+  };
+}
+
+export function runExperiment(
+  declaration: ExperimentDeclaration,
+  arms: { a: ExperimentArm; b: ExperimentArm },
+  vocabulary: OutcomeVocabulary | null,
+): ExperimentResult {
+  const a = resultOf(arms.a, vocabulary);
+  const b = resultOf(arms.b, vocabulary);
+
+  const stopping = {
+    declared: declaration.minOutcomesPerArm,
+    honoured: a.recorded >= declaration.minOutcomesPerArm && b.recorded >= declaration.minOutcomesPerArm,
+    short:
+      a.recorded < declaration.minOutcomesPerArm
+        ? a.name
+        : b.recorded < declaration.minOutcomesPerArm
+          ? b.name
+          : null,
+  };
+
+  const bare = (reason: NotSeparableReason): ExperimentResult => ({
+    a,
+    b,
+    separation: 'not-separable',
+    notSeparable: reason,
+    difference: null,
+    outcomesNeededPerArm: null,
+    stopping,
+    marginal: null,
+  });
+
+  if (a.rate === null || b.rate === null || a.interval === null || b.interval === null) {
+    return bare('nothing-recorded');
+  }
+
+  /**
+   * Newcombe's interval on the difference, built from the two Wilson
+   * intervals rather than from a normal approximation to the difference.
+   *
+   * The naive interval is symmetric and can run past 0 or 1, which at the
+   * sample sizes a real experiment starts with is not an edge case — it is
+   * most of the first week.
+   */
+  const point = a.rate - b.rate;
+  const low = point - Math.sqrt((a.rate - a.interval.low) ** 2 + (b.interval.high - b.rate) ** 2);
+  const high = point + Math.sqrt((a.interval.high - a.rate) ** 2 + (b.rate - b.interval.low) ** 2);
+  const difference = { point, low, high };
+
+  /**
+   * How many outcomes per arm would separate the difference *observed so far*.
+   *
+   * A standard two-proportion power calculation at 95% confidence and 80%
+   * power. It is an estimate about a difference that may itself be noise, and
+   * it is offered as "how much longer" rather than as a promise — but a number
+   * somebody can act on beats "not significant", which tells a reader nothing
+   * about whether to wait a day or abandon the idea.
+   */
+  const spread = Math.abs(point);
+  const outcomesNeededPerArm =
+    spread === 0
+      ? null
+      : Math.ceil(
+          (((Z_95 + Z_POWER_80) ** 2) * (a.rate * (1 - a.rate) + b.rate * (1 - b.rate))) /
+            (spread * spread),
+        );
+
+  const perCallA = arms.a.tally.parsed > 0 ? arms.a.totalUsd / arms.a.tally.parsed : 0;
+  const perCallB = arms.b.tally.parsed > 0 ? arms.b.totalUsd / arms.b.tally.parsed : 0;
+  const better = a.rate >= b.rate ? a : b;
+  const worse = better === a ? b : a;
+  const betterPerCall = better === a ? perCallA : perCallB;
+  const worsePerCall = better === a ? perCallB : perCallA;
+  const rateGap = (better.rate as number) - (worse.rate as number);
+
+  const marginal: Marginal = {
+    /**
+     * Per call on both sides, so arms that took different shares of the
+     * traffic compare. Dividing raw totals would report a marginal cost that
+     * moves when the split changes and the behaviour does not.
+     */
+    usdPerExtraSuccess:
+      betterPerCall > worsePerCall && rateGap > 0
+        ? (betterPerCall - worsePerCall) / rateGap
+        : null,
+    better: better.name,
+    dearer: betterPerCall > worsePerCall,
+  };
+
+  if (spread === 0) return { ...bare('no-difference-observed'), difference, marginal };
+  if (low <= 0 && high >= 0) {
+    return { ...bare('interval-includes-zero'), difference, outcomesNeededPerArm, marginal };
+  }
+
+  return {
+    a,
+    b,
+    separation: point > 0 ? 'a-wins' : 'b-wins',
+    notSeparable: null,
+    difference,
+    outcomesNeededPerArm,
+    stopping,
+    marginal,
+  };
+}

package/src/index.ts

@@ -50,6 +50,31 @@ export type { PlanAction, PlanActionKind, PlanAssumption, PlanDocument } from '.
 export { verifyPlan } from './verify.js';
 export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
 export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
+export { runExperiment } from './experiment.js';
+export type {
+  ArmResult,
+  ExperimentArm,
+  ExperimentDeclaration,
+  ExperimentResult,
+  Marginal,
+  NotSeparableReason,
+  Separation,
+} from './experiment.js';
+export {
+  ladderArithmetic,
+  ladderPosition,
+  validateLadder,
+  MIN_CALLS_FOR_LADDER,
+  BREAK_EVEN_BAND,
+} from './ladder.js';
+export type {
+  LadderArithmetic,
+  LadderPolicy,
+  LadderPosition,
+  LadderProblem,
+  LadderUnknown,
+  LadderVerdict,
+} from './ladder.js';
 export {
   perOutcome,
   rankPerOutcome,

package/src/ladder.ts

@@ -0,0 +1,295 @@
+/**
+ * Cheap first, escalate on measured failure — and the number that says whether
+ * that is a saving or a bill.
+ *
+ * "Route it to the cheaper model" has been a recommendation with a quality
+ * question attached since 1.23. Outcomes answered the quality question. This
+ * answers the one nobody asks out loud: **an escalation pays twice**, so a
+ * ladder is only a saving below a specific escalation rate, and above it the
+ * ladder costs more than never having built it.
+ *
+ * ## The arithmetic, stated rather than assumed
+ *
+ * Without a ladder, every call costs `dear`. With one, every call costs
+ * `cheap`, and the escalated share pays `dear` **on top** — the cheap attempt
+ * is not refunded.
+ *
+ *     with a ladder:  cheap + rate x dear
+ *     without one:    dear
+ *
+ * Those are equal at `rate = (dear - cheap) / dear`. Below it the ladder saves;
+ * above it the ladder is a more expensive way to get the same answers. A ladder
+ * sold as a saving without that number is the same head-arithmetic error `plan`
+ * was built to kill — and it is worse here, because the mistake compounds with
+ * traffic and nobody notices until a quarter is over.
+ *
+ * ## The escalation signal is the caller's
+ *
+ * Never inferred from length, latency, refusal text, a stop reason or a retry.
+ * The same refusal `outcome` makes, for a sharper reason: this is a **control
+ * loop**, not a report. A report built on a guess prints a wrong number; a
+ * control loop built on a guess sends real traffic to a more expensive model on
+ * the strength of that guess, forever, and bills for it.
+ *
+ * No signal, no ladder.
+ *
+ * ## What this module does not do
+ *
+ * It does not execute anything. A ladder escalates *after* a failure is known,
+ * which is after the answer came back and usually after something downstream
+ * judged it — so the retry belongs to the caller's own loop, not to a proxy
+ * sitting on one request. What lives here is the policy and the arithmetic that
+ * says whether the policy is worth running, measured against what actually
+ * happened.
+ */
+
+import { judgeOutcome } from './outcome.js';
+import type { OutcomeTally, OutcomeVocabulary } from './outcome.js';
+import { effectivePricing } from './pricing.js';
+import type { PricingCatalogue } from './pricing.js';
+
+/** One workload's ladder, as the config declares it. */
+export interface LadderPolicy {
+  /**
+   * Model ids, cheapest first. Two is the normal case; more is allowed and
+   * priced pairwise, because a three-rung ladder that escalates twice pays
+   * three times and the arithmetic has to say so.
+   */
+  tiers: string[];
+  /**
+   * The recorded outcome values that send the work up a tier.
+   *
+   * Declared, like the vocabulary itself. A value here that the vocabulary
+   * never declared is a configuration error rather than a silent no-op, and
+   * `validateLadder` says so.
+   */
+  escalateOn: string[];
+}
+
+export type LadderVerdict =
+  /** Measured escalation is below break-even: the ladder saves money. */
+  | 'saving'
+  /** Measured escalation is above it: the ladder costs money. */
+  | 'costing'
+  /** Within a hair of break-even, where the sign is not a claim worth making. */
+  | 'at-break-even'
+  /** Not enough was measured to say. */
+  | 'cannot-tell';
+
+export type LadderUnknown =
+  | 'no-outcomes-recorded'
+  | 'no-escalation-values-declared'
+  | 'tier-unpriced'
+  | 'too-few-calls';
+
+/**
+ * Calls a workload needs before its escalation rate is treated as a rate.
+ *
+ * The same floor `per-outcome` uses for successes, and for the same reason: a
+ * rate over fewer than ten observations moves more from one more observation
+ * than from anything a team could do about it. A control loop switched on
+ * because of nine calls is a control loop switched on for no reason.
+ */
+export const MIN_CALLS_FOR_LADDER = 10;
+
+/**
+ * How close to break-even counts as break-even.
+ *
+ * Two percentage points. Inside that band the ladder's sign flips on ordinary
+ * week-to-week variation, and reporting "saving" on Monday and "costing" on
+ * Thursday from the same policy would teach a reader to ignore the figure.
+ */
+export const BREAK_EVEN_BAND = 0.02;
+
+export interface LadderArithmetic {
+  /** What one call costs on the cheap tier. */
+  cheapUsd: number;
+  /** What one call costs on the dear tier. */
+  dearUsd: number;
+  /**
+   * The escalation rate at which the ladder stops saving, 0-1.
+   *
+   * `(dear - cheap) / dear`. Null when either tier could not be priced — never
+   * a zero, which would read as "any escalation at all loses money" and is a
+   * completely different and much more alarming claim.
+   */
+  breakEvenRate: number | null;
+}
+
+export interface LadderPosition {
+  arithmetic: LadderArithmetic;
+  /** The measured share of calls that escalated, 0-1, or null. */
+  measuredRate: number | null;
+  verdict: LadderVerdict;
+  /** Why, when the verdict is `cannot-tell`. A refusal never arrives bare. */
+  unknown: LadderUnknown | null;
+  /** Calls behind the measured rate. */
+  calls: number;
+  /** Calls whose recorded outcome was an escalation trigger. */
+  escalations: number;
+  /**
+   * What the ladder cost against what one tier alone would have, per call, or
+   * null when it cannot be computed. Negative means the ladder is cheaper.
+   */
+  deltaUsdPerCall: number | null;
+}
+
+/**
+ * The cost of one call on a model, for a described shape of work.
+ *
+ * Deliberately takes the token shape rather than reading it from a log: the
+ * break-even rate is a property of the *models and the work*, and somebody
+ * should be able to compute it before running a single call through a ladder.
+ */
+export function ladderArithmetic(
+  policy: LadderPolicy,
+  shape: { inputTokens: number; outputTokens: number },
+  catalogue: PricingCatalogue,
+  on: Date = new Date(),
+): LadderArithmetic {
+  const priceOf = (id: string | undefined): number | null => {
+    if (id === undefined) return null;
+    const model = catalogue.byId.get(id);
+    if (model === undefined) return null;
+    const rates = effectivePricing(model, on);
+    return (
+      (shape.inputTokens / 1_000_000) * rates.inputPerMTok +
+      (shape.outputTokens / 1_000_000) * rates.outputPerMTok
+    );
+  };
+
+  const cheap = priceOf(policy.tiers[0]);
+  /**
+   * The **last** tier, not the second.
+   *
+   * A three-rung ladder's alternative is the model it would have used without
+   * one, which is the top rung. Comparing against the middle would report a
+   * saving against a model nobody was going to use.
+   */
+  const dear = priceOf(policy.tiers[policy.tiers.length - 1]);
+
+  return {
+    cheapUsd: cheap ?? 0,
+    dearUsd: dear ?? 0,
+    breakEvenRate: cheap === null || dear === null || dear <= 0 ? null : (dear - cheap) / dear,
+  };
+}
+
+export function ladderPosition(
+  policy: LadderPolicy,
+  tally: OutcomeTally,
+  shape: { inputTokens: number; outputTokens: number },
+  vocabulary: OutcomeVocabulary | null,
+  catalogue: PricingCatalogue,
+  on: Date = new Date(),
+): LadderPosition {
+  const arithmetic = ladderArithmetic(policy, shape, catalogue, on);
+  const bare = (unknown: LadderUnknown): LadderPosition => ({
+    arithmetic,
+    measuredRate: null,
+    verdict: 'cannot-tell',
+    unknown,
+    calls: 0,
+    escalations: 0,
+    deltaUsdPerCall: null,
+  });
+
+  if (arithmetic.breakEvenRate === null) return bare('tier-unpriced');
+  if (policy.escalateOn.length === 0) return bare('no-escalation-values-declared');
+
+  const declared = vocabulary ?? { values: [], success: [] };
+  let calls = 0;
+  let escalations = 0;
+  for (const entry of tally.byValue) {
+    // Undeclared values are out of the denominator as well as the numerator —
+    // a typo in an exporter must not move a control loop's break-even.
+    if (judgeOutcome(entry.value, declared) === 'undeclared') continue;
+    calls += entry.calls;
+    if (policy.escalateOn.includes(entry.value)) escalations += entry.calls;
+  }
+
+  if (calls === 0) return bare('no-outcomes-recorded');
+  if (calls < MIN_CALLS_FOR_LADDER) {
+    return {
+      arithmetic,
+      measuredRate: null,
+      verdict: 'cannot-tell',
+      unknown: 'too-few-calls',
+      calls,
+      escalations,
+      deltaUsdPerCall: null,
+    };
+  }
+
+  const measuredRate = escalations / calls;
+  const withLadder = arithmetic.cheapUsd + measuredRate * arithmetic.dearUsd;
+  const deltaUsdPerCall = withLadder - arithmetic.dearUsd;
+
+  const distance = measuredRate - arithmetic.breakEvenRate;
+  const verdict: LadderVerdict =
+    Math.abs(distance) <= BREAK_EVEN_BAND ? 'at-break-even' : distance < 0 ? 'saving' : 'costing';
+
+  return { arithmetic, measuredRate, verdict, unknown: null, calls, escalations, deltaUsdPerCall };
+}
+
+export type LadderProblem =
+  | { kind: 'too-few-tiers'; tiers: number }
+  | { kind: 'unknown-model'; model: string }
+  | { kind: 'duplicate-tier'; model: string }
+  | { kind: 'tiers-not-cheapest-first'; model: string; after: string }
+  | { kind: 'escalate-on-undeclared'; value: string }
+  | { kind: 'escalate-on-a-success'; value: string };
+
+/**
+ * Everything wrong with a ladder before it is ever run.
+ *
+ * Returned rather than thrown, so a caller can report all of it at once — a
+ * config that has to be fixed one error per run is a config people give up on.
+ *
+ * The last two are the interesting ones. Escalating on a value the vocabulary
+ * never declared is a ladder that silently never fires; escalating on a value
+ * declared as a **success** is a ladder that pays twice for work that already
+ * worked, which is the most expensive possible typo in this file.
+ */
+export function validateLadder(
+  policy: LadderPolicy,
+  vocabulary: OutcomeVocabulary | null,
+  catalogue: PricingCatalogue,
+  on: Date = new Date(),
+): LadderProblem[] {
+  const problems: LadderProblem[] = [];
+  if (policy.tiers.length < 2) {
+    problems.push({ kind: 'too-few-tiers', tiers: policy.tiers.length });
+  }
+
+  const seen = new Set<string>();
+  let previous: { id: string; price: number } | null = null;
+  for (const id of policy.tiers) {
+    if (seen.has(id)) problems.push({ kind: 'duplicate-tier', model: id });
+    seen.add(id);
+    const model = catalogue.byId.get(id);
+    if (model === undefined) {
+      problems.push({ kind: 'unknown-model', model: id });
+      continue;
+    }
+    const price = effectivePricing(model, on).inputPerMTok;
+    if (previous !== null && price < previous.price) {
+      // A ladder whose rungs go down is not a ladder; it is a routing rule
+      // that escalates to something cheaper and then reports a saving for it.
+      problems.push({ kind: 'tiers-not-cheapest-first', model: id, after: previous.id });
+    }
+    previous = { id, price };
+  }
+
+  if (vocabulary !== null) {
+    for (const value of policy.escalateOn) {
+      if (!vocabulary.values.includes(value)) {
+        problems.push({ kind: 'escalate-on-undeclared', value });
+      } else if (vocabulary.success.includes(value)) {
+        problems.push({ kind: 'escalate-on-a-success', value });
+      }
+    }
+  }
+
+  return problems;
+}