@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/src/settings.ts

@@ -0,0 +1,441 @@
+import type { ReasoningEffort } from "./models.js";
+import type { Route } from "./runner.js";
+
+/**
+ * Versioned model settings. Each AI feature's settings (its model,
+ * fallback, reasoning setting, and output limit, with the instructions they
+ * ran with) are numbered versions that never change. Changing a setting
+ * makes a new version live; rolling back makes an earlier one live again;
+ * every switch is logged with who, when, why, and its evidence. A feature
+ * never changed runs on its default from code. Optional: use it, or keep
+ * your own routing and pass `route` to the runner yourself.
+ */
+
+/**
+ * A feature's model settings: live model, fallback, reasoning setting,
+ * output limit, and temperature.
+ */
+export type Settings = {
+  model: string;
+  fallbackModel: string | null;
+  /** Null: the model's own default. */
+  reasoningEffort: ReasoningEffort | null;
+  /** Null: the call's own default. */
+  maxOutputTokens: number | null;
+  /**
+   * The sampling temperature the feature sends. Null or unset: none sent,
+   * so the model's own default. Tests record it, and a test at another
+   * temperature than the live one is out of date.
+   */
+  temperature?: number | null;
+};
+
+/**
+ * One numbered version of a feature's settings, with the instructions it ran with. Never changed.
+ */
+export type SettingsVersion = Settings & {
+  id: string;
+  operation: string;
+  /** 1, 2, 3, … per feature. */
+  number: number;
+  /** The instructions' hash when this version was made. */
+  promptVersion: string | null;
+  createdAt: Date;
+};
+
+/**
+ * CHANGE: a person changed a setting. PROMOTE: a model was made live or
+ * the fallback from a test, with the test as evidence. ROLLBACK: an
+ * earlier version made live again. DEPLOY: the instructions changed in code
+ * under the live settings.
+ */
+export type ChangeKind = "CHANGE" | "PROMOTE" | "ROLLBACK" | "DEPLOY";
+
+/**
+ * One switch in a feature's history: its kind, the versions it went from and to, who, why,
+ * and the evidence.
+ */
+export type SettingsChange = {
+  id: string;
+  operation: string;
+  kind: ChangeKind;
+  fromVersionId: string | null;
+  toVersionId: string;
+  reason: string | null;
+  /** Who made it; null for the system (a deploy, the first default). */
+  by: string | null;
+  /** Why it's safe: the tests compared, the verdict. */
+  evidence: unknown;
+  createdAt: Date;
+};
+
+/** One atomic switch: maybe a new version, then live, then the log. */
+export type SettingsSwitch = {
+  /** Make a new version with these settings (numbered next). */
+  create?: { settings: Settings; promptVersion: string | null };
+  /** Or make this existing version live. */
+  versionId?: string;
+  kind: ChangeKind;
+  reason: string | null;
+  by: string | null;
+  evidence?: unknown;
+  /**
+   * The live version this switch was decided against (null: none). The
+   * store refuses the switch if another one got in first.
+   */
+  expectLive: string | null;
+};
+
+/** Where versions live. `@crowandraven/evalkeeper-prisma` and `memoryStore` implement it. */
+export interface SettingsStore {
+  liveVersion(operation: string): Promise<SettingsVersion | null>;
+  /** Newest first. */
+  listVersions(operation: string): Promise<SettingsVersion[]>;
+  /** Newest first. */
+  listChanges(
+    operation: string,
+    opts?: { limit?: number },
+  ): Promise<SettingsChange[]>;
+  /**
+   * Applies a switch atomically. Throws SettingsConflict when the live
+   * version is no longer `expectLive`.
+   */
+  switchTo(operation: string, s: SettingsSwitch): Promise<SettingsVersion>;
+}
+
+/** Another change landed first: reload and decide again. */
+export class SettingsConflict extends Error {
+  constructor(operation: string) {
+    super(
+      `The settings for ${operation} changed meanwhile. Reload and try again.`,
+    );
+    this.name = "SettingsConflict";
+  }
+}
+
+/**
+ * A feature has no settings: no version yet, and no default in code. Apps
+ * whose models are chosen in an admin page, not in code, see this until
+ * the feature's first version is recorded (`seed` or `change`).
+ */
+export class NoSettingsError extends Error {
+  /** The feature with no settings. */
+  readonly operation: string;
+  constructor(operation: string) {
+    super(
+      `${operation} has no settings yet: no version, and no default in code.`,
+    );
+    this.name = "NoSettingsError";
+    this.operation = operation;
+  }
+}
+
+/** Whether two settings are the same in every field. */
+export function sameSettings(a: Settings, b: Settings): boolean {
+  return (
+    a.model === b.model &&
+    a.fallbackModel === b.fallbackModel &&
+    a.reasoningEffort === b.reasoningEffort &&
+    a.maxOutputTokens === b.maxOutputTokens &&
+    (a.temperature ?? null) === (b.temperature ?? null)
+  );
+}
+
+/** Just the settings of a version. */
+export function settingsOf(v: Settings): Settings {
+  return {
+    model: v.model,
+    fallbackModel: v.fallbackModel,
+    reasoningEffort: v.reasoningEffort,
+    maxOutputTokens: v.maxOutputTokens,
+    temperature: v.temperature ?? null,
+  };
+}
+
+/** What changed between two settings, in plain words. */
+export function describeChange(
+  from: Settings | null,
+  to: Settings,
+  label: (model: string) => string = (m) => m,
+): string[] {
+  if (!from) return [`Live: ${label(to.model)}`];
+  const out: string[] = [];
+  if (from.model !== to.model)
+    out.push(`Live: ${label(from.model)} → ${label(to.model)}`);
+  if (from.fallbackModel !== to.fallbackModel)
+    out.push(
+      `Fallback: ${from.fallbackModel ? label(from.fallbackModel) : "none"} → ${to.fallbackModel ? label(to.fallbackModel) : "none"}`,
+    );
+  if (from.reasoningEffort !== to.reasoningEffort)
+    out.push(
+      `Reasoning: ${from.reasoningEffort ?? "default"} → ${to.reasoningEffort ?? "default"}`,
+    );
+  if (from.maxOutputTokens !== to.maxOutputTokens)
+    out.push(
+      `Output limit: ${from.maxOutputTokens ?? "default"} → ${to.maxOutputTokens ?? "default"}`,
+    );
+  if ((from.temperature ?? null) !== (to.temperature ?? null))
+    out.push(
+      `Temperature: ${from.temperature ?? "default"} → ${to.temperature ?? "default"}`,
+    );
+  return out;
+}
+
+/** A switch with the versions it went from and to, for a history page. */
+export type HistoryEntry = SettingsChange & {
+  to: SettingsVersion;
+  from: SettingsVersion | null;
+};
+
+/**
+ * Versioned settings: each feature's settings as numbered versions, with every change,
+ * promotion, rollback, and deploy logged. Its `route` is the runner's route.
+ *
+ * @example
+ * ```ts
+ * const settings = createSettings({
+ *   store: prismaSettingsStore(prisma),
+ *   defaults: (op) => DEFAULTS[op],
+ *   promptVersion: (op) => PROMPT_VERSIONS[op],
+ * });
+ * await settings.update("ticket_triage", { model: "claude-haiku-5-5" }, {
+ *   by: admin.name,
+ *   reason: "About as accurate at half the cost",
+ *   evidence: { runId },
+ * });
+ * await settings.rollBack("ticket_triage", versionId, { by: admin.name, reason: "Worse in production" });
+ * ```
+ *
+ * When models are chosen in an admin page rather than in code, return null
+ * from `defaults` and record each feature's existing config as version 1
+ * with `seed`:
+ *
+ * ```ts
+ * const settings = createSettings({ store, defaults: () => null });
+ * for (const row of await db.aiConfig.findMany())
+ *   await settings.seed(row.operation, settingsFromRow(row), {
+ *     by: null,
+ *     reason: "Imported from the admin config",
+ *   });
+ * ```
+ */
+export function createSettings(opts: {
+  store: SettingsStore;
+  /**
+   * Each feature's settings in code, used until it's first changed. Null:
+   * the feature has no default, and reading its settings throws
+   * `NoSettingsError` until a version is recorded.
+   */
+  defaults: (operation: string) => Settings | null;
+  /** The instructions' current hash, so a version records what it ran. */
+  promptVersion?: (operation: string) => string | Promise<string>;
+}) {
+  const { store } = opts;
+  const prompt = async (op: string) =>
+    opts.promptVersion ? await opts.promptVersion(op) : null;
+
+  /**
+   * The live settings: the live version's, else the default from code.
+   * Throws `NoSettingsError` when the feature has neither.
+   */
+  async function current(
+    operation: string,
+  ): Promise<{ settings: Settings; version: SettingsVersion | null }> {
+    const version = await store.liveVersion(operation);
+    if (version) return { settings: settingsOf(version), version };
+    const fallback = opts.defaults(operation);
+    if (!fallback) throw new NoSettingsError(operation);
+    return { settings: fallback, version };
+  }
+
+  /** The runner's route for a feature, from its live settings; throws `NoSettingsError` when it has none. */
+  async function route(operation: string): Promise<Route> {
+    const { settings } = await current(operation);
+    return {
+      model: settings.model,
+      fallbackModel: settings.fallbackModel,
+      reasoningEffort: settings.reasoningEffort,
+      temperature: settings.temperature ?? null,
+    };
+  }
+
+  /**
+   * Makes new settings live as a new version. The first change to a
+   * feature first records its default as version 1, so there's always
+   * something to roll back to; a feature with no default gets `next` as
+   * version 1. With `evidence`, it's a promotion. Returns null when
+   * nothing changed.
+   */
+  async function change(
+    operation: string,
+    next: Settings,
+    by: { by: string | null; reason: string | null; evidence?: unknown },
+  ): Promise<SettingsVersion | null> {
+    const promptVersion = await prompt(operation);
+    let live = await store.liveVersion(operation);
+    if (!live) {
+      const initial = opts.defaults(operation);
+      if (!initial)
+        return store.switchTo(operation, {
+          create: { settings: settingsOf(next), promptVersion },
+          kind: by.evidence === undefined ? "CHANGE" : "PROMOTE",
+          reason: by.reason,
+          by: by.by,
+          evidence: by.evidence,
+          expectLive: null,
+        });
+      if (sameSettings(initial, next)) return null;
+      live = await store.switchTo(operation, {
+        create: { settings: initial, promptVersion },
+        kind: "CHANGE",
+        reason: "The default from code, before the first change",
+        by: null,
+        expectLive: null,
+      });
+    } else if (sameSettings(settingsOf(live), next)) return null;
+    return store.switchTo(operation, {
+      create: { settings: settingsOf(next), promptVersion },
+      kind: by.evidence === undefined ? "CHANGE" : "PROMOTE",
+      reason: by.reason,
+      by: by.by,
+      evidence: by.evidence,
+      expectLive: live.id,
+    });
+  }
+
+  /**
+   * Changes only some settings (e.g. the fallback), keeping the rest.
+   * Throws `NoSettingsError` for a feature with no settings to keep: use
+   * `change` or `seed` with full settings first.
+   */
+  async function update(
+    operation: string,
+    patch: Partial<Settings>,
+    by: { by: string | null; reason: string | null; evidence?: unknown },
+  ): Promise<SettingsVersion | null> {
+    const { settings } = await current(operation);
+    return change(operation, { ...settings, ...patch }, by);
+  }
+
+  /** Makes an earlier version live again. */
+  async function rollBack(
+    operation: string,
+    versionId: string,
+    by: { by: string | null; reason: string },
+  ): Promise<SettingsVersion> {
+    const versions = await store.listVersions(operation);
+    const to = versions.find((v) => v.id === versionId);
+    if (!to) throw new Error("No such version for this feature");
+    const live = await store.liveVersion(operation);
+    if (live?.id === to.id) return to;
+    return store.switchTo(operation, {
+      versionId: to.id,
+      kind: "ROLLBACK",
+      reason: by.reason,
+      by: by.by,
+      expectLive: live?.id ?? null,
+    });
+  }
+
+  /**
+   * Instructions live in code, so a deploy can change them under live
+   * settings. When it has, records a version with the same settings and the
+   * new instructions, so the history says what changed and when. Call it
+   * where the feature's settings are read (or after each deploy).
+   */
+  async function syncPrompt(
+    operation: string,
+  ): Promise<SettingsVersion | null> {
+    const live = await store.liveVersion(operation);
+    const now = await prompt(operation);
+    if (!live || !now || now === live.promptVersion) return live;
+    try {
+      return await store.switchTo(operation, {
+        create: { settings: settingsOf(live), promptVersion: now },
+        kind: "DEPLOY",
+        reason: live.promptVersion
+          ? `Instructions changed (${live.promptVersion} → ${now})`
+          : `Instructions recorded (${now})`,
+        by: null,
+        expectLive: live.id,
+      });
+    } catch (e) {
+      // Another request recorded the same deploy first.
+      if (e instanceof SettingsConflict) return store.liveVersion(operation);
+      throw e;
+    }
+  }
+
+  /**
+   * Records existing settings as a feature's version 1, for an app moving
+   * its model choices into versioned settings (an import from its own
+   * config table). Only when the feature has no version yet: otherwise it
+   * changes nothing and returns null, so an import can run again safely.
+   * `promptVersion` defaults to the instructions' current hash.
+   */
+  async function seed(
+    operation: string,
+    initial: Settings,
+    by: {
+      by: string | null;
+      reason: string | null;
+      promptVersion?: string | null;
+    },
+  ): Promise<SettingsVersion | null> {
+    if (await store.liveVersion(operation)) return null;
+    const promptVersion =
+      by.promptVersion !== undefined
+        ? by.promptVersion
+        : await prompt(operation);
+    try {
+      return await store.switchTo(operation, {
+        create: { settings: settingsOf(initial), promptVersion },
+        kind: "CHANGE",
+        reason: by.reason,
+        by: by.by,
+        expectLive: null,
+      });
+    } catch (e) {
+      // Another request seeded it first: that version stands.
+      if (e instanceof SettingsConflict) return null;
+      throw e;
+    }
+  }
+
+  /** Every switch, newest first, with the versions it went from and to. */
+  async function history(
+    operation: string,
+    limit = 50,
+  ): Promise<HistoryEntry[]> {
+    const [changes, versions] = await Promise.all([
+      store.listChanges(operation, { limit }),
+      store.listVersions(operation),
+    ]);
+    const byId = new Map(versions.map((v) => [v.id, v]));
+    return changes.flatMap((c) => {
+      const to = byId.get(c.toVersionId);
+      if (!to) return [];
+      return [
+        {
+          ...c,
+          to,
+          from: c.fromVersionId ? (byId.get(c.fromVersionId) ?? null) : null,
+        },
+      ];
+    });
+  }
+
+  return {
+    current,
+    route,
+    change,
+    update,
+    rollBack,
+    syncPrompt,
+    seed,
+    history,
+  };
+}
+
+/** What createSettings returns. */
+export type VersionedSettings = ReturnType<typeof createSettings>;

package/src/stableJson.ts

@@ -0,0 +1,24 @@
+/**
+ * JSON with object keys sorted, so a case's input compares equal however
+ * its keys were ordered (Postgres jsonb reorders them on save).
+ */
+export function stableJson(value: unknown): string {
+  return JSON.stringify(value, (_key, v: unknown) =>
+    v && typeof v === "object" && !Array.isArray(v)
+      ? Object.fromEntries(
+          Object.entries(v as Record<string, unknown>).sort(([a], [b]) =>
+            a < b ? -1 : a > b ? 1 : 0,
+          ),
+        )
+      : v,
+  );
+}
+
+/** Drafted cases not yet in the answer key. */
+export function newStarters<T extends { input: unknown }>(
+  starter: T[] | undefined,
+  existing: { input: unknown }[],
+): T[] {
+  const have = new Set(existing.map((c) => stableJson(c.input)));
+  return (starter ?? []).filter((c) => !have.has(stableJson(c.input)));
+}

package/src/stats.ts

@@ -0,0 +1,215 @@
+import type { CheckResult } from "./types.js";
+import { calibrationError } from "./decisions.js";
+
+/**
+ * Summary numbers for a run: pass rates per check with a 95% range, so a
+ * small dataset doesn't look more certain than it is. Pure; no database.
+ */
+
+/** A pass rate with its 95% range; null when nothing was measured. */
+export type Rate = {
+  passed: number;
+  /** Outputs the check applied to. */
+  n: number;
+  rate: number | null;
+  /** 95% Wilson interval. */
+  low: number | null;
+  high: number | null;
+};
+
+/** Wilson score interval: honest for small n and rates near 0 or 1. */
+export function wilson(passed: number, n: number, z = 1.96): Rate {
+  if (n === 0) return { passed, n, rate: null, low: null, high: null };
+  const p = passed / n;
+  const denom = 1 + (z * z) / n;
+  const center = (p + (z * z) / (2 * n)) / denom;
+  const half =
+    (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denom;
+  return {
+    passed,
+    n,
+    rate: p,
+    low: Math.max(0, center - half),
+    high: Math.min(1, center + half),
+  };
+}
+
+/** One graded answer, as the metrics read it. */
+export type ScoredOutput = {
+  caseId: string;
+  repeat: number;
+  /** null when the call failed. */
+  checks: CheckResult[] | null;
+  costUsd: number;
+  ms: number;
+  reused: boolean;
+};
+
+/**
+ * A test's numbers: answers right, partial credit, each check's pass rate, consistency,
+ * cost, time, and reuse.
+ */
+export type SuiteMetrics = {
+  cases: number;
+  outputs: number;
+  failed: number;
+  /** Every applicable check passed. Failed calls count as not passing. */
+  allPass: Rate;
+  /**
+   * Partial credit summed over answers (e.g. items right of items
+   * expected), where the suite gives it; the headline when present.
+   */
+  units: { right: number; total: number } | null;
+  byCheck: Record<string, Rate>;
+  /**
+   * Of cases run more than once, the share whose all-pass result was the
+   * same every time. Low: the model is inconsistent on these cases.
+   */
+  consistency: Rate | null;
+  /** What grading's judge spent, apart from the model under test (runner). */
+  judgeCostUsd?: number;
+  /** The error most failed answers share (runner). */
+  failureReason?: string | null;
+  /**
+   * Per check, the mean of each measure its answers carried (brier,
+   * logLoss, …), and, where answers carried a confidence, the calibration
+   * error. Only checks with measures appear.
+   */
+  measures?: Record<string, CheckMeasures>;
+  /** What this run spent: reused outputs cost nothing. */
+  costUsd: number;
+  /** What one answer costs on this model, reused or not. */
+  costPerAnswer: number | null;
+  /** How long one answer takes on this model. */
+  avgMs: number;
+  reused: number;
+};
+
+/**
+ * Whether an answer passed every check that applied. A failed answer (null) didn't.
+ */
+export function allPassed(checks: CheckResult[] | null): boolean {
+  return checks !== null && checks.every((c) => c.pass !== false);
+}
+
+/** Sums each answer's partial credit; null when the suite gives none. */
+export function sumUnits(
+  outputs: { checks: CheckResult[] | null }[],
+): { right: number; total: number } | null {
+  let right = 0;
+  let total = 0;
+  let any = false;
+  for (const o of outputs)
+    for (const c of o.checks ?? [])
+      if (c.units) {
+        any = true;
+        right += c.units.right;
+        total += c.units.total;
+      }
+  return any ? { right, total } : null;
+}
+
+/** One answer's score from 0 to 1: its partial credit, else all-or-nothing. */
+export function answerScore(checks: CheckResult[] | null): number {
+  if (!checks) return 0;
+  const u = checks.find((c) => c.units)?.units;
+  if (u) return u.total ? u.right / u.total : 1;
+  return allPassed(checks) ? 1 : 0;
+}
+
+/** A test's numbers from its graded answers. */
+/** One check's measures across a test. */
+export type CheckMeasures = {
+  /** Answers that carried measures. */
+  n: number;
+  /** The mean of each measure, over the answers that carried it. */
+  mean: Record<string, number>;
+  /** Expected calibration error, from confidence and correctness; null without both. */
+  calibrationError: number | null;
+};
+
+/** One check's measures across a test's answers; null when none carried any. */
+export function measuresFor(
+  outputs: ScoredOutput[],
+  key: string,
+): CheckMeasures | null {
+  const all = outputs
+    .map((o) => o.checks?.find((c) => c.key === key)?.measures)
+    .filter((m): m is Record<string, number> => m !== undefined);
+  if (!all.length) return null;
+  const sums = new Map<string, { total: number; n: number }>();
+  for (const m of all)
+    for (const [name, v] of Object.entries(m)) {
+      const s = sums.get(name) ?? { total: 0, n: 0 };
+      s.total += v;
+      s.n++;
+      sums.set(name, s);
+    }
+  const pairs = all.flatMap((m) =>
+    m.confidence !== undefined && m.correct !== undefined
+      ? [{ confidence: m.confidence, correct: m.correct === 1 }]
+      : [],
+  );
+  return {
+    n: all.length,
+    mean: Object.fromEntries([...sums].map(([k, s]) => [k, s.total / s.n])),
+    calibrationError: calibrationError(pairs),
+  };
+}
+
+/**
+ *
+ */
+export function suiteMetrics(
+  outputs: ScoredOutput[],
+  checkKeys: string[],
+): SuiteMetrics {
+  const byCheck: Record<string, Rate> = {};
+  for (const key of checkKeys) {
+    const applied = outputs
+      .map((o) => o.checks?.find((c) => c.key === key))
+      .filter((c): c is CheckResult => c !== undefined && c.pass !== null);
+    byCheck[key] = wilson(applied.filter((c) => c.pass).length, applied.length);
+  }
+  const measures: Record<string, CheckMeasures> = {};
+  for (const key of checkKeys) {
+    const m = measuresFor(outputs, key);
+    if (m) measures[key] = m;
+  }
+  const byCase = new Map<string, boolean[]>();
+  for (const o of outputs)
+    byCase.set(o.caseId, [
+      ...(byCase.get(o.caseId) ?? []),
+      allPassed(o.checks),
+    ]);
+  const repeated = [...byCase.values()].filter((r) => r.length > 1);
+  const called = outputs.filter((o) => !o.reused);
+  // Answers with a recorded time (older reused rows have none).
+  const timed = outputs.filter((o) => o.checks && o.ms > 0);
+  return {
+    cases: byCase.size,
+    outputs: outputs.length,
+    failed: outputs.filter((o) => !o.checks).length,
+    allPass: wilson(
+      outputs.filter((o) => allPassed(o.checks)).length,
+      outputs.length,
+    ),
+    units: sumUnits(outputs),
+    byCheck,
+    ...(Object.keys(measures).length ? { measures } : {}),
+    consistency: repeated.length
+      ? wilson(
+          repeated.filter((r) => r.every((x) => x === r[0])).length,
+          repeated.length,
+        )
+      : null,
+    costUsd: called.reduce((t, o) => t + o.costUsd, 0),
+    costPerAnswer: timed.length
+      ? timed.reduce((t, o) => t + o.costUsd, 0) / timed.length
+      : null,
+    avgMs: timed.length
+      ? Math.round(timed.reduce((t, o) => t + o.ms, 0) / timed.length)
+      : 0,
+    reused: outputs.length - called.length,
+  };
+}