@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/src/store.ts

@@ -0,0 +1,167 @@
+import type { CheckResult } from "./types.js";
+import type { SuiteMetrics } from "./stats.js";
+
+/**
+ * Where Evalkeeper keeps cases, answers, tests, and outputs. The runner
+ * talks only to this interface: `@crowandraven/evalkeeper-prisma` implements it on your
+ * database, and `memoryStore()` keeps everything in memory for tests and
+ * demos. Every method is a plain read or write; the rules live in the
+ * runner.
+ */
+
+/** ALL: every case in use. SMOKE: the quick set. DEV and TEST: by split. */
+export type EvalSet = "ALL" | "SMOKE" | "DEV" | "TEST";
+/** Every set of cases a test can run on. */
+export const EVAL_SETS: EvalSet[] = ["ALL", "SMOKE", "DEV", "TEST"];
+
+/**
+ * Where an answer stands: pending review, confirmed, provisional, or rejected.
+ */
+export type LabelStatus = "PENDING" | "CONFIRMED" | "PROVISIONAL" | "REJECTED";
+
+/**
+ * An answer to a case. EXPECTED is what a good answer looks like; a case's
+ * newest CONFIRMED one is what tests grade against. PENDING answers (from a
+ * model, or not yet reviewed) are never graded against; PROVISIONAL ones are
+ * put to use before review and say so.
+ */
+export type LabelRecord = {
+  id: string;
+  kind: "EXPECTED";
+  status: LabelStatus;
+  value: unknown;
+  /** Who gave it: "OWNER", "MODEL", a reviewer's role. */
+  raterRole: string;
+  raterId: string | null;
+  note: string | null;
+  createdAt: Date;
+  updatedAt: Date;
+};
+
+/** A stored case: a frozen input, its tags and set, and its answers. */
+export type CaseRecord = {
+  id: string;
+  suite: string;
+  /** The suite's input, frozen. */
+  input: unknown;
+  tags: string[];
+  split: "DEV" | "TEST";
+  status: "DRAFT" | "ACTIVE" | "RETIRED";
+  /** Where it came from: "OWNER", "MODEL_DRAFT", "FEEDBACK", … */
+  source: string;
+  /** What it was made from (an id in your app), to avoid adding it twice. */
+  sourceRef: string | null;
+  note: string | null;
+  createdAt: Date;
+  updatedAt: Date;
+  labels: LabelRecord[];
+};
+
+/** Where a test stands. */
+export type RunStatus = "RUNNING" | "SUCCEEDED" | "FAILED";
+
+/** What the runner stores on a finished test. */
+export type RunMetrics = SuiteMetrics & {
+  judgeCostUsd: number;
+  failureReason: string | null;
+};
+
+/**
+ * A stored test: one suite on one model, with what it ran with and its metrics.
+ */
+export type RunRecord = {
+  id: string;
+  suite: string;
+  model: string;
+  /** Hash of the instructions it ran with (see `instructionsHash`). */
+  promptVersion: string;
+  /** The reasoning setting tested; null: the model's own default. */
+  reasoningEffort: string | null;
+  set: EvalSet;
+  repeats: number;
+  /** Changes when the cases or their answers do, so tests compare. */
+  datasetVersion: string | null;
+  /** The suite's grader version when last graded. */
+  graderVersion: string;
+  status: RunStatus;
+  metrics: RunMetrics | null;
+  error: string | null;
+  estimatedCostUsd: number | null;
+  /** Who started it; null for a scheduled check. */
+  startedBy: string | null;
+  startedAt: Date;
+  finishedAt: Date | null;
+};
+
+/** One stored answer in a test, with its grades, cost, and time. */
+export type OutputRecord = {
+  id: string;
+  runId: string;
+  caseId: string;
+  repeat: number;
+  /** The request's hash, so an identical one is reused, not paid twice. */
+  requestHash: string;
+  output: unknown;
+  /** Why the answer failed; null when it came back. */
+  error: string | null;
+  scores: CheckResult[] | null;
+  costUsd: number;
+  ms: number;
+  /** Copied from an identical earlier request: cost nothing this time. */
+  reused: boolean;
+  createdAt: Date;
+};
+
+/** A test to record; the store fills in the id, times, and status. */
+export type NewRun = Omit<
+  RunRecord,
+  "id" | "startedAt" | "finishedAt" | "status" | "metrics" | "error"
+>;
+/** An answer to record. */
+export type NewOutput = Omit<OutputRecord, "id" | "createdAt">;
+/** An answer to add to a case. */
+export type NewLabel = Omit<LabelRecord, "id" | "createdAt" | "updatedAt">;
+/** A case to create, with its first answers. */
+export type NewCase = Omit<
+  CaseRecord,
+  "id" | "createdAt" | "updatedAt" | "labels"
+> & {
+  labels?: NewLabel[];
+};
+
+/**
+ * Where Evalkeeper keeps cases, answers, tests, and outputs. Implement it for another
+ * database, and run describeStoreContract against it.
+ */
+export interface EvalStore {
+  /** A suite's cases that aren't retired, with their answers, oldest first. */
+  listCases(suite: string): Promise<CaseRecord[]>;
+  createCase(data: NewCase): Promise<CaseRecord>;
+  /** Adds an answer to a case (answers are never overwritten). */
+  addLabel(caseId: string, label: NewLabel): Promise<LabelRecord>;
+  /** Changes an answer's status, e.g. a person confirming a draft. */
+  setLabelStatus(labelId: string, status: LabelStatus): Promise<void>;
+  /** Changes a case's status, e.g. retiring it. */
+  setCaseStatus(caseId: string, status: CaseRecord["status"]): Promise<void>;
+  getRun(id: string): Promise<RunRecord | null>;
+  /** Newest first. */
+  listRuns(
+    suite: string,
+    opts?: { status?: RunStatus; limit?: number },
+  ): Promise<RunRecord[]>;
+  createRun(data: NewRun): Promise<RunRecord>;
+  updateRun(
+    id: string,
+    patch: Partial<Omit<RunRecord, "id">>,
+    /** Only if the run still has this status (a guard against races). */
+    onlyIf?: { status: RunStatus },
+  ): Promise<void>;
+  listOutputs(runId: string): Promise<OutputRecord[]>;
+  /** The newest original (not reused, no error) output with this hash. */
+  findReusableOutput(
+    requestHash: string,
+    since: Date,
+  ): Promise<OutputRecord | null>;
+  createOutput(data: NewOutput): Promise<OutputRecord>;
+  setOutputScores(id: string, scores: CheckResult[]): Promise<void>;
+}

package/src/suggest.ts

@@ -0,0 +1,245 @@
+import { retiresLabel, type ModelCatalog } from "./models.js";
+import type { Rate } from "./stats.js";
+import type { CheckDef, SuitePriority } from "./types.js";
+import type { Verdict } from "./compare.js";
+import { at } from "./util.js";
+
+/**
+ * Which model to make live and which the fallback, from a suite's current
+ * tests, by fixed rules so the reason is always the real one. Switching
+ * live is conservative: the candidate must score at least as high as live
+ * and make no more serious mistakes, and then needs a real reason: clearly
+ * more accurate, at least two fewer serious mistakes, or, by the suite's
+ * priority, at least 30% faster (speed) or cheaper (cost). One exception,
+ * for suites that favor cost or speed: a model that scores a little lower
+ * (within 10 points, and no clear difference) with no more serious
+ * mistakes qualifies when it costs or takes at most half as much. A suite's
+ * time limit rules out slower models, live included. Otherwise live stays, and the best-scoring other
+ * model is named as one to watch. The fallback is the most accurate model
+ * from another provider; it needn't beat live, it only runs in an outage.
+ * Models the provider is retiring are never suggested, and a retiring live
+ * model is replaced by the best-scoring one that isn't. Pure.
+ */
+
+/** One model's row, as the suggestion reads it. */
+export type SuggestRow = {
+  key: string;
+  model: string;
+  label: string;
+  variant: boolean;
+  live: boolean;
+  fallback: boolean;
+  vsLive: Verdict | null;
+  /** May only be live (e.g. a model that can't stand in for a text model). */
+  liveOnly?: boolean;
+  /** May only be the fallback. */
+  fallbackOnly?: boolean;
+  test: {
+    right: number;
+    total: number;
+    byCheck: Record<string, Rate>;
+    avgMs: number | null;
+    costPerAnswer: number | null;
+    stale: boolean;
+    /** Answers that failed; a test with any is incomplete. */
+    failedAnswers?: number;
+  } | null;
+};
+
+/**
+ * Which model to make live and which the fallback, with the reasons in plain sentences.
+ */
+export type Suggestion<R extends SuggestRow = SuggestRow> = {
+  live: R;
+  fallback: R | null;
+  /** Live stays as it is. */
+  keepLive: boolean;
+  /** Nothing to change at all: live stays and the fallback is already set. */
+  same: boolean;
+  /** Why, in plain sentences. */
+  reasons: string[];
+};
+
+/**
+ * How many serious checks a model's test failed; Infinity without a test.
+ */
+export function seriousMistakes(row: SuggestRow, checks: CheckDef[]): number {
+  const test = row.test;
+  if (!test) return Infinity;
+  return checks
+    .filter((c) => c.severity === "serious")
+    .reduce((t, c) => {
+      const r = test.byCheck[c.key];
+      return t + (r ? r.n - r.passed : 0);
+    }, 0);
+}
+
+const mistakes = (n: number) => `${n} serious mistake${n === 1 ? "" : "s"}`;
+const rate = (r: SuggestRow) => (r.test ? r.test.right / r.test.total : 0);
+const pct = (x: number) => `${Math.round(x * 100)}%`;
+const cost = (r: SuggestRow) => r.test?.costPerAnswer ?? Infinity;
+const ms = (r: SuggestRow) => r.test?.avgMs ?? Infinity;
+/** A cost or speed trade-off may give up at most this much accuracy. */
+const MAX_TRADE_DROP = 0.1;
+/** ...and must cost or take at most this share of live's. */
+const HALF = 0.5;
+/** At least 30% lower. */
+const clearlyLess = (a: number, b: number) =>
+  Number.isFinite(a) && Number.isFinite(b) && a <= b * 0.7;
+
+/**
+ * Suggests the live and fallback models from a suite's board, by fixed rules, so the
+ * reason given is always the real one. Null with nothing current to go on.
+ *
+ * @example
+ * ```ts
+ * const board = await runner.board("ticket_triage");
+ * const s = suggest(board.rows, suite.checks, suite.priority, MODELS);
+ * if (s && !s.same) console.log(s.live.label, s.fallback?.label, s.reasons);
+ * ```
+ */
+export function suggest<R extends SuggestRow>(
+  rows: R[],
+  checks: CheckDef[],
+  priority: SuitePriority = { favor: "quality" },
+  /** Your model catalog: providers (for the fallback) and retirements. */
+  models: ModelCatalog = {},
+): Suggestion<R> | null {
+  const provider = (model: string) => models[model]?.provider ?? model;
+  const retiring = (r: SuggestRow) => retiresLabel(models, r.model);
+  const limitMs = priority.maxSeconds ? priority.maxSeconds * 1000 : Infinity;
+  const tooSlow = (r: SuggestRow) => ms(r) !== Infinity && ms(r) > limitMs;
+  const tested = rows.filter(
+    (r) =>
+      r.test &&
+      !r.test.stale &&
+      !r.test.failedAnswers &&
+      !retiring(r) &&
+      !tooSlow(r),
+  );
+  if (!tested.length) return null;
+  const serious = (r: R) => seriousMistakes(r, checks);
+  const live = tested.find((r) => r.live) ?? null;
+  const reasons: string[] = [];
+  const liveRow = rows.find((r) => r.live);
+  const liveRetires = liveRow ? retiring(liveRow) : null;
+  const liveSlow =
+    liveRow?.test && !liveRow.test.stale && tooSlow(liveRow)
+      ? `${liveRow.label} takes ${(ms(liveRow) / 1000).toFixed(1)}s, over this suite's ${priority.maxSeconds}s limit.`
+      : null;
+
+  // Live: switch only for a real, safe improvement.
+  let next: R;
+  const liveable = tested.filter((r) => !r.fallbackOnly);
+  if (!live) {
+    if (!liveable.length) return null;
+    next = at(
+      [...liveable].sort(
+        (a, b) =>
+          rate(b) - rate(a) || serious(a) - serious(b) || cost(a) - cost(b),
+      ),
+      0,
+    );
+    reasons.push(
+      liveRetires || liveSlow
+        ? `${liveRetires && liveRow ? `${liveRow.label} retires in ${liveRetires}.` : liveSlow} ${next.label} scores highest of the rest (${pct(rate(next))}) with ${mistakes(serious(next))}.`
+        : `${next.label} scores highest (${pct(rate(next))}) with ${mistakes(serious(next))}. The live model has no up-to-date test to compare with.`,
+    );
+  } else {
+    const why = (r: R): string | null => {
+      if (rate(r) < rate(live)) return trade(r);
+      if (r.vsLive === "BETTER")
+        return `${r.label} is clearly more accurate than ${live.label}: ${pct(rate(r))} against ${pct(rate(live))}.`;
+      if (serious(r) <= serious(live) - 2)
+        return `${r.label} makes fewer serious mistakes than ${live.label} (${serious(r)} against ${serious(live)}), and scores at least as well (${pct(rate(r))} against ${pct(rate(live))}).`;
+      if (priority.favor === "cost" && clearlyLess(cost(r), cost(live)))
+        return `${r.label} scores at least as well as ${live.label} (${pct(rate(r))} against ${pct(rate(live))}) at ${Math.round((1 - cost(r) / cost(live)) * 100)}% less per answer.`;
+      if (priority.favor === "speed" && clearlyLess(ms(r), ms(live)))
+        return `${r.label} scores at least as well as ${live.label} (${pct(rate(r))} against ${pct(rate(live))}) and answers faster (${(ms(r) / 1000).toFixed(1)}s against ${(ms(live) / 1000).toFixed(1)}s).`;
+      return null;
+    };
+    // Cost and speed suites: a little less accurate, no clear difference,
+    // and at most half the cost (or time).
+    const trade = (r: R): string | null => {
+      if (
+        priority.favor === "quality" ||
+        r.vsLive !== "SAME" ||
+        rate(live) - rate(r) > MAX_TRADE_DROP ||
+        serious(r) > serious(live)
+      )
+        return null;
+      const close = `${r.label} is about as accurate as ${live.label} (${pct(rate(r))} against ${pct(rate(live))}: closer than these cases can tell apart), with no more serious mistakes`;
+      if (priority.favor === "cost" && cost(r) <= cost(live) * HALF)
+        return `${close}, at ${Math.round((1 - cost(r) / cost(live)) * 100)}% less per answer. This suite favors cost.`;
+      if (priority.favor === "speed" && ms(r) <= ms(live) * HALF)
+        return `${close}, and answers in ${(ms(r) / 1000).toFixed(1)}s against ${(ms(live) / 1000).toFixed(1)}s. This suite favors speed.`;
+      return null;
+    };
+    const candidates = liveable
+      .filter(
+        (r) =>
+          !r.live &&
+          r.vsLive !== "WORSE" &&
+          serious(r) <= serious(live) &&
+          why(r),
+      )
+      .sort(
+        (a, b) =>
+          Number(b.vsLive === "BETTER") - Number(a.vsLive === "BETTER") ||
+          // A trade-off only when nothing scores as well.
+          Number(rate(b) >= rate(live)) - Number(rate(a) >= rate(live)) ||
+          rate(b) - rate(a) ||
+          serious(a) - serious(b) ||
+          (priority.favor === "speed"
+            ? ms(a) - ms(b) || cost(a) - cost(b)
+            : cost(a) - cost(b) || ms(a) - ms(b)),
+      );
+    if (candidates.length) {
+      next = at(candidates, 0);
+      const reason = why(next);
+      if (reason) reasons.push(reason);
+    } else {
+      next = live;
+      reasons.push(
+        `Keep ${live.label}: no other model is safely better on these cases.`,
+      );
+      // The one to watch: scored higher, but not by enough to tell.
+      const watch = tested
+        .filter((r) => !r.live && rate(r) > rate(live))
+        .sort((a, b) => rate(b) - rate(a))[0];
+      if (watch)
+        reasons.push(
+          `${watch.label} scored highest (${pct(rate(watch))} against ${pct(rate(live))}), but with this many cases the gap can't be told from noise${serious(watch) > serious(live) ? `, and it makes more serious mistakes (${serious(watch)} against ${serious(live)})` : ""}. Add cases, then test again.`,
+        );
+    }
+  }
+
+  // Fallback: the most accurate from another provider, at the operation's
+  // setting (the setting is shared, so setting rows don't qualify).
+  const fallback =
+    tested
+      .filter(
+        (r) =>
+          !r.variant &&
+          !r.liveOnly &&
+          r.key !== next.key &&
+          provider(r.model) !== provider(next.model),
+      )
+      .sort(
+        (a, b) =>
+          rate(b) - rate(a) || serious(a) - serious(b) || cost(a) - cost(b),
+      )[0] ?? null;
+  if (fallback)
+    reasons.push(
+      `Fallback: ${fallback.label} is the most accurate from another provider (${pct(rate(fallback))}, ${mistakes(serious(fallback))}).`,
+    );
+
+  const keepLive = next.live;
+  return {
+    live: next,
+    fallback,
+    keepLive,
+    same: keepLive && (!fallback || fallback.fallback),
+    reasons,
+  };
+}

package/src/summary.ts

@@ -0,0 +1,109 @@
+import type { SuiteMetrics } from "./stats.js";
+import type { CheckDef } from "./types.js";
+
+/**
+ * A test result in plain words, for people who don't want the statistics:
+ * how many answers were fully right, how far to trust that, and the most
+ * common problems. Pure.
+ */
+
+/** A check that failed, and how often. */
+export type Problem = {
+  key: string;
+  label: string;
+  failed: number;
+  of: number;
+};
+
+/**
+ * A test in plain words: answers right, how far to trust that, and the common problems.
+ */
+export type Summary = {
+  right: number;
+  total: number;
+  /** One sentence on how much the number can be trusted. */
+  trust: string;
+  /** Checks that failed at least once, most failures first. */
+  problems: Problem[];
+};
+
+/** Below this many answers the number is mostly noise. */
+export const TRUST_MIN = 10;
+
+/**
+ * A test's result in plain words, for people who don't want the statistics.
+ */
+export function summarize(m: SuiteMetrics, checks: CheckDef[]): Summary {
+  const right = m.allPass.passed;
+  const total = m.allPass.n;
+  const spread =
+    m.allPass.low !== null && m.allPass.high !== null
+      ? Math.round(((m.allPass.high - m.allPass.low) / 2) * 100)
+      : null;
+  const trust =
+    total < TRUST_MIN
+      ? `Only ${total} ${total === 1 ? "answer" : "answers"}: too few to trust yet. Confirm more cases, then test again.`
+      : spread !== null && spread > 10
+        ? `A rough number: with ${total} answers it could be about ${spread} points either way.`
+        : `A solid number: ${total} answers.`;
+  const problems = checks
+    .map((c) => {
+      const r = m.byCheck[c.key];
+      return r && r.n
+        ? { key: c.key, label: c.label, failed: r.n - r.passed, of: r.n }
+        : null;
+    })
+    .filter((p): p is Problem => p !== null && p.failed > 0)
+    .sort((a, b) => b.failed - a.failed);
+  return { right, total, trust, problems };
+}
+
+/**
+ * How far a score on this many cases can be trusted, and how far off it could be.
+ */
+export type Confidence = {
+  level: "too-few" | "rough" | "good" | "strong";
+  label: string;
+  /** About how many points a score could be off, either way (95%). */
+  margin: number;
+  /** What the count means for choosing a model, in one sentence. */
+  sentence: string;
+};
+
+/**
+ * How far to trust scores on this many cases. The margin is the 95% range
+ * of a pass rate (at the given rate, or 50%, the widest), in points.
+ */
+export function confidenceFor(cases: number, rate = 0.5): Confidence {
+  const p = Math.min(Math.max(rate, 0.05), 0.95);
+  const margin = cases
+    ? Math.round(1.96 * Math.sqrt((p * (1 - p)) / cases) * 100)
+    : 100;
+  const level =
+    cases < 20
+      ? "too-few"
+      : cases < 50
+        ? "rough"
+        : cases < 100
+          ? "good"
+          : "strong";
+  const label = {
+    "too-few": "too few",
+    rough: "rough",
+    good: "good",
+    strong: "strong",
+  }[level];
+  const sentence =
+    level === "too-few"
+      ? `With ${cases} cases a score could be off by about ${margin} points: only big problems show. Aim for 50.`
+      : `With ${cases} cases a score could be off by about ${margin} points either way, so models closer than that can't be told apart.${level === "rough" ? " Aim for 50." : ""}`;
+  return { level, label, margin, sentence };
+}
+
+/** The answer key size to aim for (see confidenceFor). */
+export const TARGET_CASES = 50;
+
+/** How many drafts get the key to the target: at least 10, at most 30 a batch. */
+export function draftsToTarget(casesNotRetired: number): number {
+  return Math.min(30, Math.max(10, TARGET_CASES - casesNotRetired));
+}