@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/src/testing.ts

@@ -0,0 +1,392 @@
+/**
+ * The contracts every store must meet, as Vitest tests.
+ *
+ * @module evalkeeper/testing
+ */
+import { describe, expect, it } from "vitest";
+import { z } from "zod";
+import type { FeedbackStore, NewRating } from "./feedback.js";
+import { createRunner } from "./runner.js";
+import {
+  SettingsConflict,
+  type Settings,
+  type SettingsStore,
+} from "./settings.js";
+import type { EvalStore, NewCase } from "./store.js";
+import type { Suite } from "./types.js";
+
+/**
+ * The contract every EvalStore must meet, as Vitest tests. Run it against
+ * your store to know the runner will work on it. `make` must return an
+ * empty store each time it's called.
+ *
+ * @example
+ * ```ts
+ * import { describeStoreContract } from "@crowandraven/evalkeeper/testing";
+ *
+ * describeStoreContract("my store", () => myStore(freshDatabase()));
+ * ```
+ */
+export function describeStoreContract(
+  name: string,
+  make: () => EvalStore | Promise<EvalStore>,
+) {
+  const newCase = (suite: string, over: Partial<NewCase> = {}): NewCase => ({
+    suite,
+    input: { text: "hello", nested: { n: 1 } },
+    tags: ["a", "b"],
+    split: "DEV",
+    status: "ACTIVE",
+    source: "OWNER",
+    sourceRef: null,
+    note: null,
+    labels: [
+      {
+        kind: "EXPECTED",
+        status: "CONFIRMED",
+        value: { want: "HELLO" },
+        raterRole: "OWNER",
+        raterId: null,
+        note: null,
+      },
+    ],
+    ...over,
+  });
+  const newRun = (suite: string) => ({
+    suite,
+    model: "model-a",
+    promptVersion: "v1",
+    reasoningEffort: null,
+    set: "ALL" as const,
+    repeats: 1,
+    datasetVersion: null,
+    graderVersion: "1",
+    estimatedCostUsd: null,
+    startedBy: null,
+  });
+
+  describe(`EvalStore contract: ${name}`, () => {
+    it("keeps a case's input, tags, and answers as given", async () => {
+      const store = await make();
+      const c = await store.createCase(newCase("s"));
+      const [listed] = await store.listCases("s");
+      expect(listed?.id).toBe(c.id);
+      expect(listed?.input).toEqual({ text: "hello", nested: { n: 1 } });
+      expect(listed?.tags).toEqual(["a", "b"]);
+      expect(listed?.labels).toHaveLength(1);
+      expect(listed?.labels[0]).toMatchObject({
+        kind: "EXPECTED",
+        status: "CONFIRMED",
+        value: { want: "HELLO" },
+      });
+    });
+
+    it("lists a suite's cases oldest first, leaving out retired ones", async () => {
+      const store = await make();
+      const first = await store.createCase(newCase("s"));
+      const second = await store.createCase(newCase("s"));
+      await store.createCase(newCase("other"));
+      await store.setCaseStatus(first.id, "RETIRED");
+      expect((await store.listCases("s")).map((c) => c.id)).toEqual([
+        second.id,
+      ]);
+    });
+
+    it("adds answers without overwriting, and changes their status", async () => {
+      const store = await make();
+      const c = await store.createCase(newCase("s", { labels: [] }));
+      const l = await store.addLabel(c.id, {
+        kind: "EXPECTED",
+        status: "PENDING",
+        value: { want: "X" },
+        raterRole: "MODEL",
+        raterId: null,
+        note: null,
+      });
+      await store.setLabelStatus(l.id, "CONFIRMED");
+      const [listed] = await store.listCases("s");
+      expect(listed?.labels).toHaveLength(1);
+      expect(listed?.labels[0]?.status).toBe("CONFIRMED");
+    });
+
+    it("starts runs as running, lists them newest first, and filters", async () => {
+      const store = await make();
+      const a = await store.createRun(newRun("s"));
+      await new Promise((r) => setTimeout(r, 5));
+      const b = await store.createRun(newRun("s"));
+      expect(a.status).toBe("RUNNING");
+      expect((await store.listRuns("s")).map((r) => r.id)).toEqual([
+        b.id,
+        a.id,
+      ]);
+      expect(await store.listRuns("s", { limit: 1 })).toHaveLength(1);
+      await store.updateRun(a.id, { status: "SUCCEEDED" });
+      expect(
+        (await store.listRuns("s", { status: "SUCCEEDED" })).map((r) => r.id),
+      ).toEqual([a.id]);
+      expect(await store.getRun("missing")).toBeNull();
+    });
+
+    it("updates a run only while it has the status it's guarded by", async () => {
+      const store = await make();
+      const r = await store.createRun(newRun("s"));
+      await store.updateRun(r.id, { status: "FAILED" }, { status: "RUNNING" });
+      await store.updateRun(
+        r.id,
+        { status: "SUCCEEDED" },
+        { status: "RUNNING" },
+      );
+      expect((await store.getRun(r.id))?.status).toBe("FAILED");
+    });
+
+    it("stores outputs and scores, and finds only originals to reuse", async () => {
+      const store = await make();
+      const c = await store.createCase(newCase("s"));
+      const r = await store.createRun(newRun("s"));
+      const base = {
+        runId: r.id,
+        caseId: c.id,
+        repeat: 1,
+        requestHash: "h",
+        output: { text: "HELLO" },
+        error: null,
+        scores: null,
+        costUsd: 0.01,
+        ms: 12,
+        reused: false,
+      };
+      const o = await store.createOutput(base);
+      await store.createOutput({ ...base, reused: true });
+      await store.createOutput({ ...base, error: "boom", output: null });
+      await store.setOutputScores(o.id, [{ key: "match", pass: true }]);
+      const outs = await store.listOutputs(r.id);
+      expect(outs).toHaveLength(3);
+      expect(outs.find((x) => x.id === o.id)?.scores).toEqual([
+        { key: "match", pass: true },
+      ]);
+      const found = await store.findReusableOutput("h", new Date(0));
+      expect(found?.id).toBe(o.id);
+      expect(found?.output).toEqual({ text: "HELLO" });
+      expect(
+        await store.findReusableOutput("h", new Date(Date.now() + 60_000)),
+      ).toBeNull();
+    });
+
+    it("runs a whole test through the runner", async () => {
+      const store = await make();
+      const echo: Suite<{ text: string }, { text: string }, { want: string }> =
+        {
+          key: "contract",
+          operation: "contract",
+          priority: { favor: "quality" },
+          label: "Contract",
+          description: "Uppercases text.",
+          graderVersion: "1",
+          input: z.object({ text: z.string() }),
+          labelValue: z.object({ want: z.string() }),
+          checks: [
+            {
+              key: "match",
+              label: "Matches",
+              description: "",
+              kind: "label",
+              severity: "serious",
+            },
+          ],
+          run: async (i) => ({ text: i.text.toUpperCase() }),
+          grade: async (_i, out, want) => [
+            { key: "match", pass: want ? out.text === want.want : null },
+          ],
+          inputFromForm: () => ({ text: "" }),
+          title: (i) => i.text,
+        };
+      await store.createCase(newCase("contract"));
+      const runner = createRunner({
+        store,
+        suites: [echo],
+        models: { "model-a": { label: "Model A", provider: "acme" } },
+        promptVersion: () => "v1",
+      });
+      const { run } = await runner.createRun({
+        suite: "contract",
+        model: "model-a",
+      });
+      await runner.runEval(run.id);
+      const done = await store.getRun(run.id);
+      expect(done?.status).toBe("SUCCEEDED");
+      expect(done?.metrics?.allPass.passed).toBe(1);
+    });
+  });
+}
+
+/**
+ * The contract every FeedbackStore must meet. `make` must return an empty
+ * store each time it's called.
+ */
+export function describeFeedbackStoreContract(
+  name: string,
+  make: () => FeedbackStore | Promise<FeedbackStore>,
+) {
+  const rating = (over: Partial<NewRating> = {}): NewRating => ({
+    operation: "triage",
+    targetId: "answer-1",
+    raterId: "person-1",
+    rating: "DOWN",
+    reason: "Wrong queue",
+    excerpt: "billing",
+    input: { subject: "Refund", body: "Please refund me." },
+    output: { category: "bug" },
+    ...over,
+  });
+
+  describe(`FeedbackStore contract: ${name}`, () => {
+    it("keeps a rating with its reason, input, and output, open", async () => {
+      const store = await make();
+      const f = await store.rate(rating());
+      expect(f).toMatchObject({
+        operation: "triage",
+        rating: "DOWN",
+        reason: "Wrong queue",
+        excerpt: "billing",
+        input: { subject: "Refund", body: "Please refund me." },
+        output: { category: "bug" },
+        status: "OPEN",
+        caseId: null,
+      });
+    });
+
+    it("keeps one rating per person per answer, and reopens a changed one", async () => {
+      const store = await make();
+      const first = await store.rate(rating());
+      await store.setFeedbackStatus(first.id, "DISMISSED");
+      const changed = await store.rate(rating({ rating: "UP" }));
+      expect(changed.id).toBe(first.id);
+      expect(changed.status).toBe("OPEN");
+      expect(changed.reason).toBeNull();
+      await store.rate(rating({ raterId: "person-2" }));
+      expect(await store.listFeedback()).toHaveLength(2);
+    });
+
+    it("takes a rating back", async () => {
+      const store = await make();
+      await store.rate(rating());
+      await store.clearRating("person-1", "triage", "answer-1");
+      expect(await store.listFeedback()).toHaveLength(0);
+    });
+
+    it("filters by feature, rating, status, and date, newest first", async () => {
+      const store = await make();
+      const a = await store.rate(rating({ targetId: "a" }));
+      await new Promise((r) => setTimeout(r, 5));
+      const b = await store.rate(rating({ targetId: "b", rating: "UP" }));
+      await store.rate(rating({ targetId: "c", operation: "summary" }));
+      expect(
+        (await store.listFeedback({ operation: "triage" })).map((f) => f.id),
+      ).toEqual([b.id, a.id]);
+      expect(await store.listFeedback({ rating: "UP" })).toHaveLength(1);
+      await store.setFeedbackStatus(a.id, "CASED", "case-1");
+      const cased = await store.listFeedback({ status: "CASED" });
+      expect(cased.map((f) => [f.id, f.caseId])).toEqual([[a.id, "case-1"]]);
+      expect(
+        await store.listFeedback({ since: new Date(Date.now() + 60_000) }),
+      ).toHaveLength(0);
+      expect(await store.listFeedback({ limit: 1 })).toHaveLength(1);
+    });
+  });
+}
+
+/**
+ * The contract every SettingsStore must meet. `make` must return an empty
+ * store each time it's called.
+ */
+export function describeSettingsStoreContract(
+  name: string,
+  make: () => SettingsStore | Promise<SettingsStore>,
+) {
+  const settings = (model: string): Settings => ({
+    model,
+    fallbackModel: null,
+    reasoningEffort: null,
+    maxOutputTokens: null,
+  });
+  const create = (model: string, expectLive: string | null) => ({
+    create: { settings: settings(model), promptVersion: "p1" },
+    kind: "CHANGE" as const,
+    reason: `to ${model}`,
+    by: "person-1",
+    expectLive,
+  });
+
+  describe(`SettingsStore contract: ${name}`, () => {
+    it("has nothing live until the first switch, then numbers versions", async () => {
+      const store = await make();
+      expect(await store.liveVersion("triage")).toBeNull();
+      const v1 = await store.switchTo("triage", create("a", null));
+      const v2 = await store.switchTo("triage", create("b", v1.id));
+      expect([v1.number, v2.number]).toEqual([1, 2]);
+      expect(v2).toMatchObject({
+        operation: "triage",
+        model: "b",
+        promptVersion: "p1",
+      });
+      expect((await store.liveVersion("triage"))?.id).toBe(v2.id);
+      expect((await store.listVersions("triage")).map((v) => v.number)).toEqual(
+        [2, 1],
+      );
+    });
+
+    it("logs every switch, newest first, with who, why, and evidence", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", create("a", null));
+      await store.switchTo("triage", {
+        ...create("b", v1.id),
+        kind: "PROMOTE",
+        evidence: { runId: "r1", verdict: "BETTER" },
+      });
+      const [latest, first] = await store.listChanges("triage");
+      expect(latest).toMatchObject({
+        kind: "PROMOTE",
+        fromVersionId: v1.id,
+        reason: "to b",
+        by: "person-1",
+        evidence: { runId: "r1", verdict: "BETTER" },
+      });
+      expect(first).toMatchObject({ kind: "CHANGE", fromVersionId: null });
+      expect(await store.listChanges("triage", { limit: 1 })).toHaveLength(1);
+    });
+
+    it("makes an earlier version live again without a new one", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", create("a", null));
+      const v2 = await store.switchTo("triage", create("b", v1.id));
+      await store.switchTo("triage", {
+        versionId: v1.id,
+        kind: "ROLLBACK",
+        reason: "b was worse",
+        by: "person-1",
+        expectLive: v2.id,
+      });
+      expect((await store.liveVersion("triage"))?.id).toBe(v1.id);
+      expect(await store.listVersions("triage")).toHaveLength(2);
+    });
+
+    it("refuses a switch decided against a live version that changed", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", create("a", null));
+      await store.switchTo("triage", create("b", v1.id));
+      await expect(
+        store.switchTo("triage", create("c", v1.id)),
+      ).rejects.toBeInstanceOf(SettingsConflict);
+      expect((await store.liveVersion("triage"))?.model).toBe("b");
+      expect(await store.listVersions("triage")).toHaveLength(2);
+    });
+
+    it("keeps each feature's versions and live one apart", async () => {
+      const store = await make();
+      await store.switchTo("triage", create("a", null));
+      const other = await store.switchTo("summary", create("x", null));
+      expect(other.number).toBe(1);
+      expect((await store.liveVersion("triage"))?.model).toBe("a");
+    });
+  });
+}

package/src/timeout.ts

@@ -0,0 +1,14 @@
+/** Rejects if the call takes longer than `ms`. Pure. */
+export function withTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
+  let timer: ReturnType<typeof setTimeout>;
+  return Promise.race([
+    p,
+    new Promise<never>((_, reject) => {
+      timer = setTimeout(() => {
+        reject(new Error(`No answer within ${ms / 1000}s`));
+      }, ms);
+    }),
+  ]).finally(() => {
+    clearTimeout(timer);
+  });
+}

package/src/types.ts

@@ -0,0 +1,252 @@
+import type { z } from "zod";
+import type { ReasoningEffort } from "./models.js";
+
+/**
+ * An eval suite: how to run one AI operation on a frozen case and how to
+ * grade what comes back. Code says how to evaluate; the database holds the
+ * cases and labels (EvalCase, EvalCaseLabel). See docs/evals.md.
+ */
+
+/** One check on one output. null: doesn't apply to this case. */
+export type CheckResult = {
+  key: string;
+  pass: boolean | null;
+  /** Why it failed (or a detail worth showing), in plain words. */
+  note?: string | undefined;
+  /** For matching checks: which expected item matched which output, and why. */
+  matches?: { expected: string; actual: string; why: string }[] | undefined;
+  /** "meaning": a judge model matched them; "words": shared words did. */
+  matchedBy?: "meaning" | "words" | undefined;
+  /**
+   * Partial credit: of the parts this answer should have (e.g. expected
+   * items), how many it got right. The headline score sums these.
+   */
+  units?: { right: number; total: number } | undefined;
+  /**
+   * Numbers measured on this answer, averaged across a test: a decision's
+   * `brier`, `logLoss`, `confidence`, and `correct` (see decisionCheck).
+   */
+  measures?: Record<string, number> | undefined;
+};
+
+/**
+ * One check a suite runs: what it checks, whether it needs an answer, and how serious a
+ * miss is.
+ */
+export type CheckDef = {
+  key: string;
+  label: string;
+  /** What it checks, for the page that explains the suite. */
+  description: string;
+  /** "code": needs no label. "label": compares with a confirmed label. */
+  kind: "code" | "label";
+  /**
+   * Serious: changes what the product does (which queue a ticket lands
+   * in, what a summary claims). Minor: a fair difference of judgment (a category, how finely
+   * to split). Models are ranked on serious mistakes.
+   */
+  severity: "serious" | "minor";
+  /** Pass rate it should reach; default by severity (checkBar). */
+  bar?: number;
+};
+
+/** The pass rate a check should reach: its own, else 95% serious, 75% minor. */
+export function checkBar(c: CheckDef): number {
+  return c.bar ?? (c.severity === "serious" ? 0.95 : 0.75);
+}
+
+/** Meets the bar, is within 10 points of it, or is further below. */
+export function barStatus(
+  rate: number,
+  bar: number,
+): "meets" | "close" | "below" {
+  return rate >= bar ? "meets" : rate >= bar - 0.1 ? "close" : "below";
+}
+
+/**
+ * What a suite cares about once accuracy holds. Accuracy is always the
+ * floor; this is the one thing worth switching models for beyond it.
+ */
+export type SuitePriority = {
+  /** speed: someone waits on it. cost: runs in the background, often.
+   * quality: people read and act on it, so only accuracy counts. */
+  favor: "speed" | "cost" | "quality";
+  /** Models slower than this on average are never suggested. */
+  maxSeconds?: number;
+};
+
+/** Each priority, explained for the suite page. */
+export const FAVOR_TEXT: Record<SuitePriority["favor"], string> = {
+  speed:
+    "speed: someone waits on this, so a model that scores as well and answers clearly faster is worth switching to, and so is one about as accurate (within 10 points) in half the time",
+  cost: "cost: this runs in the background, often, so a model that scores as well and is clearly cheaper is worth switching to, and so is one about as accurate (within 10 points) at half the cost",
+  quality:
+    "quality: people read and act on this, so only clearly better accuracy or fewer serious mistakes is worth switching for",
+};
+
+/**
+ * An eval suite: how to call one AI feature on a model and how to grade what comes back.
+ * Code says how to evaluate; the store holds the cases and answers.
+ */
+export type Suite<Input, Output, Label> = {
+  key: string;
+  /** The AI feature it tests, as your app names it ("summarize_ticket"). */
+  operation: string;
+  priority: SuitePriority;
+  label: string;
+  description: string;
+  /** Bump when checks change, so runs say which grading they used. */
+  graderVersion: string;
+  input: z.ZodType<Input>;
+  /** The EXPECTED label's value. */
+  labelValue: z.ZodType<Label>;
+  checks: CheckDef[];
+  /** Calls the operation on exactly this model, recorded as an eval. */
+  run(
+    input: Input,
+    ctx: {
+      model: string;
+      /** A reasoning setting to test; undefined: the operation's own. */
+      reasoningEffort?: ReasoningEffort | null;
+      onCost: (usd: number) => void;
+    },
+  ): Promise<Output>;
+  /**
+   * Grades one output. `label` is the confirmed one, or null. A check that
+   * asks a judge model reports its cost through `ctx.onCost`.
+   */
+  grade(
+    input: Input,
+    output: Output,
+    label: Label | null,
+    ctx?: { onCost: (usd: number) => void },
+  ): Promise<CheckResult[]>;
+  /**
+   * Checks that compare a test's answers with each other, such as ranking.
+   * Gets every graded answer of one repeat; returns extra checks for each,
+   * in the same order. Pure; runs after a test and on every regrade.
+   */
+  gradeTogether?(
+    answers: { input: Input; output: Output; label: Label | null }[],
+  ): CheckResult[][];
+  /** Reads a case's input from the add/edit form; throws on bad input. */
+  inputFromForm(form: FormData): Input;
+  /** Reads an answer from the add/edit form; throws on bad input. */
+  labelFromForm?(form: FormData): Label;
+  /**
+   * The operation's output as an answer, so a thumbs-up can propose it
+   * (pending until a person confirms it). Omit when outputs and answers
+   * differ too much to propose one.
+   */
+  labelFromOutput?(output: Output): Label;
+  /**
+   * The form's fields, so pages can render one for any suite. Every field
+   * has a plain label and help: people confirming answers weeks later
+   * won't remember what a field means.
+   */
+  fields?: { input: FormField[]; label: FormField[] };
+  /** Drafted cases to start from, added as drafts with PENDING labels. */
+  starter?: {
+    input: Input;
+    label: Label;
+    tags: string[];
+    split: "DEV" | "TEST";
+    smoke?: boolean;
+    note: string;
+  }[];
+  /**
+   * Drafts new cases aimed at what the answer key is missing, for a person
+   * to review (they arrive with PENDING answers).
+   */
+  draftMore?(args: {
+    existing: { input: Input; tags: string[] }[];
+    count: number;
+    focus: string | null;
+  }): Promise<{ input: Input; label: Label; tags: string[]; note: string }[]>;
+  /**
+   * How long one answer may take before it counts as failed. Default 60s;
+   * multi-step agent turns need more. Keep it under the run budget.
+   */
+  answerTimeoutMs?: number;
+  /**
+   * How many answers a test asks for at once. Default 6; slow operations
+   * need more to finish before the platform's time limit.
+   */
+  concurrency?: number;
+  /**
+   * Graded as decisions (typed answers with probabilities, as from the AI
+   * SDK's `experimental_decide`), so pages show probability quality and
+   * calibration next to accuracy, and which models' probabilities are
+   * native or prompted.
+   */
+  decisions?: boolean;
+  /** What the headline counts: "items", "tickets", … Default "answers". */
+  unitNoun?: string;
+  /**
+   * "typed": cases are text someone types (added by hand, or drafted).
+   * "imported": cases are frozen from the app's own data, then rated by a
+   * person.
+   * "fixture": cases are structured inputs, shown with describe() and
+   * edited as JSON.
+   */
+  caseKind?: "typed" | "imported" | "fixture";
+  /** A short line naming the case in lists. */
+  title(input: Input): string;
+  /** Fixture suites: the whole input, readable, for reviewing a case. */
+  describe?(input: Input): string;
+  /** Fixture suites: what the expected answer's fields mean. */
+  labelHelp?: string;
+};
+
+/**
+ * One field of a suite's add/edit form. `name` is the form key (and the
+ * input or answer property it fills). "lines": one item per line, read as
+ * a list.
+ */
+export type FormField = {
+  name: string;
+  label: string;
+  help: string;
+  kind: "text" | "textarea" | "select" | "lines";
+  /** For "select". */
+  options?: readonly string[];
+  required?: boolean;
+};
+
+/** A value as plain text: strings as they are, other values as JSON. */
+export function asText(v: unknown): string {
+  if (v === undefined || v === null) return "";
+  if (typeof v === "string") return v;
+  if (typeof v === "number" || typeof v === "boolean") return String(v);
+  return JSON.stringify(v);
+}
+
+/** A form value as text: "" when missing or a file. */
+export function formText(form: FormData, name: string): string {
+  const v = form.get(name);
+  return typeof v === "string" ? v.trim() : "";
+}
+
+/** A field's value from a stored input or answer, as the form shows it. */
+export function fieldValue(field: FormField, value: unknown): string {
+  const v =
+    value && typeof value === "object"
+      ? (value as Record<string, unknown>)[field.name]
+      : undefined;
+  if (Array.isArray(v)) return v.map(asText).join("\n");
+  return asText(v);
+}
+
+/** A "lines" field's value as a list: one item per non-empty line. */
+export function linesOf(form: FormData, name: string): string[] {
+  return formText(form, name)
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(Boolean);
+}
+
+/**
+ * A suite of any shape, for registries. Its methods take and return
+ * `unknown`; a suite's own code keeps its precise types.
+ */
+export type AnySuite = Suite<unknown, unknown, unknown>;