@crowandraven/evalkeeper 0.2.0 → 0.3.0

package/src/importCases.ts

@@ -0,0 +1,252 @@
+import { nextSplit } from "./quickSet.js";
+import { stableJson } from "./stableJson.js";
+import type { EvalStore, NewCase, NewLabel } from "./store.js";
+import type { AnySuite } from "./types.js";
+
+/** One row of an import, mapped to the suite's shapes. */
+export type ImportedCase = {
+  /** Stable within its source (a row id in the file), so a rerun skips it. */
+  id: string;
+  /** The suite's input; checked against `suite.input`. */
+  input: unknown;
+  /** The answer, checked against `suite.labelValue`; null: none yet. */
+  label: unknown;
+  tags?: string[];
+  note?: string | null;
+  /** This row's set, instead of the import's `split`. */
+  split?: "DEV" | "TEST";
+};
+
+/**
+ * How imported answers arrive. Pending (the default): a person confirms
+ * each one on the pages before tests use it. Provisional: tests use them
+ * now, every score says it rests on unreviewed answers, and the rows carry
+ * `tag` so the review queue is findable. Use provisional only when the
+ * owner said so; `allowedBy` and `at` record who and when. Nothing is
+ * imported confirmed: only a person confirms an answer.
+ */
+export type ImportAnswers =
+  | { status: "PENDING" }
+  | {
+      status: "PROVISIONAL";
+      /** Who drafted the answers: an agent's name, recorded as the rater. */
+      rater: string;
+      /** The person who allowed using them before review. */
+      allowedBy: string;
+      /** When they allowed it; default now. */
+      at?: Date;
+      /** Added to each row's tags; default "needs-review". */
+      tag?: string;
+    };
+
+/** A row that couldn't be imported, and why. */
+export type ImportProblem = { id: string; problem: string };
+
+/** Rows that don't fit the suite. Nothing was written. */
+export class CaseImportError extends Error {
+  readonly problems: ImportProblem[];
+  constructor(source: string, problems: ImportProblem[]) {
+    super(
+      `${problems.length} row${problems.length === 1 ? "" : "s"} in ${source} can't be imported, so none were: ${problems
+        .slice(0, 5)
+        .map((p) => `${p.id} (${p.problem})`)
+        .join("; ")}${problems.length > 5 ? "; …" : ""}`,
+    );
+    this.name = "CaseImportError";
+    this.problems = problems;
+  }
+}
+
+/** What an import did. */
+export type ImportResult = {
+  /** Cases created. */
+  added: number;
+  /** Rows imported before (by source and id), left alone. */
+  skipped: number;
+};
+
+const sourceRefOf = (source: string, id: string) => `${source}#${id}`;
+
+/**
+ * Imports existing test data as cases for one suite. Every row is mapped
+ * and checked against the suite's input and answer schemas first; if any
+ * row fails, nothing is written and `CaseImportError` lists them. Rows
+ * imported before (same source and id, retired ones included) are skipped,
+ * so it can run again after the file grows.
+ *
+ * Splits: "DEV" for cases you tune against, "TEST" for a held-out file,
+ * "auto" (the default) for new cases, split like cases added by hand.
+ *
+ * Run it against your development database; an app's own guard should
+ * refuse a production one.
+ *
+ * @example
+ * ```ts
+ * const { added, skipped } = await importCases({
+ *   store,
+ *   suite: ticketTriage,
+ *   source: "tickets-heldout.json",
+ *   rows: JSON.parse(await readFile("tickets-heldout.json", "utf8")),
+ *   map: (r) => ({ id: r.id, input: { body: r.text }, label: { queue: r.queue } }),
+ *   split: "TEST",
+ * });
+ * ```
+ */
+export async function importCases<Row>(opts: {
+  store: EvalStore;
+  suite: AnySuite;
+  /** Where the rows came from, such as a file name. */
+  source: string;
+  rows: readonly Row[];
+  map: (row: Row, index: number) => ImportedCase;
+  split?: "DEV" | "TEST" | "auto";
+  answers?: ImportAnswers;
+}): Promise<ImportResult> {
+  const { store, suite, source } = opts;
+  const answers = opts.answers ?? { status: "PENDING" as const };
+
+  // 1. Map and check every row before writing anything.
+  const problems: ImportProblem[] = [];
+  const seen = new Set<string>();
+  const mapped: (ImportedCase & { input: unknown; label: unknown })[] = [];
+  opts.rows.forEach((row, i) => {
+    let c: ImportedCase;
+    try {
+      c = opts.map(row, i);
+    } catch (e) {
+      problems.push({
+        id: `row ${i + 1}`,
+        problem: e instanceof Error ? e.message : String(e),
+      });
+      return;
+    }
+    const id = c.id || `row ${i + 1}`;
+    if (!c.id) problems.push({ id, problem: "no id" });
+    else if (seen.has(c.id))
+      problems.push({ id, problem: "the same id as an earlier row" });
+    seen.add(c.id);
+    const input = suite.input.safeParse(c.input);
+    if (!input.success)
+      problems.push({
+        id,
+        problem: `input: ${input.error.issues[0]?.message ?? "invalid"}`,
+      });
+    let label: unknown = null;
+    if (c.label !== null && c.label !== undefined) {
+      const l = suite.labelValue.safeParse(c.label);
+      if (!l.success)
+        problems.push({
+          id,
+          problem: `answer: ${l.error.issues[0]?.message ?? "invalid"}`,
+        });
+      else label = l.data;
+    }
+    if (input.success) mapped.push({ ...c, input: input.data, label });
+  });
+  if (problems.length) throw new CaseImportError(source, problems);
+
+  // 2. Skip what's already there, retired cases included.
+  const existing = await store.listCases(suite.key, { includeRetired: true });
+  const refs = new Set(existing.map((c) => c.sourceRef));
+  const counts = { DEV: 0, TEST: 0 };
+  for (const c of existing) if (c.status !== "RETIRED") counts[c.split] += 1;
+
+  const at =
+    answers.status === "PROVISIONAL" ? (answers.at ?? new Date()) : null;
+  let added = 0;
+  let skipped = 0;
+  for (const c of mapped) {
+    const sourceRef = sourceRefOf(source, c.id);
+    if (refs.has(sourceRef)) {
+      skipped += 1;
+      continue;
+    }
+    const split =
+      c.split ??
+      (opts.split === "DEV" || opts.split === "TEST"
+        ? opts.split
+        : nextSplit(counts));
+    counts[split] += 1;
+    const labels: NewLabel[] = [];
+    if (c.label !== null)
+      labels.push(
+        answers.status === "PROVISIONAL"
+          ? {
+              kind: "EXPECTED",
+              status: "PROVISIONAL",
+              value: c.label,
+              raterRole: "AGENT",
+              raterId: answers.rater,
+              note: `Drafted by ${answers.rater}; ${answers.allowedBy} allowed testing on it before review (${at?.toISOString().slice(0, 10)}).`,
+            }
+          : {
+              kind: "EXPECTED",
+              status: "PENDING",
+              value: c.label,
+              raterRole: "IMPORT",
+              raterId: null,
+              note: null,
+            },
+      );
+    const tags = [...(c.tags ?? [])];
+    if (answers.status === "PROVISIONAL" && c.label !== null) {
+      const tag = answers.tag ?? "needs-review";
+      if (!tags.includes(tag)) tags.push(tag);
+    }
+    const data: NewCase = {
+      suite: suite.key,
+      input: c.input,
+      tags,
+      split,
+      // In use only with an answer tests may grade against.
+      status:
+        c.label !== null && answers.status === "PROVISIONAL"
+          ? "ACTIVE"
+          : "DRAFT",
+      source: "IMPORT",
+      sourceRef,
+      note: c.note ?? null,
+      labels,
+    };
+    await store.createCase(data);
+    refs.add(sourceRef);
+    added += 1;
+  }
+  return { added, skipped };
+}
+
+/** A short, stable id for a starter case: a hash of its input. */
+function starterId(input: unknown): string {
+  // FNV-1a: the core stays free of node:crypto; this only names rows.
+  let h = 0x811c9dc5;
+  for (const ch of stableJson(input)) {
+    h ^= ch.charCodeAt(0);
+    h = Math.imul(h, 0x01000193) >>> 0;
+  }
+  return h.toString(16).padStart(8, "0");
+}
+
+/**
+ * Adds a suite's `starter` cases as drafts with pending answers, for a
+ * person to confirm on the pages. Safe to run again: starters already
+ * added (by their input) are skipped.
+ */
+export function importStarters(
+  store: EvalStore,
+  suite: AnySuite,
+): Promise<ImportResult> {
+  return importCases({
+    store,
+    suite,
+    source: `starter:${suite.key}`,
+    rows: suite.starter ?? [],
+    map: (s) => ({
+      id: starterId(s.input),
+      input: s.input,
+      label: s.label,
+      tags: s.tags,
+      note: s.note,
+      split: s.split,
+    }),
+  });
+}

package/src/index.ts

@@ -31,3 +31,4 @@ export * from "./store.js";
 export * from "./labels.js";
 // Server-only (node:crypto): import from "@crowandraven/evalkeeper/runner".
 // In-memory store for tests and demos: "@crowandraven/evalkeeper/memory".
+export * from "./importCases.js";

package/src/labels.ts

@@ -1,11 +1,16 @@
-import type { LabelRecord } from "./store.js";
+/** What the answer-in-use rule needs from a label: the store's, or an app's own row. */
+export type LabelLike = { status: string; updatedAt: Date };
 
 /**
  * The EXPECTED answer tests use: the newest confirmed one, or failing that a
  * provisional one (put to use before review). Pending answers never count.
+ * Generic, so pages that read their own label rows (with extra columns)
+ * share the rule and keep their row type.
  */
-export function confirmedLabel(labels: LabelRecord[]): LabelRecord | null {
-  const newest = (status: LabelRecord["status"]) =>
+export function confirmedLabel<L extends LabelLike>(
+  labels: readonly L[],
+): L | null {
+  const newest = (status: string) =>
     labels
       .filter((l) => l.status === status)
       .sort((a, b) => +b.updatedAt - +a.updatedAt)[0] ?? null;
@@ -13,6 +18,6 @@ export function confirmedLabel(labels: LabelRecord[]): LabelRecord | null {
 }
 
 /** In use on an answer nobody has reviewed yet. */
-export function isProvisional(labels: LabelRecord[]): boolean {
+export function isProvisional(labels: readonly LabelLike[]): boolean {
   return confirmedLabel(labels)?.status === "PROVISIONAL";
 }

package/src/memoryStore.ts

@@ -163,9 +163,13 @@ export function memoryStore(): EvalStore &
         f.updatedAt = now();
       }
     },
-    async listCases(suite) {
+    async listCases(suite, opts) {
       return cases
-        .filter((c) => c.suite === suite && c.status !== "RETIRED")
+        .filter(
+          (c) =>
+            c.suite === suite &&
+            (opts?.includeRetired || c.status !== "RETIRED"),
+        )
         .sort((a, b) => +a.createdAt - +b.createdAt);
     },
     async createCase({ labels = [], ...data }) {

package/src/models.ts

@@ -3,10 +3,27 @@
  * pass a catalog in; Evalkeeper never picks a provider itself. Pure.
  */
 
-/** How hard a reasoning model thinks, where the model takes a setting. */
-export const REASONING_EFFORTS = ["none", "low", "medium", "high"] as const;
-/** A reasoning setting a model may accept. */
-export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
+/**
+ * The reasoning settings providers name, lowest first, so a setting one
+ * model lacks can be placed against the ones it has. A model's own
+ * `efforts` say which it takes.
+ */
+export const REASONING_EFFORTS = [
+  "none",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+  "max",
+] as const;
+/** A setting from REASONING_EFFORTS. */
+export type KnownReasoningEffort = (typeof REASONING_EFFORTS)[number];
+/**
+ * A reasoning setting a model may accept: one of REASONING_EFFORTS, or any
+ * other name a provider uses. The catalog's `efforts` order them per model.
+ */
+export type ReasoningEffort = KnownReasoningEffort | (string & {});
 
 /**
  * What Evalkeeper needs to know about one model: its label, provider, prices, reasoning
@@ -112,8 +129,10 @@ export function priceCall(
 
 /**
  * The effort a model actually gets for a requested one: the same if it
- * takes it; for "none" on a model without it, its lowest; otherwise its own
- * default. Undefined: send nothing.
+ * takes it; one below the lowest it takes (say "none" on a model that
+ * starts at "minimal"), its lowest; one above its highest, its highest;
+ * otherwise (a name it doesn't place) its own default. Undefined: send
+ * nothing.
  */
 export function effortFor(
   models: ModelCatalog,
@@ -123,5 +142,14 @@ export function effortFor(
   const efforts = models[model]?.efforts;
   if (!requested || !efforts) return undefined;
   if (efforts.includes(requested)) return requested;
-  return requested === "none" ? efforts[0] : undefined;
+  const rank = (e: string) =>
+    (REASONING_EFFORTS as readonly string[]).indexOf(e);
+  const want = rank(requested);
+  const lowest = efforts[0];
+  const highest = efforts.at(-1);
+  if (want < 0 || lowest === undefined || highest === undefined)
+    return undefined;
+  if (rank(lowest) >= 0 && want < rank(lowest)) return lowest;
+  if (rank(highest) >= 0 && want > rank(highest)) return highest;
+  return undefined;
 }

package/src/runner.ts

@@ -105,11 +105,18 @@ export type RunnerOptions = {
     models: string[];
     fallbackModels?: string[];
   };
+  /**
+   * Which retiring models (`ModelInfo.retiring`) the board lists.
+   * "always" (the default): every candidate, though a retiring one is never
+   * suggested. "inUse": only one a feature still runs on, as its live or
+   * fallback model, so nobody spends a test on a model that's going away.
+   */
+  retiringOnBoard?: "always" | "inUse";
 };
 
 /**
  * Which model a feature runs on now, its fallback, its reasoning setting,
- * and its temperature.
+ * its temperature, and its output cap.
  */
 export type Route = {
   model: string;
@@ -117,6 +124,8 @@ export type Route = {
   reasoningEffort: ReasoningEffort | null;
   /** Null or unset: no temperature sent. */
   temperature?: number | null;
+  /** The output-token cap; null or unset: the call's own default. */
+  maxOutputTokens?: number | null;
 };
 
 /**
@@ -149,6 +158,8 @@ export function requestHashFor(r: {
   repeat: number;
   /** Left out of the hash when null, so requests without one reuse as before. */
   temperature?: number | null;
+  /** Left out of the hash when null, like temperature. */
+  maxOutputTokens?: number | null;
 }): string {
   return sha(
     stableJson({
@@ -159,6 +170,9 @@ export function requestHashFor(r: {
       input: r.input,
       repeat: r.repeat,
       ...(r.temperature != null ? { temperature: r.temperature } : {}),
+      ...(r.maxOutputTokens != null
+        ? { maxOutputTokens: r.maxOutputTokens }
+        : {}),
     }),
     64,
   );
@@ -267,6 +281,12 @@ export function createRunner(opts: RunnerOptions) {
     return (await opts.route(operation)).temperature ?? null;
   }
 
+  /** The operation's live output cap: the route's, else the call's own. */
+  async function liveCapFor(operation: string): Promise<number | null> {
+    if (!opts.route) return null;
+    return (await opts.route(operation)).maxOutputTokens ?? null;
+  }
+
   /**
    * What a test would cost. Answers an identical request gave in the last
    * 7 days are reused for free, so only the rest are priced: a retest
@@ -285,6 +305,8 @@ export function createRunner(opts: RunnerOptions) {
       reasoningEffort?: ReasoningEffort | null;
       /** The temperature to test; undefined: the operation's live one. */
       temperature?: number | null;
+      /** The output cap to test; undefined: the operation's live one. */
+      maxOutputTokens?: number | null;
     } = {},
   ): Promise<{
     cases: number;
@@ -295,17 +317,21 @@ export function createRunner(opts: RunnerOptions) {
     costUsd: number | null;
   }> {
     const suite = suiteOrThrow(suiteKey);
-    const [cases, perCall, prompt, effort, temperature] = await Promise.all([
-      casesFor(suite.key, set),
-      opts.estimateCallCost?.(suite.operation, model) ?? null,
-      opts.promptVersion(suite.operation),
-      o.reasoningEffort !== undefined
-        ? o.reasoningEffort
-        : liveEffortFor(suite.operation),
-      o.temperature !== undefined
-        ? o.temperature
-        : liveTemperatureFor(suite.operation),
-    ]);
+    const [cases, perCall, prompt, effort, temperature, cap] =
+      await Promise.all([
+        casesFor(suite.key, set),
+        opts.estimateCallCost?.(suite.operation, model) ?? null,
+        opts.promptVersion(suite.operation),
+        o.reasoningEffort !== undefined
+          ? o.reasoningEffort
+          : liveEffortFor(suite.operation),
+        o.temperature !== undefined
+          ? o.temperature
+          : liveTemperatureFor(suite.operation),
+        o.maxOutputTokens !== undefined
+          ? o.maxOutputTokens
+          : liveCapFor(suite.operation),
+      ]);
     const answers = cases.length * repeats;
     const since = new Date(Date.now() - REUSE_DAYS * 86_400_000);
     const reused = o.fresh
@@ -323,6 +349,7 @@ export function createRunner(opts: RunnerOptions) {
                     input: c.input,
                     repeat: i + 1,
                     temperature,
+                    maxOutputTokens: cap,
                   }),
                   since,
                 ),
@@ -350,6 +377,8 @@ export function createRunner(opts: RunnerOptions) {
     reasoningEffort?: ReasoningEffort | null;
     /** A temperature to test; undefined: the operation's live one. */
     temperature?: number | null;
+    /** An output cap to test; undefined: the operation's live one. */
+    maxOutputTokens?: number | null;
     /** For the estimate: pass the same `fresh` to runEval. */
     fresh?: boolean;
   }) {
@@ -364,10 +393,15 @@ export function createRunner(opts: RunnerOptions) {
       args.temperature !== undefined
         ? args.temperature
         : await liveTemperatureFor(suite.operation);
+    const maxOutputTokens =
+      args.maxOutputTokens !== undefined
+        ? args.maxOutputTokens
+        : await liveCapFor(suite.operation);
     const [estimate, promptVersion] = await Promise.all([
       estimateRun(suite.key, args.model, set, repeats, {
         reasoningEffort,
         temperature,
+        maxOutputTokens,
         fresh: Boolean(args.fresh),
       }),
       opts.promptVersion(suite.operation),
@@ -380,6 +414,7 @@ export function createRunner(opts: RunnerOptions) {
       reasoningEffort:
         args.reasoningEffort !== undefined ? args.reasoningEffort : liveEffort,
       temperature,
+      maxOutputTokens,
       set,
       repeats,
       datasetVersion: null,
@@ -516,6 +551,7 @@ export function createRunner(opts: RunnerOptions) {
           input: c.input,
           repeat,
           temperature: run.temperature ?? null,
+          maxOutputTokens: run.maxOutputTokens ?? null,
         });
         const earlier = o.fresh
           ? null
@@ -543,9 +579,9 @@ export function createRunner(opts: RunnerOptions) {
           const output = await withTimeout(
             suite.run(suite.input.parse(c.input), {
               model: run.model,
-              reasoningEffort:
-                (run.reasoningEffort as ReasoningEffort | null) ?? null,
+              reasoningEffort: run.reasoningEffort ?? null,
               temperature: run.temperature ?? null,
+              maxOutputTokens: run.maxOutputTokens ?? null,
               onCost: (usd) => (cost += usd),
             }),
             timeout,
@@ -693,6 +729,7 @@ export function createRunner(opts: RunnerOptions) {
       startedBy: SCHEDULED,
       reasoningEffort: route.reasoningEffort,
       temperature: route.temperature ?? null,
+      maxOutputTokens: route.maxOutputTokens ?? null,
     });
     return run.id;
   }
@@ -801,7 +838,7 @@ export function createRunner(opts: RunnerOptions) {
     // The setting a model actually gets: models without reasoning settings
     // ignore it, so their rows don't split by it.
     const realEffort = (model: string, effort: string | null) =>
-      effortFor(opts.models, model, effort as ReasoningEffort | null) ?? null;
+      effortFor(opts.models, model, effort) ?? null;
     const keyOf = (model: string, effort: string | null) =>
       `${model}|${realEffort(model, effort) ?? ""}`;
     const runKey = (r: RunRecord) => keyOf(r.model, r.reasoningEffort);
@@ -815,11 +852,14 @@ export function createRunner(opts: RunnerOptions) {
         latestDone.set(runKey(r), r);
     }
     const liveTemperature = route.temperature ?? null;
-    // A test at another temperature than the live one answered differently.
+    const liveCap = route.maxOutputTokens ?? null;
+    // A test at another temperature or output cap than the live ones
+    // answered differently.
     const isCurrent = (r: RunRecord) =>
       r.datasetVersion === current &&
       r.promptVersion === prompt &&
-      (r.temperature ?? null) === liveTemperature;
+      (r.temperature ?? null) === liveTemperature &&
+      (r.maxOutputTokens ?? null) === liveCap;
 
     const liveKey = keyOf(route.model, defaultEffort);
     const liveRun = latestDone.get(liveKey);
@@ -829,7 +869,8 @@ export function createRunner(opts: RunnerOptions) {
       liveRun !== undefined &&
       r.datasetVersion === liveRun.datasetVersion &&
       r.promptVersion === liveRun.promptVersion &&
-      (r.temperature ?? null) === (liveRun.temperature ?? null);
+      (r.temperature ?? null) === (liveRun.temperature ?? null) &&
+      (r.maxOutputTokens ?? null) === (liveRun.maxOutputTokens ?? null);
     const outputs = new Map<string, RunOutput[]>();
     for (const r of latestDone.values())
       outputs.set(
@@ -846,7 +887,7 @@ export function createRunner(opts: RunnerOptions) {
     const checkKeys = suite.checks.map((c) => c.key);
 
     const seen = new Set<string>();
-    const keys = [
+    let keys = [
       ...models.map((m) => ({
         model: m,
         effort: defaultEffort,
@@ -865,6 +906,13 @@ export function createRunner(opts: RunnerOptions) {
       seen.add(k);
       return true;
     });
+    // A retiring model stays on the board only while a feature runs on it.
+    const inUse = (model: string) =>
+      model === route.model || model === route.fallbackModel;
+    if (opts.retiringOnBoard === "inUse")
+      keys = keys.filter(
+        ({ model }) => !opts.models[model]?.retiring || inUse(model),
+      );
     const complete = (r: RunRecord | undefined) => !(r?.metrics?.failed ?? 0);
     const estimates = new Map(
       await Promise.all(

package/src/settings.ts

@@ -256,6 +256,7 @@ export function createSettings(opts: {
       fallbackModel: settings.fallbackModel,
       reasoningEffort: settings.reasoningEffort,
       temperature: settings.temperature ?? null,
+      maxOutputTokens: settings.maxOutputTokens,
     };
   }
 

package/src/store.ts

@@ -82,6 +82,11 @@ export type RunRecord = {
    * test at another temperature than the live one shows as out of date.
    */
   temperature?: number | null;
+  /**
+   * The output-token cap tested; null or unset: the call's own default.
+   * Kept like temperature: a test at another cap is out of date.
+   */
+  maxOutputTokens?: number | null;
   set: EvalSet;
   repeats: number;
   /** Changes when the cases or their answers do, so tests compare. */
@@ -139,8 +144,15 @@ export type NewCase = Omit<
  * database, and run describeStoreContract against it.
  */
 export interface EvalStore {
-  /** A suite's cases that aren't retired, with their answers, oldest first. */
-  listCases(suite: string): Promise<CaseRecord[]>;
+  /**
+   * A suite's cases that aren't retired, with their answers, oldest first.
+   * With `includeRetired`, retired ones too (an import checks them, so a
+   * case someone retired isn't added again).
+   */
+  listCases(
+    suite: string,
+    opts?: { includeRetired?: boolean },
+  ): Promise<CaseRecord[]>;
   createCase(data: NewCase): Promise<CaseRecord>;
   /** Adds an answer to a case (answers are never overwritten). */
   addLabel(caseId: string, label: NewLabel): Promise<LabelRecord>;

package/src/testing.ts

@@ -90,6 +90,10 @@ export function describeStoreContract(
       expect((await store.listCases("s")).map((c) => c.id)).toEqual([
         second.id,
       ]);
+      // On request, retired ones too, still oldest first.
+      expect(
+        (await store.listCases("s", { includeRetired: true })).map((c) => c.id),
+      ).toEqual([first.id, second.id]);
     });
 
     it("adds answers without overwriting, and changes their status", async () => {
@@ -135,6 +139,20 @@ export function describeStoreContract(
       expect((await store.getRun(none.id))?.temperature ?? null).toBeNull();
     });
 
+    it("keeps the output cap a run was tested at", async () => {
+      const store = await make();
+      const capped = await store.createRun({
+        ...newRun("s"),
+        maxOutputTokens: 4000,
+      });
+      const none = await store.createRun({
+        ...newRun("s"),
+        maxOutputTokens: null,
+      });
+      expect((await store.getRun(capped.id))?.maxOutputTokens).toBe(4000);
+      expect((await store.getRun(none.id))?.maxOutputTokens ?? null).toBeNull();
+    });
+
     it("updates a run only while it has the status it's guarded by", async () => {
       const store = await make();
       const r = await store.createRun(newRun("s"));
@@ -389,6 +407,21 @@ export function describeSettingsStoreContract(
       expect(await store.listVersions("triage")).toHaveLength(2);
     });
 
+    it("keeps a version's reasoning setting as named, even a newer one", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", {
+        ...create("a", null),
+        create: {
+          settings: { ...settings("a"), reasoningEffort: "xhigh" },
+          promptVersion: "p1",
+        },
+      });
+      expect(v1.reasoningEffort).toBe("xhigh");
+      expect((await store.liveVersion("triage"))?.reasoningEffort).toBe(
+        "xhigh",
+      );
+    });
+
     it("keeps a version's temperature", async () => {
       const store = await make();
       const v1 = await store.switchTo("triage", {