@crowandraven/evalkeeper 0.2.0 → 0.4.0

package/src/models.ts

@@ -3,10 +3,27 @@
  * pass a catalog in; Evalkeeper never picks a provider itself. Pure.
  */
 
-/** How hard a reasoning model thinks, where the model takes a setting. */
-export const REASONING_EFFORTS = ["none", "low", "medium", "high"] as const;
-/** A reasoning setting a model may accept. */
-export type ReasoningEffort = (typeof REASONING_EFFORTS)[number];
+/**
+ * The reasoning settings providers name, lowest first, so a setting one
+ * model lacks can be placed against the ones it has. A model's own
+ * `efforts` say which it takes.
+ */
+export const REASONING_EFFORTS = [
+  "none",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+  "max",
+] as const;
+/** A setting from REASONING_EFFORTS. */
+export type KnownReasoningEffort = (typeof REASONING_EFFORTS)[number];
+/**
+ * A reasoning setting a model may accept: one of REASONING_EFFORTS, or any
+ * other name a provider uses. The catalog's `efforts` order them per model.
+ */
+export type ReasoningEffort = KnownReasoningEffort | (string & {});
 
 /**
  * What Evalkeeper needs to know about one model: its label, provider, prices, reasoning
@@ -112,8 +129,10 @@ export function priceCall(
 
 /**
  * The effort a model actually gets for a requested one: the same if it
- * takes it; for "none" on a model without it, its lowest; otherwise its own
- * default. Undefined: send nothing.
+ * takes it; one below the lowest it takes (say "none" on a model that
+ * starts at "minimal"), its lowest; one above its highest, its highest;
+ * otherwise (a name it doesn't place) its own default. Undefined: send
+ * nothing.
  */
 export function effortFor(
   models: ModelCatalog,
@@ -123,5 +142,14 @@ export function effortFor(
   const efforts = models[model]?.efforts;
   if (!requested || !efforts) return undefined;
   if (efforts.includes(requested)) return requested;
-  return requested === "none" ? efforts[0] : undefined;
+  const rank = (e: string) =>
+    (REASONING_EFFORTS as readonly string[]).indexOf(e);
+  const want = rank(requested);
+  const lowest = efforts[0];
+  const highest = efforts.at(-1);
+  if (want < 0 || lowest === undefined || highest === undefined)
+    return undefined;
+  if (rank(lowest) >= 0 && want < rank(lowest)) return lowest;
+  if (rank(highest) >= 0 && want > rank(highest)) return highest;
+  return undefined;
 }

package/src/runner.ts

@@ -24,6 +24,7 @@ import type {
   EvalStore,
   RunMetrics,
   RunRecord,
+  OutputRecord,
 } from "./store.js";
 import { withTimeout } from "./timeout.js";
 import { SCHEDULED } from "./weekly.js";
@@ -105,11 +106,18 @@ export type RunnerOptions = {
     models: string[];
     fallbackModels?: string[];
   };
+  /**
+   * Which retiring models (`ModelInfo.retiring`) the board lists.
+   * "always" (the default): every candidate, though a retiring one is never
+   * suggested. "inUse": only one a feature still runs on, as its live or
+   * fallback model, so nobody spends a test on a model that's going away.
+   */
+  retiringOnBoard?: "always" | "inUse";
 };
 
 /**
  * Which model a feature runs on now, its fallback, its reasoning setting,
- * and its temperature.
+ * its temperature, and its output cap.
  */
 export type Route = {
   model: string;
@@ -117,6 +125,8 @@ export type Route = {
   reasoningEffort: ReasoningEffort | null;
   /** Null or unset: no temperature sent. */
   temperature?: number | null;
+  /** The output-token cap; null or unset: the call's own default. */
+  maxOutputTokens?: number | null;
 };
 
 /**
@@ -149,6 +159,8 @@ export function requestHashFor(r: {
   repeat: number;
   /** Left out of the hash when null, so requests without one reuse as before. */
   temperature?: number | null;
+  /** Left out of the hash when null, like temperature. */
+  maxOutputTokens?: number | null;
 }): string {
   return sha(
     stableJson({
@@ -159,6 +171,9 @@ export function requestHashFor(r: {
       input: r.input,
       repeat: r.repeat,
       ...(r.temperature != null ? { temperature: r.temperature } : {}),
+      ...(r.maxOutputTokens != null
+        ? { maxOutputTokens: r.maxOutputTokens }
+        : {}),
     }),
     64,
   );
@@ -267,6 +282,12 @@ export function createRunner(opts: RunnerOptions) {
     return (await opts.route(operation)).temperature ?? null;
   }
 
+  /** The operation's live output cap: the route's, else the call's own. */
+  async function liveCapFor(operation: string): Promise<number | null> {
+    if (!opts.route) return null;
+    return (await opts.route(operation)).maxOutputTokens ?? null;
+  }
+
   /**
    * What a test would cost. Answers an identical request gave in the last
    * 7 days are reused for free, so only the rest are priced: a retest
@@ -285,6 +306,8 @@ export function createRunner(opts: RunnerOptions) {
       reasoningEffort?: ReasoningEffort | null;
       /** The temperature to test; undefined: the operation's live one. */
       temperature?: number | null;
+      /** The output cap to test; undefined: the operation's live one. */
+      maxOutputTokens?: number | null;
     } = {},
   ): Promise<{
     cases: number;
@@ -295,41 +318,53 @@ export function createRunner(opts: RunnerOptions) {
     costUsd: number | null;
   }> {
     const suite = suiteOrThrow(suiteKey);
-    const [cases, perCall, prompt, effort, temperature] = await Promise.all([
-      casesFor(suite.key, set),
-      opts.estimateCallCost?.(suite.operation, model) ?? null,
-      opts.promptVersion(suite.operation),
-      o.reasoningEffort !== undefined
-        ? o.reasoningEffort
-        : liveEffortFor(suite.operation),
-      o.temperature !== undefined
-        ? o.temperature
-        : liveTemperatureFor(suite.operation),
-    ]);
+    const [cases, perCall, prompt, effort, temperature, cap] =
+      await Promise.all([
+        casesFor(suite.key, set),
+        opts.estimateCallCost?.(suite.operation, model) ?? null,
+        opts.promptVersion(suite.operation),
+        o.reasoningEffort !== undefined
+          ? o.reasoningEffort
+          : liveEffortFor(suite.operation),
+        o.temperature !== undefined
+          ? o.temperature
+          : liveTemperatureFor(suite.operation),
+        o.maxOutputTokens !== undefined
+          ? o.maxOutputTokens
+          : liveCapFor(suite.operation),
+      ]);
     const answers = cases.length * repeats;
     const since = new Date(Date.now() - REUSE_DAYS * 86_400_000);
-    const reused = o.fresh
-      ? 0
-      : (
-          await Promise.all(
-            cases.flatMap((c) =>
-              Array.from({ length: repeats }, (_, i) =>
-                store.findReusableOutput(
-                  requestHashFor({
-                    suite: suite.key,
-                    model,
-                    effort,
-                    prompt,
-                    input: c.input,
-                    repeat: i + 1,
-                    temperature,
-                  }),
-                  since,
-                ),
+    const hashes = o.fresh
+      ? []
+      : cases.flatMap((c) =>
+          Array.from({ length: repeats }, (_, i) =>
+            requestHashFor({
+              suite: suite.key,
+              model,
+              effort,
+              prompt,
+              input: c.input,
+              repeat: i + 1,
+              temperature,
+              maxOutputTokens: cap,
+            }),
+          ),
+        );
+    // One query where the store can: a lookup per answer is slow on a
+    // remote database.
+    const found = store.findReusableHashes
+      ? await store.findReusableHashes(hashes, since)
+      : new Set(
+          (
+            await Promise.all(
+              hashes.map(async (h) =>
+                (await store.findReusableOutput(h, since)) ? h : null,
               ),
-            ),
-          )
-        ).filter(Boolean).length;
+            )
+          ).filter((h): h is string => h !== null),
+        );
+    const reused = hashes.filter((h) => found.has(h)).length;
     return {
       cases: cases.length,
       answers,
@@ -350,6 +385,8 @@ export function createRunner(opts: RunnerOptions) {
     reasoningEffort?: ReasoningEffort | null;
     /** A temperature to test; undefined: the operation's live one. */
     temperature?: number | null;
+    /** An output cap to test; undefined: the operation's live one. */
+    maxOutputTokens?: number | null;
     /** For the estimate: pass the same `fresh` to runEval. */
     fresh?: boolean;
   }) {
@@ -364,10 +401,15 @@ export function createRunner(opts: RunnerOptions) {
       args.temperature !== undefined
         ? args.temperature
         : await liveTemperatureFor(suite.operation);
+    const maxOutputTokens =
+      args.maxOutputTokens !== undefined
+        ? args.maxOutputTokens
+        : await liveCapFor(suite.operation);
     const [estimate, promptVersion] = await Promise.all([
       estimateRun(suite.key, args.model, set, repeats, {
         reasoningEffort,
         temperature,
+        maxOutputTokens,
         fresh: Boolean(args.fresh),
       }),
       opts.promptVersion(suite.operation),
@@ -380,6 +422,7 @@ export function createRunner(opts: RunnerOptions) {
       reasoningEffort:
         args.reasoningEffort !== undefined ? args.reasoningEffort : liveEffort,
       temperature,
+      maxOutputTokens,
       set,
       repeats,
       datasetVersion: null,
@@ -516,6 +559,7 @@ export function createRunner(opts: RunnerOptions) {
           input: c.input,
           repeat,
           temperature: run.temperature ?? null,
+          maxOutputTokens: run.maxOutputTokens ?? null,
         });
         const earlier = o.fresh
           ? null
@@ -543,9 +587,9 @@ export function createRunner(opts: RunnerOptions) {
           const output = await withTimeout(
             suite.run(suite.input.parse(c.input), {
               model: run.model,
-              reasoningEffort:
-                (run.reasoningEffort as ReasoningEffort | null) ?? null,
+              reasoningEffort: run.reasoningEffort ?? null,
               temperature: run.temperature ?? null,
+              maxOutputTokens: run.maxOutputTokens ?? null,
               onCost: (usd) => (cost += usd),
             }),
             timeout,
@@ -693,6 +737,7 @@ export function createRunner(opts: RunnerOptions) {
       startedBy: SCHEDULED,
       reasoningEffort: route.reasoningEffort,
       temperature: route.temperature ?? null,
+      maxOutputTokens: route.maxOutputTokens ?? null,
     });
     return run.id;
   }
@@ -708,11 +753,30 @@ export function createRunner(opts: RunnerOptions) {
     return id;
   }
 
+  /**
+   * Several tests' outputs by test id: one call where the store can, so a
+   * remote database isn't asked once per test.
+   */
+  async function outputsByRun(
+    runIds: string[],
+  ): Promise<Map<string, OutputRecord[]>> {
+    const byRun = new Map(runIds.map((id) => [id, [] as OutputRecord[]]));
+    if (!runIds.length) return byRun;
+    const rows = store.listOutputsFor
+      ? await store.listOutputsFor(runIds)
+      : (await Promise.all(runIds.map((id) => store.listOutputs(id)))).flat();
+    for (const o of rows) byRun.get(o.runId)?.push(o);
+    return byRun;
+  }
+
   /** Marks tests the server stopped mid-way as failed. */
   async function stopStuckRuns(suiteKey: string, now = Date.now()) {
-    for (const r of await store.listRuns(suiteKey, { status: "RUNNING" })) {
-      if (now - +r.startedAt <= STUCK_MS) continue;
-      const outs = await store.listOutputs(r.id);
+    const old = (await store.listRuns(suiteKey, { status: "RUNNING" })).filter(
+      (r) => now - +r.startedAt > STUCK_MS,
+    );
+    const outputs = await outputsByRun(old.map((r) => r.id));
+    for (const r of old) {
+      const outs = outputs.get(r.id) ?? [];
       const last = outs.reduce<Date | null>(
         (m, x) => (!m || x.createdAt > m ? x.createdAt : m),
         null,
@@ -783,12 +847,12 @@ export function createRunner(opts: RunnerOptions) {
     if (!opts.route || !opts.candidates)
       throw new Error("The board needs the route and candidates options");
     const suite = suiteOrThrow(suiteKey);
-    await stopStuckRuns(suite.key);
-    const [route, cases] = await Promise.all([
+    const [route, cases, prompt] = await Promise.all([
       opts.route(suite.operation),
       casesFor(suite.key, "ALL"),
+      opts.promptVersion(suite.operation),
+      stopStuckRuns(suite.key),
     ]);
-    const prompt = await opts.promptVersion(suite.operation);
     const current = datasetVersion(cases);
     const { models, fallbackModels = models } = opts.candidates(
       suite.operation,
@@ -801,7 +865,7 @@ export function createRunner(opts: RunnerOptions) {
     // The setting a model actually gets: models without reasoning settings
     // ignore it, so their rows don't split by it.
     const realEffort = (model: string, effort: string | null) =>
-      effortFor(opts.models, model, effort as ReasoningEffort | null) ?? null;
+      effortFor(opts.models, model, effort) ?? null;
     const keyOf = (model: string, effort: string | null) =>
       `${model}|${realEffort(model, effort) ?? ""}`;
     const runKey = (r: RunRecord) => keyOf(r.model, r.reasoningEffort);
@@ -815,11 +879,14 @@ export function createRunner(opts: RunnerOptions) {
         latestDone.set(runKey(r), r);
     }
     const liveTemperature = route.temperature ?? null;
-    // A test at another temperature than the live one answered differently.
+    const liveCap = route.maxOutputTokens ?? null;
+    // A test at another temperature or output cap than the live ones
+    // answered differently.
     const isCurrent = (r: RunRecord) =>
       r.datasetVersion === current &&
       r.promptVersion === prompt &&
-      (r.temperature ?? null) === liveTemperature;
+      (r.temperature ?? null) === liveTemperature &&
+      (r.maxOutputTokens ?? null) === liveCap;
 
     const liveKey = keyOf(route.model, defaultEffort);
     const liveRun = latestDone.get(liveKey);
@@ -829,12 +896,15 @@ export function createRunner(opts: RunnerOptions) {
       liveRun !== undefined &&
       r.datasetVersion === liveRun.datasetVersion &&
       r.promptVersion === liveRun.promptVersion &&
-      (r.temperature ?? null) === (liveRun.temperature ?? null);
+      (r.temperature ?? null) === (liveRun.temperature ?? null) &&
+      (r.maxOutputTokens ?? null) === (liveRun.maxOutputTokens ?? null);
     const outputs = new Map<string, RunOutput[]>();
-    for (const r of latestDone.values())
+    for (const [id, outs] of await outputsByRun(
+      [...latestDone.values()].map((r) => r.id),
+    ))
       outputs.set(
-        r.id,
-        (await store.listOutputs(r.id)).map((x) => ({
+        id,
+        outs.map((x) => ({
           caseId: x.caseId,
           checks: x.error ? null : (x.scores ?? []),
           costUsd: x.costUsd,
@@ -846,7 +916,7 @@ export function createRunner(opts: RunnerOptions) {
     const checkKeys = suite.checks.map((c) => c.key);
 
     const seen = new Set<string>();
-    const keys = [
+    let keys = [
       ...models.map((m) => ({
         model: m,
         effort: defaultEffort,
@@ -865,6 +935,13 @@ export function createRunner(opts: RunnerOptions) {
       seen.add(k);
       return true;
     });
+    // A retiring model stays on the board only while a feature runs on it.
+    const inUse = (model: string) =>
+      model === route.model || model === route.fallbackModel;
+    if (opts.retiringOnBoard === "inUse")
+      keys = keys.filter(
+        ({ model }) => !opts.models[model]?.retiring || inUse(model),
+      );
     const complete = (r: RunRecord | undefined) => !(r?.metrics?.failed ?? 0);
     const estimates = new Map(
       await Promise.all(

package/src/settings.ts

@@ -256,6 +256,7 @@ export function createSettings(opts: {
       fallbackModel: settings.fallbackModel,
       reasoningEffort: settings.reasoningEffort,
       temperature: settings.temperature ?? null,
+      maxOutputTokens: settings.maxOutputTokens,
     };
   }
 

package/src/stats.ts

@@ -92,21 +92,34 @@ export function allPassed(checks: CheckResult[] | null): boolean {
   return checks !== null && checks.every((c) => c.pass !== false);
 }
 
-/** Sums each answer's partial credit; null when the suite gives none. */
+/**
+ * Sums each answer's partial credit; null when the suite gives none. A
+ * failed answer (no checks) earns nothing and costs what a graded answer
+ * is worth on average, so a model that fails answers can't score 100%.
+ */
 export function sumUnits(
   outputs: { checks: CheckResult[] | null }[],
 ): { right: number; total: number } | null {
   let right = 0;
   let total = 0;
-  let any = false;
-  for (const o of outputs)
-    for (const c of o.checks ?? [])
+  let graded = 0;
+  let failed = 0;
+  for (const o of outputs) {
+    if (!o.checks) {
+      failed++;
+      continue;
+    }
+    let counted = false;
+    for (const c of o.checks)
       if (c.units) {
-        any = true;
+        counted = true;
         right += c.units.right;
         total += c.units.total;
       }
-  return any ? { right, total } : null;
+    if (counted) graded++;
+  }
+  if (!graded) return null;
+  return { right, total: total + failed * (total / graded) };
 }
 
 /** One answer's score from 0 to 1: its partial credit, else all-or-nothing. */

package/src/store.ts

@@ -82,6 +82,11 @@ export type RunRecord = {
    * test at another temperature than the live one shows as out of date.
    */
   temperature?: number | null;
+  /**
+   * The output-token cap tested; null or unset: the call's own default.
+   * Kept like temperature: a test at another cap is out of date.
+   */
+  maxOutputTokens?: number | null;
   set: EvalSet;
   repeats: number;
   /** Changes when the cases or their answers do, so tests compare. */
@@ -139,8 +144,15 @@ export type NewCase = Omit<
  * database, and run describeStoreContract against it.
  */
 export interface EvalStore {
-  /** A suite's cases that aren't retired, with their answers, oldest first. */
-  listCases(suite: string): Promise<CaseRecord[]>;
+  /**
+   * A suite's cases that aren't retired, with their answers, oldest first.
+   * With `includeRetired`, retired ones too (an import checks them, so a
+   * case someone retired isn't added again).
+   */
+  listCases(
+    suite: string,
+    opts?: { includeRetired?: boolean },
+  ): Promise<CaseRecord[]>;
   createCase(data: NewCase): Promise<CaseRecord>;
   /** Adds an answer to a case (answers are never overwritten). */
   addLabel(caseId: string, label: NewLabel): Promise<LabelRecord>;
@@ -167,6 +179,20 @@ export interface EvalStore {
     requestHash: string,
     since: Date,
   ): Promise<OutputRecord | null>;
+  /**
+   * The outputs of several tests in one call, for the board on a remote
+   * database. Optional: without it the runner calls listOutputs per test.
+   */
+  listOutputsFor?(runIds: string[]): Promise<OutputRecord[]>;
+  /**
+   * Which of these request hashes have an output findReusableOutput would
+   * return, in one call, for estimating a test's cost. Optional: without it
+   * the runner looks each hash up.
+   */
+  findReusableHashes?(
+    requestHashes: string[],
+    since: Date,
+  ): Promise<Set<string>>;
   createOutput(data: NewOutput): Promise<OutputRecord>;
   setOutputScores(id: string, scores: CheckResult[]): Promise<void>;
 }

package/src/testing.ts

@@ -84,12 +84,19 @@ export function describeStoreContract(
     it("lists a suite's cases oldest first, leaving out retired ones", async () => {
       const store = await make();
       const first = await store.createCase(newCase("s"));
+      // Apart in time: two cases made in the same millisecond have no
+      // oldest one.
+      await new Promise((r) => setTimeout(r, 5));
       const second = await store.createCase(newCase("s"));
       await store.createCase(newCase("other"));
       await store.setCaseStatus(first.id, "RETIRED");
       expect((await store.listCases("s")).map((c) => c.id)).toEqual([
         second.id,
       ]);
+      // On request, retired ones too, still oldest first.
+      expect(
+        (await store.listCases("s", { includeRetired: true })).map((c) => c.id),
+      ).toEqual([first.id, second.id]);
     });
 
     it("adds answers without overwriting, and changes their status", async () => {
@@ -135,6 +142,20 @@ export function describeStoreContract(
       expect((await store.getRun(none.id))?.temperature ?? null).toBeNull();
     });
 
+    it("keeps the output cap a run was tested at", async () => {
+      const store = await make();
+      const capped = await store.createRun({
+        ...newRun("s"),
+        maxOutputTokens: 4000,
+      });
+      const none = await store.createRun({
+        ...newRun("s"),
+        maxOutputTokens: null,
+      });
+      expect((await store.getRun(capped.id))?.maxOutputTokens).toBe(4000);
+      expect((await store.getRun(none.id))?.maxOutputTokens ?? null).toBeNull();
+    });
+
     it("updates a run only while it has the status it's guarded by", async () => {
       const store = await make();
       const r = await store.createRun(newRun("s"));
@@ -180,6 +201,61 @@ export function describeStoreContract(
       ).toBeNull();
     });
 
+    it("lists several tests' outputs and reusable hashes in one call, where it can", async () => {
+      const store = await make();
+      const c = await store.createCase(newCase("s"));
+      const [r1, r2, r3] = await Promise.all([
+        store.createRun(newRun("s")),
+        store.createRun(newRun("s")),
+        store.createRun(newRun("s")),
+      ]);
+      const base = {
+        caseId: c.id,
+        repeat: 1,
+        output: { text: "HELLO" },
+        error: null,
+        scores: null,
+        costUsd: 0.01,
+        ms: 12,
+        reused: false,
+      };
+      await store.createOutput({ ...base, runId: r1.id, requestHash: "a" });
+      await store.createOutput({ ...base, runId: r2.id, requestHash: "b" });
+      await store.createOutput({
+        ...base,
+        runId: r2.id,
+        requestHash: "c",
+        reused: true,
+      });
+      await store.createOutput({
+        ...base,
+        runId: r3.id,
+        requestHash: "d",
+        error: "boom",
+        output: null,
+      });
+      if (store.listOutputsFor) {
+        const outs = await store.listOutputsFor([r1.id, r2.id]);
+        expect(outs.map((o) => o.requestHash).sort()).toEqual(["a", "b", "c"]);
+        expect(await store.listOutputsFor([])).toEqual([]);
+      }
+      if (store.findReusableHashes) {
+        expect(
+          [
+            ...(await store.findReusableHashes(
+              ["a", "b", "c", "d", "e"],
+              new Date(0),
+            )),
+          ].sort(),
+        ).toEqual(["a", "b"]);
+        expect(
+          (await store.findReusableHashes(["a"], new Date(Date.now() + 60_000)))
+            .size,
+        ).toBe(0);
+        expect((await store.findReusableHashes([], new Date(0))).size).toBe(0);
+      }
+    });
+
     it("runs a whole test through the runner", async () => {
       const store = await make();
       const echo: Suite<{ text: string }, { text: string }, { want: string }> =
@@ -389,6 +465,21 @@ export function describeSettingsStoreContract(
       expect(await store.listVersions("triage")).toHaveLength(2);
     });
 
+    it("keeps a version's reasoning setting as named, even a newer one", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", {
+        ...create("a", null),
+        create: {
+          settings: { ...settings("a"), reasoningEffort: "xhigh" },
+          promptVersion: "p1",
+        },
+      });
+      expect(v1.reasoningEffort).toBe("xhigh");
+      expect((await store.liveVersion("triage"))?.reasoningEffort).toBe(
+        "xhigh",
+      );
+    });
+
     it("keeps a version's temperature", async () => {
       const store = await make();
       const v1 = await store.switchTo("triage", {

package/src/types.ts

@@ -114,6 +114,11 @@ export type Suite<Input, Output, Label> = {
        * temperature was versioned.
        */
       temperature?: number | null;
+      /**
+       * The output-token cap the test runs at, like temperature. Null: send
+       * none (the call's own default).
+       */
+      maxOutputTokens?: number | null;
       onCost: (usd: number) => void;
     },
   ): Promise<Output>;

package/src/weekly.ts

@@ -104,8 +104,8 @@ export function weeklySummary(args: {
       continue;
     }
     if (r.status !== "SUCCEEDED") continue;
-    // The last check of the same model, temperature, instructions, and
-    // grading, a day or more earlier: anything else changed for a known
+    // The last check of the same model, temperature, output cap,
+    // instructions, and grading, a day or more earlier: anything else changed for a known
     // reason.
     const prior = scheduled.find(
       (p) =>
@@ -113,6 +113,7 @@ export function weeklySummary(args: {
         p.status === "SUCCEEDED" &&
         p.model === r.model &&
         (p.temperature ?? null) === (r.temperature ?? null) &&
+        (p.maxOutputTokens ?? null) === (r.maxOutputTokens ?? null) &&
         p.promptVersion === r.promptVersion &&
         p.graderVersion === r.graderVersion &&
         +r.startedAt - +p.startedAt >= DAY,