@crowandraven/evalkeeper 0.1.1 → 0.3.0

package/src/runner.ts

@@ -9,6 +9,7 @@ import { compareRuns, type RunOutput } from "./compare.js";
 import { isStuck, shouldContinue } from "./legs.js";
 import {
   effortFor,
+  isBaseline,
   type ModelCatalog,
   type ReasoningEffort,
 } from "./models.js";
@@ -104,15 +105,27 @@ export type RunnerOptions = {
     models: string[];
     fallbackModels?: string[];
   };
+  /**
+   * Which retiring models (`ModelInfo.retiring`) the board lists.
+   * "always" (the default): every candidate, though a retiring one is never
+   * suggested. "inUse": only one a feature still runs on, as its live or
+   * fallback model, so nobody spends a test on a model that's going away.
+   */
+  retiringOnBoard?: "always" | "inUse";
 };
 
 /**
- * Which model a feature runs on now, its fallback, and its reasoning setting.
+ * Which model a feature runs on now, its fallback, its reasoning setting,
+ * its temperature, and its output cap.
  */
 export type Route = {
   model: string;
   fallbackModel: string | null;
   reasoningEffort: ReasoningEffort | null;
+  /** Null or unset: no temperature sent. */
+  temperature?: number | null;
+  /** The output-token cap; null or unset: the call's own default. */
+  maxOutputTokens?: number | null;
 };
 
 /**
@@ -143,6 +156,10 @@ export function requestHashFor(r: {
   prompt: string;
   input: unknown;
   repeat: number;
+  /** Left out of the hash when null, so requests without one reuse as before. */
+  temperature?: number | null;
+  /** Left out of the hash when null, like temperature. */
+  maxOutputTokens?: number | null;
 }): string {
   return sha(
     stableJson({
@@ -152,6 +169,10 @@ export function requestHashFor(r: {
       prompt: r.prompt,
       input: r.input,
       repeat: r.repeat,
+      ...(r.temperature != null ? { temperature: r.temperature } : {}),
+      ...(r.maxOutputTokens != null
+        ? { maxOutputTokens: r.maxOutputTokens }
+        : {}),
     }),
     64,
   );
@@ -254,6 +275,18 @@ export function createRunner(opts: RunnerOptions) {
     return null;
   }
 
+  /** The operation's live temperature: the route's, else none. */
+  async function liveTemperatureFor(operation: string): Promise<number | null> {
+    if (!opts.route) return null;
+    return (await opts.route(operation)).temperature ?? null;
+  }
+
+  /** The operation's live output cap: the route's, else the call's own. */
+  async function liveCapFor(operation: string): Promise<number | null> {
+    if (!opts.route) return null;
+    return (await opts.route(operation)).maxOutputTokens ?? null;
+  }
+
   /**
    * What a test would cost. Answers an identical request gave in the last
    * 7 days are reused for free, so only the rest are priced: a retest
@@ -270,6 +303,10 @@ export function createRunner(opts: RunnerOptions) {
       fresh?: boolean;
       /** The setting to test; undefined: the operation's live one. */
       reasoningEffort?: ReasoningEffort | null;
+      /** The temperature to test; undefined: the operation's live one. */
+      temperature?: number | null;
+      /** The output cap to test; undefined: the operation's live one. */
+      maxOutputTokens?: number | null;
     } = {},
   ): Promise<{
     cases: number;
@@ -280,14 +317,21 @@ export function createRunner(opts: RunnerOptions) {
     costUsd: number | null;
   }> {
     const suite = suiteOrThrow(suiteKey);
-    const [cases, perCall, prompt, effort] = await Promise.all([
-      casesFor(suite.key, set),
-      opts.estimateCallCost?.(suite.operation, model) ?? null,
-      opts.promptVersion(suite.operation),
-      o.reasoningEffort !== undefined
-        ? o.reasoningEffort
-        : liveEffortFor(suite.operation),
-    ]);
+    const [cases, perCall, prompt, effort, temperature, cap] =
+      await Promise.all([
+        casesFor(suite.key, set),
+        opts.estimateCallCost?.(suite.operation, model) ?? null,
+        opts.promptVersion(suite.operation),
+        o.reasoningEffort !== undefined
+          ? o.reasoningEffort
+          : liveEffortFor(suite.operation),
+        o.temperature !== undefined
+          ? o.temperature
+          : liveTemperatureFor(suite.operation),
+        o.maxOutputTokens !== undefined
+          ? o.maxOutputTokens
+          : liveCapFor(suite.operation),
+      ]);
     const answers = cases.length * repeats;
     const since = new Date(Date.now() - REUSE_DAYS * 86_400_000);
     const reused = o.fresh
@@ -304,6 +348,8 @@ export function createRunner(opts: RunnerOptions) {
                     prompt,
                     input: c.input,
                     repeat: i + 1,
+                    temperature,
+                    maxOutputTokens: cap,
                   }),
                   since,
                 ),
@@ -329,6 +375,10 @@ export function createRunner(opts: RunnerOptions) {
     startedBy?: string | null;
     /** A reasoning setting to test; undefined: the operation's own. */
     reasoningEffort?: ReasoningEffort | null;
+    /** A temperature to test; undefined: the operation's live one. */
+    temperature?: number | null;
+    /** An output cap to test; undefined: the operation's live one. */
+    maxOutputTokens?: number | null;
     /** For the estimate: pass the same `fresh` to runEval. */
     fresh?: boolean;
   }) {
@@ -339,9 +389,19 @@ export function createRunner(opts: RunnerOptions) {
       args.reasoningEffort !== undefined
         ? args.reasoningEffort
         : await liveEffortFor(suite.operation);
+    const temperature =
+      args.temperature !== undefined
+        ? args.temperature
+        : await liveTemperatureFor(suite.operation);
+    const maxOutputTokens =
+      args.maxOutputTokens !== undefined
+        ? args.maxOutputTokens
+        : await liveCapFor(suite.operation);
     const [estimate, promptVersion] = await Promise.all([
       estimateRun(suite.key, args.model, set, repeats, {
         reasoningEffort,
+        temperature,
+        maxOutputTokens,
         fresh: Boolean(args.fresh),
       }),
       opts.promptVersion(suite.operation),
@@ -353,6 +413,8 @@ export function createRunner(opts: RunnerOptions) {
       promptVersion,
       reasoningEffort:
         args.reasoningEffort !== undefined ? args.reasoningEffort : liveEffort,
+      temperature,
+      maxOutputTokens,
       set,
       repeats,
       datasetVersion: null,
@@ -488,6 +550,8 @@ export function createRunner(opts: RunnerOptions) {
           prompt: run.promptVersion,
           input: c.input,
           repeat,
+          temperature: run.temperature ?? null,
+          maxOutputTokens: run.maxOutputTokens ?? null,
         });
         const earlier = o.fresh
           ? null
@@ -515,8 +579,9 @@ export function createRunner(opts: RunnerOptions) {
           const output = await withTimeout(
             suite.run(suite.input.parse(c.input), {
               model: run.model,
-              reasoningEffort:
-                (run.reasoningEffort as ReasoningEffort | null) ?? null,
+              reasoningEffort: run.reasoningEffort ?? null,
+              temperature: run.temperature ?? null,
+              maxOutputTokens: run.maxOutputTokens ?? null,
               onCost: (usd) => (cost += usd),
             }),
             timeout,
@@ -663,6 +728,8 @@ export function createRunner(opts: RunnerOptions) {
       set: "SMOKE",
       startedBy: SCHEDULED,
       reasoningEffort: route.reasoningEffort,
+      temperature: route.temperature ?? null,
+      maxOutputTokens: route.maxOutputTokens ?? null,
     });
     return run.id;
   }
@@ -771,7 +838,7 @@ export function createRunner(opts: RunnerOptions) {
     // The setting a model actually gets: models without reasoning settings
     // ignore it, so their rows don't split by it.
     const realEffort = (model: string, effort: string | null) =>
-      effortFor(opts.models, model, effort as ReasoningEffort | null) ?? null;
+      effortFor(opts.models, model, effort) ?? null;
     const keyOf = (model: string, effort: string | null) =>
       `${model}|${realEffort(model, effort) ?? ""}`;
     const runKey = (r: RunRecord) => keyOf(r.model, r.reasoningEffort);
@@ -784,8 +851,15 @@ export function createRunner(opts: RunnerOptions) {
       if (r.status === "SUCCEEDED" && !latestDone.has(runKey(r)))
         latestDone.set(runKey(r), r);
     }
+    const liveTemperature = route.temperature ?? null;
+    const liveCap = route.maxOutputTokens ?? null;
+    // A test at another temperature or output cap than the live ones
+    // answered differently.
     const isCurrent = (r: RunRecord) =>
-      r.datasetVersion === current && r.promptVersion === prompt;
+      r.datasetVersion === current &&
+      r.promptVersion === prompt &&
+      (r.temperature ?? null) === liveTemperature &&
+      (r.maxOutputTokens ?? null) === liveCap;
 
     const liveKey = keyOf(route.model, defaultEffort);
     const liveRun = latestDone.get(liveKey);
@@ -794,7 +868,9 @@ export function createRunner(opts: RunnerOptions) {
     const sameAsLive = (r: RunRecord) =>
       liveRun !== undefined &&
       r.datasetVersion === liveRun.datasetVersion &&
-      r.promptVersion === liveRun.promptVersion;
+      r.promptVersion === liveRun.promptVersion &&
+      (r.temperature ?? null) === (liveRun.temperature ?? null) &&
+      (r.maxOutputTokens ?? null) === (liveRun.maxOutputTokens ?? null);
     const outputs = new Map<string, RunOutput[]>();
     for (const r of latestDone.values())
       outputs.set(
@@ -811,7 +887,7 @@ export function createRunner(opts: RunnerOptions) {
     const checkKeys = suite.checks.map((c) => c.key);
 
     const seen = new Set<string>();
-    const keys = [
+    let keys = [
       ...models.map((m) => ({
         model: m,
         effort: defaultEffort,
@@ -830,6 +906,13 @@ export function createRunner(opts: RunnerOptions) {
       seen.add(k);
       return true;
     });
+    // A retiring model stays on the board only while a feature runs on it.
+    const inUse = (model: string) =>
+      model === route.model || model === route.fallbackModel;
+    if (opts.retiringOnBoard === "inUse")
+      keys = keys.filter(
+        ({ model }) => !opts.models[model]?.retiring || inUse(model),
+      );
     const complete = (r: RunRecord | undefined) => !(r?.metrics?.failed ?? 0);
     const estimates = new Map(
       await Promise.all(
@@ -863,6 +946,7 @@ export function createRunner(opts: RunnerOptions) {
               .verdict
           : null;
       const label = opts.models[model]?.label ?? model;
+      const baseline = isBaseline(opts.models, model);
       return {
         key,
         model,
@@ -873,6 +957,7 @@ export function createRunner(opts: RunnerOptions) {
           : label,
         live: key === liveKey,
         fallback: !variant && model === route.fallbackModel,
+        baseline,
         testing: testing.has(key),
         failed:
           lastRun?.status === "FAILED"
@@ -900,6 +985,7 @@ export function createRunner(opts: RunnerOptions) {
         vsLive,
         // Only on up-to-date tests: never promote on an old answer key.
         canMakeLive:
+          !baseline &&
           models.includes(model) &&
           fresh &&
           complete(r) &&
@@ -907,6 +993,7 @@ export function createRunner(opts: RunnerOptions) {
         liveOnly: !fallbackModels.includes(model),
         fallbackOnly: !models.includes(model),
         canMakeFallback:
+          !baseline &&
           fallbackModels.includes(model) &&
           !variant &&
           model !== route.model &&

package/src/settings.ts

@@ -12,7 +12,8 @@ import type { Route } from "./runner.js";
  */
 
 /**
- * A feature's model settings: live model, fallback, reasoning setting, and output limit.
+ * A feature's model settings: live model, fallback, reasoning setting,
+ * output limit, and temperature.
  */
 export type Settings = {
   model: string;
@@ -21,6 +22,12 @@ export type Settings = {
   reasoningEffort: ReasoningEffort | null;
   /** Null: the call's own default. */
   maxOutputTokens: number | null;
+  /**
+   * The sampling temperature the feature sends. Null or unset: none sent,
+   * so the model's own default. Tests record it, and a test at another
+   * temperature than the live one is out of date.
+   */
+  temperature?: number | null;
 };
 
 /**
@@ -106,13 +113,31 @@ export class SettingsConflict extends Error {
   }
 }
 
+/**
+ * A feature has no settings: no version yet, and no default in code. Apps
+ * whose models are chosen in an admin page, not in code, see this until
+ * the feature's first version is recorded (`seed` or `change`).
+ */
+export class NoSettingsError extends Error {
+  /** The feature with no settings. */
+  readonly operation: string;
+  constructor(operation: string) {
+    super(
+      `${operation} has no settings yet: no version, and no default in code.`,
+    );
+    this.name = "NoSettingsError";
+    this.operation = operation;
+  }
+}
+
 /** Whether two settings are the same in every field. */
 export function sameSettings(a: Settings, b: Settings): boolean {
   return (
     a.model === b.model &&
     a.fallbackModel === b.fallbackModel &&
     a.reasoningEffort === b.reasoningEffort &&
-    a.maxOutputTokens === b.maxOutputTokens
+    a.maxOutputTokens === b.maxOutputTokens &&
+    (a.temperature ?? null) === (b.temperature ?? null)
   );
 }
 
@@ -123,6 +148,7 @@ export function settingsOf(v: Settings): Settings {
     fallbackModel: v.fallbackModel,
     reasoningEffort: v.reasoningEffort,
     maxOutputTokens: v.maxOutputTokens,
+    temperature: v.temperature ?? null,
   };
 }
 
@@ -148,6 +174,10 @@ export function describeChange(
     out.push(
       `Output limit: ${from.maxOutputTokens ?? "default"} → ${to.maxOutputTokens ?? "default"}`,
     );
+  if ((from.temperature ?? null) !== (to.temperature ?? null))
+    out.push(
+      `Temperature: ${from.temperature ?? "default"} → ${to.temperature ?? "default"}`,
+    );
   return out;
 }
 
@@ -175,11 +205,28 @@ export type HistoryEntry = SettingsChange & {
  * });
  * await settings.rollBack("ticket_triage", versionId, { by: admin.name, reason: "Worse in production" });
  * ```
+ *
+ * When models are chosen in an admin page rather than in code, return null
+ * from `defaults` and record each feature's existing config as version 1
+ * with `seed`:
+ *
+ * ```ts
+ * const settings = createSettings({ store, defaults: () => null });
+ * for (const row of await db.aiConfig.findMany())
+ *   await settings.seed(row.operation, settingsFromRow(row), {
+ *     by: null,
+ *     reason: "Imported from the admin config",
+ *   });
+ * ```
  */
 export function createSettings(opts: {
   store: SettingsStore;
-  /** Each feature's settings in code, used until it's first changed. */
-  defaults: (operation: string) => Settings;
+  /**
+   * Each feature's settings in code, used until it's first changed. Null:
+   * the feature has no default, and reading its settings throws
+   * `NoSettingsError` until a version is recorded.
+   */
+  defaults: (operation: string) => Settings | null;
   /** The instructions' current hash, so a version records what it ran. */
   promptVersion?: (operation: string) => string | Promise<string>;
 }) {
@@ -187,32 +234,38 @@ export function createSettings(opts: {
   const prompt = async (op: string) =>
     opts.promptVersion ? await opts.promptVersion(op) : null;
 
-  /** The live settings: the live version's, else the default from code. */
+  /**
+   * The live settings: the live version's, else the default from code.
+   * Throws `NoSettingsError` when the feature has neither.
+   */
   async function current(
     operation: string,
   ): Promise<{ settings: Settings; version: SettingsVersion | null }> {
     const version = await store.liveVersion(operation);
-    return {
-      settings: version ? settingsOf(version) : opts.defaults(operation),
-      version,
-    };
+    if (version) return { settings: settingsOf(version), version };
+    const fallback = opts.defaults(operation);
+    if (!fallback) throw new NoSettingsError(operation);
+    return { settings: fallback, version };
   }
 
-  /** The runner's route for a feature, from its live settings. */
+  /** The runner's route for a feature, from its live settings; throws `NoSettingsError` when it has none. */
   async function route(operation: string): Promise<Route> {
     const { settings } = await current(operation);
     return {
       model: settings.model,
       fallbackModel: settings.fallbackModel,
       reasoningEffort: settings.reasoningEffort,
+      temperature: settings.temperature ?? null,
+      maxOutputTokens: settings.maxOutputTokens,
     };
   }
 
   /**
    * Makes new settings live as a new version. The first change to a
    * feature first records its default as version 1, so there's always
-   * something to roll back to. With `evidence`, it's a promotion. Returns
-   * null when nothing changed.
+   * something to roll back to; a feature with no default gets `next` as
+   * version 1. With `evidence`, it's a promotion. Returns null when
+   * nothing changed.
    */
   async function change(
     operation: string,
@@ -223,6 +276,15 @@ export function createSettings(opts: {
     let live = await store.liveVersion(operation);
     if (!live) {
       const initial = opts.defaults(operation);
+      if (!initial)
+        return store.switchTo(operation, {
+          create: { settings: settingsOf(next), promptVersion },
+          kind: by.evidence === undefined ? "CHANGE" : "PROMOTE",
+          reason: by.reason,
+          by: by.by,
+          evidence: by.evidence,
+          expectLive: null,
+        });
       if (sameSettings(initial, next)) return null;
       live = await store.switchTo(operation, {
         create: { settings: initial, promptVersion },
@@ -242,7 +304,11 @@ export function createSettings(opts: {
     });
   }
 
-  /** Changes only some settings (e.g. the fallback), keeping the rest. */
+  /**
+   * Changes only some settings (e.g. the fallback), keeping the rest.
+   * Throws `NoSettingsError` for a feature with no settings to keep: use
+   * `change` or `seed` with full settings first.
+   */
   async function update(
     operation: string,
     patch: Partial<Settings>,
@@ -301,6 +367,42 @@ export function createSettings(opts: {
     }
   }
 
+  /**
+   * Records existing settings as a feature's version 1, for an app moving
+   * its model choices into versioned settings (an import from its own
+   * config table). Only when the feature has no version yet: otherwise it
+   * changes nothing and returns null, so an import can run again safely.
+   * `promptVersion` defaults to the instructions' current hash.
+   */
+  async function seed(
+    operation: string,
+    initial: Settings,
+    by: {
+      by: string | null;
+      reason: string | null;
+      promptVersion?: string | null;
+    },
+  ): Promise<SettingsVersion | null> {
+    if (await store.liveVersion(operation)) return null;
+    const promptVersion =
+      by.promptVersion !== undefined
+        ? by.promptVersion
+        : await prompt(operation);
+    try {
+      return await store.switchTo(operation, {
+        create: { settings: settingsOf(initial), promptVersion },
+        kind: "CHANGE",
+        reason: by.reason,
+        by: by.by,
+        expectLive: null,
+      });
+    } catch (e) {
+      // Another request seeded it first: that version stands.
+      if (e instanceof SettingsConflict) return null;
+      throw e;
+    }
+  }
+
   /** Every switch, newest first, with the versions it went from and to. */
   async function history(
     operation: string,
@@ -324,7 +426,16 @@ export function createSettings(opts: {
     });
   }
 
-  return { current, route, change, update, rollBack, syncPrompt, history };
+  return {
+    current,
+    route,
+    change,
+    update,
+    rollBack,
+    syncPrompt,
+    seed,
+    history,
+  };
 }
 
 /** What createSettings returns. */

package/src/store.ts

@@ -77,6 +77,16 @@ export type RunRecord = {
   promptVersion: string;
   /** The reasoning setting tested; null: the model's own default. */
   reasoningEffort: string | null;
+  /**
+   * The temperature tested; null or unset: none sent. Stores keep it, so a
+   * test at another temperature than the live one shows as out of date.
+   */
+  temperature?: number | null;
+  /**
+   * The output-token cap tested; null or unset: the call's own default.
+   * Kept like temperature: a test at another cap is out of date.
+   */
+  maxOutputTokens?: number | null;
   set: EvalSet;
   repeats: number;
   /** Changes when the cases or their answers do, so tests compare. */
@@ -134,8 +144,15 @@ export type NewCase = Omit<
  * database, and run describeStoreContract against it.
  */
 export interface EvalStore {
-  /** A suite's cases that aren't retired, with their answers, oldest first. */
-  listCases(suite: string): Promise<CaseRecord[]>;
+  /**
+   * A suite's cases that aren't retired, with their answers, oldest first.
+   * With `includeRetired`, retired ones too (an import checks them, so a
+   * case someone retired isn't added again).
+   */
+  listCases(
+    suite: string,
+    opts?: { includeRetired?: boolean },
+  ): Promise<CaseRecord[]>;
   createCase(data: NewCase): Promise<CaseRecord>;
   /** Adds an answer to a case (answers are never overwritten). */
   addLabel(caseId: string, label: NewLabel): Promise<LabelRecord>;

package/src/suggest.ts

@@ -1,4 +1,4 @@
-import { retiresLabel, type ModelCatalog } from "./models.js";
+import { isBaseline, retiresLabel, type ModelCatalog } from "./models.js";
 import type { Rate } from "./stats.js";
 import type { CheckDef, SuitePriority } from "./types.js";
 import type { Verdict } from "./compare.js";
@@ -109,8 +109,10 @@ export function suggest<R extends SuggestRow>(
   const retiring = (r: SuggestRow) => retiresLabel(models, r.model);
   const limitMs = priority.maxSeconds ? priority.maxSeconds * 1000 : Infinity;
   const tooSlow = (r: SuggestRow) => ms(r) !== Infinity && ms(r) > limitMs;
+  // Baselines are code, compared on the board but never suggested.
   const tested = rows.filter(
     (r) =>
+      !isBaseline(models, r.model) &&
       r.test &&
       !r.test.stale &&
       !r.test.failedAnswers &&

package/src/testing.ts

@@ -90,6 +90,10 @@ export function describeStoreContract(
       expect((await store.listCases("s")).map((c) => c.id)).toEqual([
         second.id,
       ]);
+      // On request, retired ones too, still oldest first.
+      expect(
+        (await store.listCases("s", { includeRetired: true })).map((c) => c.id),
+      ).toEqual([first.id, second.id]);
     });
 
     it("adds answers without overwriting, and changes their status", async () => {
@@ -127,6 +131,28 @@ export function describeStoreContract(
       expect(await store.getRun("missing")).toBeNull();
     });
 
+    it("keeps the temperature a run was tested at", async () => {
+      const store = await make();
+      const warm = await store.createRun({ ...newRun("s"), temperature: 0.7 });
+      const none = await store.createRun({ ...newRun("s"), temperature: null });
+      expect((await store.getRun(warm.id))?.temperature).toBe(0.7);
+      expect((await store.getRun(none.id))?.temperature ?? null).toBeNull();
+    });
+
+    it("keeps the output cap a run was tested at", async () => {
+      const store = await make();
+      const capped = await store.createRun({
+        ...newRun("s"),
+        maxOutputTokens: 4000,
+      });
+      const none = await store.createRun({
+        ...newRun("s"),
+        maxOutputTokens: null,
+      });
+      expect((await store.getRun(capped.id))?.maxOutputTokens).toBe(4000);
+      expect((await store.getRun(none.id))?.maxOutputTokens ?? null).toBeNull();
+    });
+
     it("updates a run only while it has the status it's guarded by", async () => {
       const store = await make();
       const r = await store.createRun(newRun("s"));
@@ -381,6 +407,36 @@ export function describeSettingsStoreContract(
       expect(await store.listVersions("triage")).toHaveLength(2);
     });
 
+    it("keeps a version's reasoning setting as named, even a newer one", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", {
+        ...create("a", null),
+        create: {
+          settings: { ...settings("a"), reasoningEffort: "xhigh" },
+          promptVersion: "p1",
+        },
+      });
+      expect(v1.reasoningEffort).toBe("xhigh");
+      expect((await store.liveVersion("triage"))?.reasoningEffort).toBe(
+        "xhigh",
+      );
+    });
+
+    it("keeps a version's temperature", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", {
+        ...create("a", null),
+        create: {
+          settings: { ...settings("a"), temperature: 0.4 },
+          promptVersion: "p1",
+        },
+      });
+      expect(v1.temperature).toBe(0.4);
+      expect((await store.liveVersion("triage"))?.temperature).toBe(0.4);
+      const v2 = await store.switchTo("triage", create("b", v1.id));
+      expect(v2.temperature ?? null).toBeNull();
+    });
+
     it("keeps each feature's versions and live one apart", async () => {
       const store = await make();
       await store.switchTo("triage", create("a", null));

package/src/types.ts

@@ -108,6 +108,17 @@ export type Suite<Input, Output, Label> = {
       model: string;
       /** A reasoning setting to test; undefined: the operation's own. */
       reasoningEffort?: ReasoningEffort | null;
+      /**
+       * The temperature the test runs at: the operation's live one when the
+       * test was started. Null: send none. Undefined from runners before
+       * temperature was versioned.
+       */
+      temperature?: number | null;
+      /**
+       * The output-token cap the test runs at, like temperature. Null: send
+       * none (the call's own default).
+       */
+      maxOutputTokens?: number | null;
       onCost: (usd: number) => void;
     },
   ): Promise<Output>;

package/src/weekly.ts

@@ -104,13 +104,16 @@ export function weeklySummary(args: {
       continue;
     }
     if (r.status !== "SUCCEEDED") continue;
-    // The last check of the same model, instructions, and grading, a day or
-    // more earlier: anything else changed for a known reason.
+    // The last check of the same model, temperature, output cap,
+    // instructions, and grading, a day or more earlier: anything else changed for a known
+    // reason.
     const prior = scheduled.find(
       (p) =>
         p.suite === r.suite &&
         p.status === "SUCCEEDED" &&
         p.model === r.model &&
+        (p.temperature ?? null) === (r.temperature ?? null) &&
+        (p.maxOutputTokens ?? null) === (r.maxOutputTokens ?? null) &&
         p.promptVersion === r.promptVersion &&
         p.graderVersion === r.graderVersion &&
         +r.startedAt - +p.startedAt >= DAY,