@crowandraven/evalkeeper 0.1.1 → 0.2.0

package/src/runner.ts

@@ -9,6 +9,7 @@ import { compareRuns, type RunOutput } from "./compare.js";
 import { isStuck, shouldContinue } from "./legs.js";
 import {
   effortFor,
+  isBaseline,
   type ModelCatalog,
   type ReasoningEffort,
 } from "./models.js";
@@ -107,12 +108,15 @@ export type RunnerOptions = {
 };
 
 /**
- * Which model a feature runs on now, its fallback, and its reasoning setting.
+ * Which model a feature runs on now, its fallback, its reasoning setting,
+ * and its temperature.
  */
 export type Route = {
   model: string;
   fallbackModel: string | null;
   reasoningEffort: ReasoningEffort | null;
+  /** Null or unset: no temperature sent. */
+  temperature?: number | null;
 };
 
 /**
@@ -143,6 +147,8 @@ export function requestHashFor(r: {
   prompt: string;
   input: unknown;
   repeat: number;
+  /** Left out of the hash when null, so requests without one reuse as before. */
+  temperature?: number | null;
 }): string {
   return sha(
     stableJson({
@@ -152,6 +158,7 @@ export function requestHashFor(r: {
       prompt: r.prompt,
       input: r.input,
       repeat: r.repeat,
+      ...(r.temperature != null ? { temperature: r.temperature } : {}),
     }),
     64,
   );
@@ -254,6 +261,12 @@ export function createRunner(opts: RunnerOptions) {
     return null;
   }
 
+  /** The operation's live temperature: the route's, else none. */
+  async function liveTemperatureFor(operation: string): Promise<number | null> {
+    if (!opts.route) return null;
+    return (await opts.route(operation)).temperature ?? null;
+  }
+
   /**
    * What a test would cost. Answers an identical request gave in the last
    * 7 days are reused for free, so only the rest are priced: a retest
@@ -270,6 +283,8 @@ export function createRunner(opts: RunnerOptions) {
       fresh?: boolean;
       /** The setting to test; undefined: the operation's live one. */
       reasoningEffort?: ReasoningEffort | null;
+      /** The temperature to test; undefined: the operation's live one. */
+      temperature?: number | null;
     } = {},
   ): Promise<{
     cases: number;
@@ -280,13 +295,16 @@ export function createRunner(opts: RunnerOptions) {
     costUsd: number | null;
   }> {
     const suite = suiteOrThrow(suiteKey);
-    const [cases, perCall, prompt, effort] = await Promise.all([
+    const [cases, perCall, prompt, effort, temperature] = await Promise.all([
       casesFor(suite.key, set),
       opts.estimateCallCost?.(suite.operation, model) ?? null,
       opts.promptVersion(suite.operation),
       o.reasoningEffort !== undefined
         ? o.reasoningEffort
         : liveEffortFor(suite.operation),
+      o.temperature !== undefined
+        ? o.temperature
+        : liveTemperatureFor(suite.operation),
     ]);
     const answers = cases.length * repeats;
     const since = new Date(Date.now() - REUSE_DAYS * 86_400_000);
@@ -304,6 +322,7 @@ export function createRunner(opts: RunnerOptions) {
                     prompt,
                     input: c.input,
                     repeat: i + 1,
+                    temperature,
                   }),
                   since,
                 ),
@@ -329,6 +348,8 @@ export function createRunner(opts: RunnerOptions) {
     startedBy?: string | null;
     /** A reasoning setting to test; undefined: the operation's own. */
     reasoningEffort?: ReasoningEffort | null;
+    /** A temperature to test; undefined: the operation's live one. */
+    temperature?: number | null;
     /** For the estimate: pass the same `fresh` to runEval. */
     fresh?: boolean;
   }) {
@@ -339,9 +360,14 @@ export function createRunner(opts: RunnerOptions) {
       args.reasoningEffort !== undefined
         ? args.reasoningEffort
         : await liveEffortFor(suite.operation);
+    const temperature =
+      args.temperature !== undefined
+        ? args.temperature
+        : await liveTemperatureFor(suite.operation);
     const [estimate, promptVersion] = await Promise.all([
       estimateRun(suite.key, args.model, set, repeats, {
         reasoningEffort,
+        temperature,
         fresh: Boolean(args.fresh),
       }),
       opts.promptVersion(suite.operation),
@@ -353,6 +379,7 @@ export function createRunner(opts: RunnerOptions) {
       promptVersion,
       reasoningEffort:
         args.reasoningEffort !== undefined ? args.reasoningEffort : liveEffort,
+      temperature,
       set,
       repeats,
       datasetVersion: null,
@@ -488,6 +515,7 @@ export function createRunner(opts: RunnerOptions) {
           prompt: run.promptVersion,
           input: c.input,
           repeat,
+          temperature: run.temperature ?? null,
         });
         const earlier = o.fresh
           ? null
@@ -517,6 +545,7 @@ export function createRunner(opts: RunnerOptions) {
               model: run.model,
               reasoningEffort:
                 (run.reasoningEffort as ReasoningEffort | null) ?? null,
+              temperature: run.temperature ?? null,
               onCost: (usd) => (cost += usd),
             }),
             timeout,
@@ -663,6 +692,7 @@ export function createRunner(opts: RunnerOptions) {
       set: "SMOKE",
       startedBy: SCHEDULED,
       reasoningEffort: route.reasoningEffort,
+      temperature: route.temperature ?? null,
     });
     return run.id;
   }
@@ -784,8 +814,12 @@ export function createRunner(opts: RunnerOptions) {
       if (r.status === "SUCCEEDED" && !latestDone.has(runKey(r)))
         latestDone.set(runKey(r), r);
     }
+    const liveTemperature = route.temperature ?? null;
+    // A test at another temperature than the live one answered differently.
     const isCurrent = (r: RunRecord) =>
-      r.datasetVersion === current && r.promptVersion === prompt;
+      r.datasetVersion === current &&
+      r.promptVersion === prompt &&
+      (r.temperature ?? null) === liveTemperature;
 
     const liveKey = keyOf(route.model, defaultEffort);
     const liveRun = latestDone.get(liveKey);
@@ -794,7 +828,8 @@ export function createRunner(opts: RunnerOptions) {
     const sameAsLive = (r: RunRecord) =>
       liveRun !== undefined &&
       r.datasetVersion === liveRun.datasetVersion &&
-      r.promptVersion === liveRun.promptVersion;
+      r.promptVersion === liveRun.promptVersion &&
+      (r.temperature ?? null) === (liveRun.temperature ?? null);
     const outputs = new Map<string, RunOutput[]>();
     for (const r of latestDone.values())
       outputs.set(
@@ -863,6 +898,7 @@ export function createRunner(opts: RunnerOptions) {
               .verdict
           : null;
       const label = opts.models[model]?.label ?? model;
+      const baseline = isBaseline(opts.models, model);
       return {
         key,
         model,
@@ -873,6 +909,7 @@ export function createRunner(opts: RunnerOptions) {
           : label,
         live: key === liveKey,
         fallback: !variant && model === route.fallbackModel,
+        baseline,
         testing: testing.has(key),
         failed:
           lastRun?.status === "FAILED"
@@ -900,6 +937,7 @@ export function createRunner(opts: RunnerOptions) {
         vsLive,
         // Only on up-to-date tests: never promote on an old answer key.
         canMakeLive:
+          !baseline &&
           models.includes(model) &&
           fresh &&
           complete(r) &&
@@ -907,6 +945,7 @@ export function createRunner(opts: RunnerOptions) {
         liveOnly: !fallbackModels.includes(model),
         fallbackOnly: !models.includes(model),
         canMakeFallback:
+          !baseline &&
           fallbackModels.includes(model) &&
           !variant &&
           model !== route.model &&

package/src/settings.ts

@@ -12,7 +12,8 @@ import type { Route } from "./runner.js";
  */
 
 /**
- * A feature's model settings: live model, fallback, reasoning setting, and output limit.
+ * A feature's model settings: live model, fallback, reasoning setting,
+ * output limit, and temperature.
  */
 export type Settings = {
   model: string;
@@ -21,6 +22,12 @@ export type Settings = {
   reasoningEffort: ReasoningEffort | null;
   /** Null: the call's own default. */
   maxOutputTokens: number | null;
+  /**
+   * The sampling temperature the feature sends. Null or unset: none sent,
+   * so the model's own default. Tests record it, and a test at another
+   * temperature than the live one is out of date.
+   */
+  temperature?: number | null;
 };
 
 /**
@@ -106,13 +113,31 @@ export class SettingsConflict extends Error {
   }
 }
 
+/**
+ * A feature has no settings: no version yet, and no default in code. Apps
+ * whose models are chosen in an admin page, not in code, see this until
+ * the feature's first version is recorded (`seed` or `change`).
+ */
+export class NoSettingsError extends Error {
+  /** The feature with no settings. */
+  readonly operation: string;
+  constructor(operation: string) {
+    super(
+      `${operation} has no settings yet: no version, and no default in code.`,
+    );
+    this.name = "NoSettingsError";
+    this.operation = operation;
+  }
+}
+
 /** Whether two settings are the same in every field. */
 export function sameSettings(a: Settings, b: Settings): boolean {
   return (
     a.model === b.model &&
     a.fallbackModel === b.fallbackModel &&
     a.reasoningEffort === b.reasoningEffort &&
-    a.maxOutputTokens === b.maxOutputTokens
+    a.maxOutputTokens === b.maxOutputTokens &&
+    (a.temperature ?? null) === (b.temperature ?? null)
   );
 }
 
@@ -123,6 +148,7 @@ export function settingsOf(v: Settings): Settings {
     fallbackModel: v.fallbackModel,
     reasoningEffort: v.reasoningEffort,
     maxOutputTokens: v.maxOutputTokens,
+    temperature: v.temperature ?? null,
   };
 }
 
@@ -148,6 +174,10 @@ export function describeChange(
     out.push(
       `Output limit: ${from.maxOutputTokens ?? "default"} → ${to.maxOutputTokens ?? "default"}`,
     );
+  if ((from.temperature ?? null) !== (to.temperature ?? null))
+    out.push(
+      `Temperature: ${from.temperature ?? "default"} → ${to.temperature ?? "default"}`,
+    );
   return out;
 }
 
@@ -175,11 +205,28 @@ export type HistoryEntry = SettingsChange & {
  * });
  * await settings.rollBack("ticket_triage", versionId, { by: admin.name, reason: "Worse in production" });
  * ```
+ *
+ * When models are chosen in an admin page rather than in code, return null
+ * from `defaults` and record each feature's existing config as version 1
+ * with `seed`:
+ *
+ * ```ts
+ * const settings = createSettings({ store, defaults: () => null });
+ * for (const row of await db.aiConfig.findMany())
+ *   await settings.seed(row.operation, settingsFromRow(row), {
+ *     by: null,
+ *     reason: "Imported from the admin config",
+ *   });
+ * ```
  */
 export function createSettings(opts: {
   store: SettingsStore;
-  /** Each feature's settings in code, used until it's first changed. */
-  defaults: (operation: string) => Settings;
+  /**
+   * Each feature's settings in code, used until it's first changed. Null:
+   * the feature has no default, and reading its settings throws
+   * `NoSettingsError` until a version is recorded.
+   */
+  defaults: (operation: string) => Settings | null;
   /** The instructions' current hash, so a version records what it ran. */
   promptVersion?: (operation: string) => string | Promise<string>;
 }) {
@@ -187,32 +234,37 @@ export function createSettings(opts: {
   const prompt = async (op: string) =>
     opts.promptVersion ? await opts.promptVersion(op) : null;
 
-  /** The live settings: the live version's, else the default from code. */
+  /**
+   * The live settings: the live version's, else the default from code.
+   * Throws `NoSettingsError` when the feature has neither.
+   */
   async function current(
     operation: string,
   ): Promise<{ settings: Settings; version: SettingsVersion | null }> {
     const version = await store.liveVersion(operation);
-    return {
-      settings: version ? settingsOf(version) : opts.defaults(operation),
-      version,
-    };
+    if (version) return { settings: settingsOf(version), version };
+    const fallback = opts.defaults(operation);
+    if (!fallback) throw new NoSettingsError(operation);
+    return { settings: fallback, version };
   }
 
-  /** The runner's route for a feature, from its live settings. */
+  /** The runner's route for a feature, from its live settings; throws `NoSettingsError` when it has none. */
   async function route(operation: string): Promise<Route> {
     const { settings } = await current(operation);
     return {
       model: settings.model,
       fallbackModel: settings.fallbackModel,
       reasoningEffort: settings.reasoningEffort,
+      temperature: settings.temperature ?? null,
     };
   }
 
   /**
    * Makes new settings live as a new version. The first change to a
    * feature first records its default as version 1, so there's always
-   * something to roll back to. With `evidence`, it's a promotion. Returns
-   * null when nothing changed.
+   * something to roll back to; a feature with no default gets `next` as
+   * version 1. With `evidence`, it's a promotion. Returns null when
+   * nothing changed.
    */
   async function change(
     operation: string,
@@ -223,6 +275,15 @@ export function createSettings(opts: {
     let live = await store.liveVersion(operation);
     if (!live) {
       const initial = opts.defaults(operation);
+      if (!initial)
+        return store.switchTo(operation, {
+          create: { settings: settingsOf(next), promptVersion },
+          kind: by.evidence === undefined ? "CHANGE" : "PROMOTE",
+          reason: by.reason,
+          by: by.by,
+          evidence: by.evidence,
+          expectLive: null,
+        });
       if (sameSettings(initial, next)) return null;
       live = await store.switchTo(operation, {
         create: { settings: initial, promptVersion },
@@ -242,7 +303,11 @@ export function createSettings(opts: {
     });
   }
 
-  /** Changes only some settings (e.g. the fallback), keeping the rest. */
+  /**
+   * Changes only some settings (e.g. the fallback), keeping the rest.
+   * Throws `NoSettingsError` for a feature with no settings to keep: use
+   * `change` or `seed` with full settings first.
+   */
   async function update(
     operation: string,
     patch: Partial<Settings>,
@@ -301,6 +366,42 @@ export function createSettings(opts: {
     }
   }
 
+  /**
+   * Records existing settings as a feature's version 1, for an app moving
+   * its model choices into versioned settings (an import from its own
+   * config table). Only when the feature has no version yet: otherwise it
+   * changes nothing and returns null, so an import can run again safely.
+   * `promptVersion` defaults to the instructions' current hash.
+   */
+  async function seed(
+    operation: string,
+    initial: Settings,
+    by: {
+      by: string | null;
+      reason: string | null;
+      promptVersion?: string | null;
+    },
+  ): Promise<SettingsVersion | null> {
+    if (await store.liveVersion(operation)) return null;
+    const promptVersion =
+      by.promptVersion !== undefined
+        ? by.promptVersion
+        : await prompt(operation);
+    try {
+      return await store.switchTo(operation, {
+        create: { settings: settingsOf(initial), promptVersion },
+        kind: "CHANGE",
+        reason: by.reason,
+        by: by.by,
+        expectLive: null,
+      });
+    } catch (e) {
+      // Another request seeded it first: that version stands.
+      if (e instanceof SettingsConflict) return null;
+      throw e;
+    }
+  }
+
   /** Every switch, newest first, with the versions it went from and to. */
   async function history(
     operation: string,
@@ -324,7 +425,16 @@ export function createSettings(opts: {
     });
   }
 
-  return { current, route, change, update, rollBack, syncPrompt, history };
+  return {
+    current,
+    route,
+    change,
+    update,
+    rollBack,
+    syncPrompt,
+    seed,
+    history,
+  };
 }
 
 /** What createSettings returns. */

package/src/store.ts

@@ -77,6 +77,11 @@ export type RunRecord = {
   promptVersion: string;
   /** The reasoning setting tested; null: the model's own default. */
   reasoningEffort: string | null;
+  /**
+   * The temperature tested; null or unset: none sent. Stores keep it, so a
+   * test at another temperature than the live one shows as out of date.
+   */
+  temperature?: number | null;
   set: EvalSet;
   repeats: number;
   /** Changes when the cases or their answers do, so tests compare. */

package/src/suggest.ts

@@ -1,4 +1,4 @@
-import { retiresLabel, type ModelCatalog } from "./models.js";
+import { isBaseline, retiresLabel, type ModelCatalog } from "./models.js";
 import type { Rate } from "./stats.js";
 import type { CheckDef, SuitePriority } from "./types.js";
 import type { Verdict } from "./compare.js";
@@ -109,8 +109,10 @@ export function suggest<R extends SuggestRow>(
   const retiring = (r: SuggestRow) => retiresLabel(models, r.model);
   const limitMs = priority.maxSeconds ? priority.maxSeconds * 1000 : Infinity;
   const tooSlow = (r: SuggestRow) => ms(r) !== Infinity && ms(r) > limitMs;
+  // Baselines are code, compared on the board but never suggested.
   const tested = rows.filter(
     (r) =>
+      !isBaseline(models, r.model) &&
       r.test &&
       !r.test.stale &&
       !r.test.failedAnswers &&

package/src/testing.ts

@@ -127,6 +127,14 @@ export function describeStoreContract(
       expect(await store.getRun("missing")).toBeNull();
     });
 
+    it("keeps the temperature a run was tested at", async () => {
+      const store = await make();
+      const warm = await store.createRun({ ...newRun("s"), temperature: 0.7 });
+      const none = await store.createRun({ ...newRun("s"), temperature: null });
+      expect((await store.getRun(warm.id))?.temperature).toBe(0.7);
+      expect((await store.getRun(none.id))?.temperature ?? null).toBeNull();
+    });
+
     it("updates a run only while it has the status it's guarded by", async () => {
       const store = await make();
       const r = await store.createRun(newRun("s"));
@@ -381,6 +389,21 @@ export function describeSettingsStoreContract(
       expect(await store.listVersions("triage")).toHaveLength(2);
     });
 
+    it("keeps a version's temperature", async () => {
+      const store = await make();
+      const v1 = await store.switchTo("triage", {
+        ...create("a", null),
+        create: {
+          settings: { ...settings("a"), temperature: 0.4 },
+          promptVersion: "p1",
+        },
+      });
+      expect(v1.temperature).toBe(0.4);
+      expect((await store.liveVersion("triage"))?.temperature).toBe(0.4);
+      const v2 = await store.switchTo("triage", create("b", v1.id));
+      expect(v2.temperature ?? null).toBeNull();
+    });
+
     it("keeps each feature's versions and live one apart", async () => {
       const store = await make();
       await store.switchTo("triage", create("a", null));

package/src/types.ts

@@ -108,6 +108,12 @@ export type Suite<Input, Output, Label> = {
       model: string;
       /** A reasoning setting to test; undefined: the operation's own. */
       reasoningEffort?: ReasoningEffort | null;
+      /**
+       * The temperature the test runs at: the operation's live one when the
+       * test was started. Null: send none. Undefined from runners before
+       * temperature was versioned.
+       */
+      temperature?: number | null;
       onCost: (usd: number) => void;
     },
   ): Promise<Output>;

package/src/weekly.ts

@@ -104,13 +104,15 @@ export function weeklySummary(args: {
       continue;
     }
     if (r.status !== "SUCCEEDED") continue;
-    // The last check of the same model, instructions, and grading, a day or
-    // more earlier: anything else changed for a known reason.
+    // The last check of the same model, temperature, instructions, and
+    // grading, a day or more earlier: anything else changed for a known
+    // reason.
     const prior = scheduled.find(
       (p) =>
         p.suite === r.suite &&
         p.status === "SUCCEEDED" &&
         p.model === r.model &&
+        (p.temperature ?? null) === (r.temperature ?? null) &&
         p.promptVersion === r.promptVersion &&
         p.graderVersion === r.graderVersion &&
         +r.startedAt - +p.startedAt >= DAY,