@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/src/upkeepRules.ts

@@ -0,0 +1,46 @@
+/**
+ * Rules for the weekly eval check, pure so they're tested: when a quick-set
+ * score has drifted, when a model is about to retire, and when people's
+ * thumbs-down rate needs a look.
+ */
+
+/** A drop this big on the quick set is worth a look. */
+export const DRIFT_DROP = 0.15;
+/** Warn about a live or fallback model this long before it retires. */
+export const RETIRE_WARN_DAYS = 60;
+/** Thumbs: at least this many ratings, at least this share down. */
+export const THUMBS_MIN = 5;
+/**
+ * The share of thumbs down that flags a feature (with at least THUMBS_MIN ratings).
+ */
+export const THUMBS_DOWN_SHARE = 0.2;
+
+/** This week's quick-set score against last week's: a drop worth a look. */
+export function drifted(previous: number, latest: number): boolean {
+  return previous - latest >= DRIFT_DROP;
+}
+
+/**
+ * Days until a "YYYY-MM" or "YYYY-MM-DD" retirement; a month counts from
+ * its first day. Negative once it's passed.
+ */
+export function daysUntil(retiring: string, today: Date): number {
+  const [y = 1970, m = 1, d = 1] = retiring.split("-").map(Number);
+  const when = Date.UTC(y, (m || 1) - 1, d || 1);
+  return Math.floor((when - today.getTime()) / 86_400_000);
+}
+
+/** Whether a model retires within the warning window, or already has. */
+export function retiresSoon(retiring: string | undefined, today: Date) {
+  return (
+    retiring !== undefined && daysUntil(retiring, today) <= RETIRE_WARN_DAYS
+  );
+}
+
+/**
+ * Whether a feature's ratings need a look: enough of them, and enough down.
+ */
+export function thumbsAlert(up: number, down: number): boolean {
+  const n = up + down;
+  return n >= THUMBS_MIN && down / n >= THUMBS_DOWN_SHARE;
+}

package/src/util.ts

@@ -0,0 +1,17 @@
+/**
+ * The item at `i`, for indexes the caller has already checked (a loop
+ * bound, a non-empty list). Throws instead of returning undefined, so a
+ * wrong index fails loudly rather than as NaN or a missing row.
+ */
+export function at<T>(items: readonly T[], i: number): T {
+  if (i < 0 || i >= items.length)
+    throw new RangeError(`Index ${i} is outside 0..${items.length - 1}`);
+  return items[i] as T;
+}
+
+/** A map's value for a key the caller knows is there. Throws otherwise. */
+export function mustGet<K, V>(map: ReadonlyMap<K, V>, key: K): V {
+  const v = map.get(key);
+  if (v === undefined) throw new Error(`Missing ${String(key)}`);
+  return v;
+}

package/src/weekly.ts

@@ -0,0 +1,169 @@
+import type { ThumbsSummary } from "./feedback.js";
+import { retiresLabel, type ModelCatalog } from "./models.js";
+import type { RunMetrics, RunRecord } from "./store.js";
+import { daysUntil, drifted, retiresSoon } from "./upkeepRules.js";
+
+/**
+ * The weekly check: each suite's quick set on its live model, with fresh
+ * answers, so a provider quietly changing a model under the same name shows
+ * up as drift. The summary also flags models retiring soon and features
+ * people rate poorly. It only reports; a person decides what to change.
+ * Pure.
+ */
+
+/** `startedBy` on tests the weekly check starts, so they're told apart. */
+export const SCHEDULED = "schedule";
+
+const DAY = 86_400_000;
+
+/**
+ * What the last weekly check found: drift, unfinished checks, retiring models, and
+ * thumbs-down, plus when it ran and what it cost.
+ */
+export type WeeklySummary = {
+  /** When the last weekly check started; null before the first. */
+  lastAt: Date | null;
+  /** What last week's quick sets cost (reused answers are free). */
+  costUsd: number;
+  /** Quick sets still running. */
+  running: number;
+  drift: {
+    suite: string;
+    label: string;
+    model: string;
+    previous: number;
+    latest: number;
+  }[];
+  /** Quick sets that didn't finish, and why. */
+  failed: { suite: string; label: string; error: string }[];
+  /** Live and fallback models retiring within the warning window. */
+  retiring: {
+    suite: string;
+    label: string;
+    model: string;
+    role: "live" | "fallback";
+    when: string;
+    days: number;
+  }[];
+  /** Features with a high thumbs-down share this week. */
+  thumbs: ThumbsSummary[];
+  /** Ran, and nothing needs a look. */
+  clear: boolean;
+};
+
+/** A test's headline score: parts right where it gives partial credit. */
+export function headline(m: RunMetrics | null): number | null {
+  if (!m) return null;
+  return m.units && m.units.total
+    ? m.units.right / m.units.total
+    : (m.allPass.rate ?? 0);
+}
+
+/**
+ * What the last weekly check found, from every suite's tests.
+ *
+ * @example
+ * ```ts
+ * const runs = (await Promise.all(SUITES.map((s) => store.listRuns(s.key, { limit: 50 })))).flat();
+ * const weekly = weeklySummary({ runs, suites: SUITES, routes, models: MODELS, thumbs });
+ * if (!weekly.clear) console.log(weekly.drift, weekly.retiring);
+ * ```
+ */
+export function weeklySummary(args: {
+  /** Every suite's tests, newest first or not. */
+  runs: RunRecord[];
+  suites: { key: string; label: string; operation: string }[];
+  /** Each operation's live and fallback model. */
+  routes: Record<string, { model: string; fallbackModel: string | null }>;
+  models: ModelCatalog;
+  /** From thumbsSummary over the last week; only flagged ones are shown. */
+  thumbs?: ThumbsSummary[];
+  today?: Date;
+}): WeeklySummary {
+  const today = args.today ?? new Date();
+  const label = (key: string) =>
+    args.suites.find((s) => s.key === key)?.label ?? key;
+  const scheduled = args.runs
+    .filter((r) => r.startedBy === SCHEDULED && r.set === "SMOKE")
+    .sort((a, b) => +b.startedAt - +a.startedAt);
+  const lastAt = scheduled[0]?.startedAt ?? null;
+  // The last weekly batch: everything started within a day of the newest.
+  const batch = lastAt
+    ? scheduled.filter((r) => +lastAt - +r.startedAt < DAY)
+    : [];
+
+  const drift: WeeklySummary["drift"] = [];
+  const failed: WeeklySummary["failed"] = [];
+  for (const r of batch) {
+    if (r.status === "FAILED") {
+      failed.push({
+        suite: r.suite,
+        label: label(r.suite),
+        error: r.error ?? "unknown error",
+      });
+      continue;
+    }
+    if (r.status !== "SUCCEEDED") continue;
+    // The last check of the same model, temperature, instructions, and
+    // grading, a day or more earlier: anything else changed for a known
+    // reason.
+    const prior = scheduled.find(
+      (p) =>
+        p.suite === r.suite &&
+        p.status === "SUCCEEDED" &&
+        p.model === r.model &&
+        (p.temperature ?? null) === (r.temperature ?? null) &&
+        p.promptVersion === r.promptVersion &&
+        p.graderVersion === r.graderVersion &&
+        +r.startedAt - +p.startedAt >= DAY,
+    );
+    const latest = headline(r.metrics);
+    const previous = headline(prior?.metrics ?? null);
+    if (latest !== null && previous !== null && drifted(previous, latest))
+      drift.push({
+        suite: r.suite,
+        label: label(r.suite),
+        model: args.models[r.model]?.label ?? r.model,
+        previous,
+        latest,
+      });
+  }
+
+  const retiring: WeeklySummary["retiring"] = [];
+  for (const s of args.suites) {
+    const route = args.routes[s.operation];
+    if (!route) continue;
+    for (const [model, role] of [
+      [route.model, "live"],
+      [route.fallbackModel, "fallback"],
+    ] as const) {
+      const when = model ? args.models[model]?.retiring : undefined;
+      if (model && when && retiresSoon(when, today))
+        retiring.push({
+          suite: s.key,
+          label: s.label,
+          model: args.models[model]?.label ?? model,
+          role,
+          when: retiresLabel(args.models, model) ?? when,
+          days: daysUntil(when, today),
+        });
+    }
+  }
+
+  const thumbs = (args.thumbs ?? []).filter((t) => t.alert);
+  return {
+    lastAt,
+    costUsd: batch.reduce((t, r) => t + (r.metrics?.costUsd ?? 0), 0),
+    running: batch.filter((r) => r.status === "RUNNING").length,
+    drift,
+    failed,
+    retiring,
+    thumbs,
+    clear:
+      lastAt !== null &&
+      !drift.length &&
+      !failed.length &&
+      !retiring.length &&
+      !thumbs.length,
+  };
+}

package/starter/index.ts

@@ -0,0 +1,86 @@
+// Your app's Evalkeeper wiring: the one file the copied eval pages import.
+// Written by `npx @crowandraven/evalkeeper add pages`; it's yours now. Fill in each TODO.
+import "server-only";
+import {
+  createSettings,
+  type AnySuite,
+  type ModelCatalog,
+  type Settings,
+} from "@crowandraven/evalkeeper";
+import { createRunner, type Route } from "@crowandraven/evalkeeper/runner";
+import {
+  prismaFeedbackStore,
+  prismaSettingsStore,
+  prismaStore,
+} from "@crowandraven/evalkeeper-prisma";
+// TODO: your app's Prisma client, with Evalkeeper's models added to the
+// schema (node_modules/@crowandraven/evalkeeper-prisma/schema/evalkeeper.prisma).
+import { prisma } from "@/lib/prisma";
+
+/** TODO: the models your app can use, with prices and retirement dates. */
+export const MODELS: ModelCatalog = {};
+
+/** Models you can test now, e.g. only those whose API key is set. */
+export function availableModels(): string[] {
+  return Object.keys(MODELS);
+}
+
+/**
+ * The models a feature can be tested on now. TODO: return decision models
+ * for features graded as decisions, if you have any.
+ */
+export function modelsFor(_operation: string): string[] {
+  return availableModels();
+}
+
+/** TODO: one suite per AI feature (see the README's "Write a suite"). */
+export const SUITES: AnySuite[] = [];
+export const SUITE_BY_KEY: Record<string, AnySuite> = Object.fromEntries(
+  SUITES.map((s) => [s.key, s]),
+);
+
+/**
+ * TODO: each feature's instructions hash (instructionsHash from
+ * @crowandraven/evalkeeper-ai-sdk), so editing a prompt marks older tests out of date.
+ */
+const PROMPT_VERSIONS: Record<string, string> = {};
+
+/** TODO: each feature's settings in code, used until someone changes them. */
+const DEFAULTS: Record<string, Settings> = {};
+
+export const store = prismaStore(prisma);
+export const feedbackStore = prismaFeedbackStore(prisma);
+
+export const settings = createSettings({
+  store: prismaSettingsStore(prisma),
+  defaults: (operation) => {
+    const d = DEFAULTS[operation];
+    if (!d) throw new Error(`No default settings for ${operation}`);
+    return d;
+  },
+  promptVersion: (operation) => PROMPT_VERSIONS[operation] ?? "unregistered",
+});
+
+export const routeFor = (operation: string): Promise<Route> =>
+  settings.route(operation);
+
+export const runner = createRunner({
+  store,
+  suites: SUITES,
+  models: MODELS,
+  promptVersion: (operation) => PROMPT_VERSIONS[operation] ?? "unregistered",
+  route: routeFor,
+  candidates: (operation) => ({ models: modelsFor(operation) }),
+});
+
+/**
+ * TODO: who may use the eval pages. Every page and action calls this
+ * first. Check your session, throw when the person isn't an admin, and
+ * return who they are (recorded as who made each change). Until you do,
+ * nobody can open the pages.
+ */
+export async function requireAdmin(): Promise<{ id: string; name: string }> {
+  throw new Error(
+    "Evalkeeper: write requireAdmin in lib/evalkeeper/index.ts so only admins can use the eval pages.",
+  );
+}

package/templates/app/api/cron/evals-weekly/route.ts

@@ -0,0 +1,24 @@
+import { after, type NextRequest, NextResponse } from "next/server";
+import { isAuthorizedCron } from "@/lib/evalkeeper/cronAuth";
+import { runWeeklyCheck, startWeeklyCheck } from "@/lib/evalkeeper/weekly";
+
+/** The checks run after the response, within this function's time limit. */
+export const maxDuration = 300;
+
+/**
+ * Mondays (vercel.json): the weekly eval check. Answers at once, then
+ * runs each suite's quick set on its live model. Results show on the
+ * evals hub. Call it with "Authorization: Bearer $CRON_SECRET".
+ */
+export async function GET(request: NextRequest) {
+  if (
+    !isAuthorizedCron(
+      request.headers.get("authorization"),
+      process.env.CRON_SECRET,
+    )
+  )
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  const ids = await startWeeklyCheck();
+  after(() => runWeeklyCheck(ids));
+  return NextResponse.json({ started: ids.length }, { status: 202 });
+}