@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/templates/components/evalkeeper/ui.tsx

@@ -0,0 +1,103 @@
+import Link from "next/link";
+import { ArrowLeft } from "lucide-react";
+import type { HealthStatus } from "@crowandraven/evalkeeper";
+
+export const pct = (x: number) => `${Math.round(x * 100)}%`;
+
+const STATUS: Record<HealthStatus, { label: string; tone: keyof typeof CHIP }> =
+  {
+    good: { label: "Good", tone: "live" },
+    fair: { label: "Fair", tone: "warn" },
+    improve: { label: "Needs work", tone: "serious" },
+    few: { label: "Few cases", tone: "warn" },
+    stale: { label: "Retest", tone: "warn" },
+    untested: { label: "Untested", tone: "neutral" },
+  };
+
+/** A suite's health as a chip; the card's details say why. */
+export function StatusBadge({ status }: { status: HealthStatus }) {
+  const s = STATUS[status];
+  return <Chip tone={s.tone}>{s.label}</Chip>;
+}
+
+export function Card({
+  children,
+  className = "",
+}: {
+  children: React.ReactNode;
+  className?: string;
+}) {
+  return (
+    <section
+      className={`rounded-lg border border-line bg-card p-5 ${className}`}
+    >
+      {children}
+    </section>
+  );
+}
+
+/** A score with a thin bar, for tables. */
+export function Score({ right, total }: { right: number; total: number }) {
+  const rate = total ? right / total : 0;
+  return (
+    <div className="flex items-center gap-2">
+      <span className="w-10 tabular-nums">{pct(rate)}</span>
+      <span className="h-1.5 w-20 rounded-full bg-tint">
+        <span
+          className="block h-1.5 rounded-full bg-accent"
+          style={{ width: `${rate * 100}%` }}
+        />
+      </span>
+      <span className="text-xs text-muted tabular-nums">
+        {right}/{total}
+      </span>
+    </div>
+  );
+}
+
+const CHIP = {
+  live: "bg-emerald-100 text-emerald-800 dark:bg-emerald-500/15 dark:text-emerald-300",
+  fallback: "bg-sky-100 text-sky-800 dark:bg-sky-500/15 dark:text-sky-200",
+  warn: "bg-amber-100 text-amber-800 dark:bg-amber-500/15 dark:text-amber-200",
+  serious: "bg-red-50 text-red-800 dark:bg-red-500/10 dark:text-red-200",
+  neutral: "bg-tint text-muted",
+} as const;
+
+/** A chip's color. */
+export type ChipTone = keyof typeof CHIP;
+
+/** A small rounded label: Live, Fallback, Retires …, serious, minor. */
+export function Chip({
+  tone,
+  children,
+}: {
+  tone: keyof typeof CHIP;
+  children: React.ReactNode;
+}) {
+  return (
+    <span
+      className={`inline-flex h-6 items-center gap-1 whitespace-nowrap rounded-full px-2.5 text-xs font-medium leading-none ${CHIP[tone]}`}
+    >
+      {children}
+    </span>
+  );
+}
+
+/** A link back to the page above, with an arrow. */
+export function BackLink({
+  href,
+  children,
+}: {
+  href: string;
+  children: React.ReactNode;
+}) {
+  return (
+    <Link
+      href={href}
+      className="inline-flex items-center gap-1 text-sm text-muted hover:text-ink"
+    >
+      <ArrowLeft aria-hidden className="h-4 w-4" />
+      {children}
+    </Link>
+  );
+}

package/templates/lib/evalkeeper/actions.ts

@@ -0,0 +1,263 @@
+"use server";
+
+import { after } from "next/server";
+import { revalidatePath } from "next/cache";
+import { caseFromFeedback, formText, suggest } from "@crowandraven/evalkeeper";
+import {
+  MODELS,
+  SUITES,
+  modelsFor,
+  feedbackStore,
+  requireAdmin,
+  runner,
+  settings,
+  store,
+} from ".";
+import { EVALS_BASE, evalsPath } from "./paths";
+import { runWeeklyCheck, startWeeklyCheck } from "./weekly";
+
+/**
+ * Every action on the eval pages. Each one checks requireAdmin() first,
+ * and records the admin as who made a change.
+ */
+
+const suiteFrom = (form: FormData) =>
+  runner.suiteOrThrow(formText(form, "suite"));
+
+export async function testModel(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  const model = formText(form, "model");
+  if (!modelsFor(suite.operation).includes(model))
+    throw new Error("Unknown model");
+  const { run } = await runner.createRun({ suite: suite.key, model });
+  // Answers come back after the response; the page refreshes until done.
+  after(() => runner.runEval(run.id));
+  revalidatePath(evalsPath(suite.key));
+}
+
+export async function testAll(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  for (const model of modelsFor(suite.operation)) {
+    const { run } = await runner.createRun({ suite: suite.key, model });
+    after(() => runner.runEval(run.id));
+  }
+  revalidatePath(evalsPath(suite.key));
+}
+
+/**
+ * The evidence for a promotion: the model's current test, the live
+ * model's, and how they compared, so the history can show why.
+ */
+async function evidenceFor(suiteKey: string, model: string) {
+  const board = await runner.board(suiteKey);
+  const row = board.rows.find((r) => r.model === model && !r.variant);
+  if (!row?.test || row.test.stale)
+    throw new Error("Test this model on the current cases first");
+  return {
+    row,
+    evidence: {
+      runId: row.test.runId,
+      liveRunId: board.liveRunId,
+      vsLive: row.vsLive,
+      score: { right: row.test.right, total: row.test.total },
+    },
+  };
+}
+
+const VS: Record<string, string> = {
+  BETTER: "more accurate than live",
+  SAME: "about as accurate as live",
+  WORSE: "less accurate than live",
+  TOO_FEW: "too few cases to compare",
+};
+
+export async function makeLive(form: FormData) {
+  const admin = await requireAdmin();
+  const suite = suiteFrom(form);
+  const model = formText(form, "model");
+  const { row, evidence } = await evidenceFor(suite.key, model);
+  await settings.update(
+    suite.operation,
+    { model },
+    {
+      by: admin.name,
+      reason: `Made ${row.label} live: ${evidence.score.right}/${evidence.score.total}, ${VS[row.vsLive ?? ""] ?? "no comparison"}`,
+      evidence,
+    },
+  );
+  revalidatePath(EVALS_BASE, "layout");
+}
+
+export async function makeFallback(form: FormData) {
+  const admin = await requireAdmin();
+  const suite = suiteFrom(form);
+  const model = formText(form, "model");
+  const { row, evidence } = await evidenceFor(suite.key, model);
+  await settings.update(
+    suite.operation,
+    { fallbackModel: model },
+    {
+      by: admin.name,
+      reason: `Made ${row.label} the fallback: ${evidence.score.right}/${evidence.score.total}, from another provider`,
+      evidence,
+    },
+  );
+  revalidatePath(EVALS_BASE, "layout");
+}
+
+/** Applies the suggestion: its live and fallback models, in one change. */
+export async function useSuggestion(form: FormData) {
+  const admin = await requireAdmin();
+  const suite = suiteFrom(form);
+  const board = await runner.board(suite.key);
+  const s = suggest(board.rows, suite.checks, suite.priority, MODELS);
+  if (!s || s.same) return;
+  await settings.update(
+    suite.operation,
+    {
+      model: s.live.model,
+      fallbackModel: s.fallback?.model ?? null,
+    },
+    {
+      by: admin.name,
+      reason: s.reasons.join(" "),
+      evidence: {
+        suggestion: s.reasons,
+        live: s.live.test?.runId ?? null,
+        fallback: s.fallback?.test?.runId ?? null,
+      },
+    },
+  );
+  revalidatePath(EVALS_BASE, "layout");
+}
+
+/** Makes an earlier version live again; a reason is required. */
+export async function rollBackSettings(form: FormData) {
+  const admin = await requireAdmin();
+  const suite = suiteFrom(form);
+  const reason = formText(form, "reason");
+  if (!reason) throw new Error("Say why you're rolling back");
+  await settings.rollBack(suite.operation, formText(form, "version"), {
+    by: admin.name,
+    reason,
+  });
+  revalidatePath(EVALS_BASE, "layout");
+}
+
+/** A person confirms an answer: the only way a draft gets used. */
+export async function confirmAnswer(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  await store.setLabelStatus(formText(form, "label"), "CONFIRMED");
+  after(() => runner.rescoreRecent(suite.key));
+  revalidatePath(evalsPath(suite.key));
+}
+
+export async function rescore(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  await runner.rescoreRecent(suite.key);
+  revalidatePath(evalsPath(suite.key));
+}
+
+/** A person adds a case. They wrote its answer, so it's confirmed. */
+export async function addCase(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  if (!suite.labelFromForm) throw new Error("This suite has no answer form");
+  await store.createCase({
+    suite: suite.key,
+    input: suite.inputFromForm(form),
+    tags: formText(form, "tags")
+      .split(",")
+      .map((t) => t.trim())
+      .filter(Boolean),
+    split: formText(form, "split") === "TEST" ? "TEST" : "DEV",
+    status: "ACTIVE",
+    source: "OWNER",
+    sourceRef: null,
+    note: null,
+    labels: [
+      {
+        kind: "EXPECTED",
+        status: "CONFIRMED",
+        value: suite.labelFromForm(form),
+        raterRole: "OWNER",
+        raterId: null,
+        note: null,
+      },
+    ],
+  });
+  revalidatePath(evalsPath(suite.key));
+}
+
+/**
+ * A person changes a case's answer: a new confirmed answer, with the old
+ * ones kept as history. Stored answers are rescored for free.
+ */
+export async function editAnswer(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  if (!suite.labelFromForm) throw new Error("This suite has no answer form");
+  const caseId = formText(form, "case");
+  const known = (await store.listCases(suite.key)).some((c) => c.id === caseId);
+  if (!known) throw new Error("Unknown case");
+  await store.addLabel(caseId, {
+    kind: "EXPECTED",
+    status: "CONFIRMED",
+    value: suite.labelFromForm(form),
+    raterRole: "OWNER",
+    raterId: null,
+    note: null,
+  });
+  // A draft (from feedback) is in use once it has an answer.
+  await store.setCaseStatus(caseId, "ACTIVE");
+  after(() => runner.rescoreRecent(suite.key));
+  revalidatePath(evalsPath(suite.key));
+}
+
+/** Takes a case out of use; its stored answers stay. */
+export async function retireCase(form: FormData) {
+  await requireAdmin();
+  const suite = suiteFrom(form);
+  const caseId = formText(form, "case");
+  const known = (await store.listCases(suite.key)).some((c) => c.id === caseId);
+  if (!known) throw new Error("Unknown case");
+  await store.setCaseStatus(caseId, "RETIRED");
+  revalidatePath(evalsPath(suite.key));
+}
+
+async function find(id: string) {
+  const f = (await feedbackStore.listFeedback()).find((x) => x.id === id);
+  if (!f) throw new Error("Unknown rating");
+  return f;
+}
+
+/** Makes a draft case from a rating; a person finishes its answer. */
+export async function makeCase(form: FormData) {
+  await requireAdmin();
+  const f = await find(formText(form, "feedback"));
+  const suite = SUITES.find((s) => s.operation === f.operation);
+  const draft = suite ? caseFromFeedback(f, suite) : null;
+  if (!draft || !suite) throw new Error("This rating can't become a case");
+  const c = await store.createCase(draft);
+  await feedbackStore.setFeedbackStatus(f.id, "CASED", c.id);
+  revalidatePath(EVALS_BASE, "layout");
+}
+
+export async function dismissFeedback(form: FormData) {
+  await requireAdmin();
+  const f = await find(formText(form, "feedback"));
+  await feedbackStore.setFeedbackStatus(f.id, "DISMISSED");
+  revalidatePath(evalsPath("feedback"));
+}
+
+export async function runWeeklyNow(): Promise<void> {
+  await requireAdmin();
+  // Recorded before the page re-renders, so it shows them running.
+  const ids = await startWeeklyCheck();
+  after(() => runWeeklyCheck(ids));
+  revalidatePath(EVALS_BASE);
+}

package/templates/lib/evalkeeper/cronAuth.ts

@@ -0,0 +1,18 @@
+import { timingSafeEqual } from "node:crypto";
+
+/**
+ * Whether a request carries the cron secret ("Authorization: Bearer …").
+ * Fails closed when no secret is set, and compares in constant time so the
+ * secret can't be guessed a character at a time.
+ */
+export function isAuthorizedCron(
+  header: string | null,
+  secret: string | undefined,
+): boolean {
+  if (!secret || !header) return false;
+  const presented = Buffer.from(header);
+  const expected = Buffer.from(`Bearer ${secret}`);
+  return (
+    presented.length === expected.length && timingSafeEqual(presented, expected)
+  );
+}

package/templates/lib/evalkeeper/paths.ts

@@ -0,0 +1,11 @@
+/**
+ * Where the eval pages live in your app. The CLI sets this to the base you
+ * choose (`npx @crowandraven/evalkeeper add pages --base admin/evals`); every link and
+ * refresh on the pages is built from it.
+ */
+export const EVALS_BASE = "/evals";
+
+/** A path under the eval pages: evalsPath("ticket_triage", "runs", id). */
+export function evalsPath(...parts: string[]): string {
+  return [EVALS_BASE, ...parts.map(encodeURIComponent)].join("/");
+}

package/templates/lib/evalkeeper/time.ts

@@ -0,0 +1,4 @@
+/** The moment `days` days ago, for "this week" summaries. */
+export function daysAgo(days: number): Date {
+  return new Date(Date.now() - days * 86_400_000);
+}

package/templates/lib/evalkeeper/weekly.ts

@@ -0,0 +1,51 @@
+import "server-only";
+import {
+  thumbsSummary,
+  weeklySummary,
+  type WeeklySummary,
+} from "@crowandraven/evalkeeper";
+import { MODELS, SUITES, feedbackStore, routeFor, runner, store } from ".";
+import { daysAgo } from "./time";
+
+/**
+ * The weekly check: each suite's quick set on its live model. `start`
+ * records every suite's test first (so the hub shows them running), then
+ * `run` answers them one suite after another. On a platform with a request
+ * limit, give the runner `requestLimitMs` and `continueRun` so a slow suite
+ * carries on in a fresh request.
+ */
+export async function startWeeklyCheck(): Promise<string[]> {
+  const ids: string[] = [];
+  for (const suite of SUITES) {
+    const id = await runner.startDriftCheck(suite.key);
+    if (id) ids.push(id);
+  }
+  return ids;
+}
+
+export async function runWeeklyCheck(ids: string[]): Promise<void> {
+  // runEval never throws: a suite that fails says why on its own test.
+  for (const id of ids) await runner.runEval(id, { fresh: true });
+}
+
+/** Where the weekly check stands, for the evals hub. */
+export async function loadWeekly(): Promise<WeeklySummary> {
+  const [runs, routes, feedback] = await Promise.all([
+    Promise.all(SUITES.map((s) => store.listRuns(s.key, { limit: 50 }))).then(
+      (r) => r.flat(),
+    ),
+    Promise.all(
+      SUITES.map(
+        async (s) => [s.operation, await routeFor(s.operation)] as const,
+      ),
+    ).then((r) => Object.fromEntries(r)),
+    feedbackStore.listFeedback({ since: daysAgo(7) }),
+  ]);
+  return weeklySummary({
+    runs,
+    suites: SUITES,
+    routes,
+    models: MODELS,
+    thumbs: thumbsSummary(feedback, daysAgo(7)),
+  });
+}