@crowandraven/evalkeeper 0.2.0 → 0.4.0

package/templates/app/evals/[suite]/page.tsx

@@ -7,7 +7,6 @@ import {
   money,
   probabilityQuality,
   perAnswer,
-  type ReasoningEffort,
   retiresLabel,
   seconds,
   seriousMistakes,
@@ -56,17 +55,23 @@ export default async function SuitePage({
 }) {
   await requireAdmin();
   const { suite: key } = await params;
-  if (!SUITE_BY_KEY[key]) notFound();
-  const b = await runner.board(key);
-  // Records a deploy if the instructions changed under the live settings.
-  await settings.syncPrompt(b.suite.operation);
+  const operation = SUITE_BY_KEY[key]?.operation;
+  if (!operation) notFound();
+  // The board and the settings load together: on a remote database each
+  // waits on round trips, so one after the other doubles the wait.
+  const [b, [history, liveSettings]] = await Promise.all([
+    runner.board(key),
+    // Records a deploy if the instructions changed under the live
+    // settings, before reading them.
+    settings
+      .syncPrompt(operation)
+      .then(() =>
+        Promise.all([settings.history(operation), settings.current(operation)]),
+      ),
+  ]);
   const { suite, rows } = b;
   const s = suggest(rows, suite.checks, suite.priority, MODELS);
   const testing = rows.some((r) => r.testing);
-  const [history, liveSettings] = await Promise.all([
-    settings.history(suite.operation),
-    settings.current(suite.operation),
-  ]);
   // What testing each model would cost now, counting answers it would reuse.
   const estimates = new Map(
     await Promise.all(
@@ -75,7 +80,7 @@ export default async function SuitePage({
           [
             r.key,
             await runner.estimateRun(key, r.model, "ALL", 1, {
-              reasoningEffort: r.effort as ReasoningEffort | null,
+              reasoningEffort: r.effort,
             }),
           ] as const,
       ),

package/templates/app/evals/feedback/page.tsx

@@ -1,4 +1,3 @@
-import Link from "next/link";
 import {
   asText,
   thumbsSummary,
@@ -103,11 +102,17 @@ export default async function FeedbackPage() {
         </h2>
         {open.length === 0 && (
           <p className="text-sm text-muted">
-            Nothing waiting. Rate an answer on the{" "}
-            <Link href="/try" className="underline">
-              try it
-            </Link>{" "}
-            page to see one here.
+            Nothing waiting. Ratings show up here once an AI feature has a
+            thumbs up and down (step 8 of{" "}
+            <a
+              href="https://github.com/seanhudson/evalkeeper/blob/main/docs/adding-to-your-app.md#8-add-thumbs-to-your-ai-answers-optional"
+              target="_blank"
+              rel="noreferrer"
+              className="underline"
+            >
+              the guide
+            </a>
+            ).
           </p>
         )}
         <div className="space-y-3">

package/templates/app/evals/page.tsx

@@ -120,6 +120,7 @@ export default async function EvalsHub() {
           Suites
           <InfoTip term="suites" />
         </h2>
+        {boards.length === 0 && <NoSuites />}
         <div className="space-y-4">
           {boards.map((b, i) => {
             const h = suiteHealth(
@@ -147,6 +148,45 @@ export default async function EvalsHub() {
   );
 }
 
+/** Before the first suite: what a suite is, and how to add one. */
+function NoSuites() {
+  return (
+    <Card className="readable space-y-3 text-sm">
+      <p className="font-semibold">No suites yet</p>
+      <p className="text-muted">
+        A suite tests one AI feature: its cases, the answers a person confirms,
+        and the checks that grade any model against them. To add one:
+      </p>
+      <ol className="list-decimal space-y-2 pl-4">
+        <li>
+          Write it: the input and answer shapes with zod, two or three checks,{" "}
+          <code>run</code>, and <code>grade</code>.
+        </li>
+        <li>
+          Add it to <code>SUITES</code> in <code>lib/evalkeeper/index.ts</code>,
+          with its instructions hash.
+        </li>
+        <li>
+          Add cases (here, or with <code>importCases</code> from files you
+          have), confirm their answers, and test models.
+        </li>
+      </ol>
+      <p className="text-muted">
+        Step by step:{" "}
+        <a
+          href="https://github.com/seanhudson/evalkeeper/blob/main/docs/adding-to-your-app.md#5-write-your-first-suite"
+          target="_blank"
+          rel="noreferrer"
+          className="underline"
+        >
+          Write your first suite
+        </a>
+        .
+      </p>
+    </Card>
+  );
+}
+
 function Stat({
   title,
   term,
@@ -239,21 +279,21 @@ function SuiteCard({
       </div>
 
       {s && (
-        <div className="space-y-0.5 text-sm">
-          <p>
-            <span className="font-medium">v{version?.number ?? 1}</span>{" "}
-            <span className="text-muted">
-              · {since ? `live since ${day(since)}` : "defaults from code"}
-            </span>
-          </p>
-          <p className="text-xs text-muted">
-            {label(s.model)}
-            {s.fallbackModel
-              ? ` · fallback ${label(s.fallbackModel)}`
-              : " · no fallback"}
-            {s.reasoningEffort ? ` · reasoning ${s.reasoningEffort}` : ""}
-          </p>
-        </div>
+        <p className="text-sm text-muted">
+          <span className="font-semibold text-ink">{label(s.model)}</span>
+          {s.fallbackModel ? (
+            <>
+              {" "}
+              · fallback{" "}
+              <span className="font-semibold text-ink">
+                {label(s.fallbackModel)}
+              </span>
+            </>
+          ) : (
+            " · no fallback"
+          )}
+          {s.reasoningEffort ? ` · reasoning ${s.reasoningEffort}` : ""}
+        </p>
       )}
 
       <p className="text-sm">
@@ -289,6 +329,10 @@ function SuiteCard({
           <summary className="flex list-none items-center justify-between gap-2 p-3 text-sm font-medium">
             Versions
             <span className="flex items-center gap-1 text-xs font-normal text-muted">
+              <span className="font-medium text-ink">
+                v{version?.number ?? 1}
+              </span>{" "}
+              {since ? `live since ${day(since)}` : "defaults from code"} ·{" "}
               {versions} version{versions === 1 ? "" : "s"}
               <ChevronDown
                 aria-hidden

package/templates/components/evalkeeper/AddCase.tsx

@@ -2,6 +2,7 @@
 
 import { useState } from "react";
 import { Plus } from "lucide-react";
+import { QUIET_BUTTON } from "./Submit";
 import { Card } from "./ui";
 
 /**
@@ -31,7 +32,7 @@ export function AddCaseHeader({
               onClick={() => {
                 setOpen(!open);
               }}
-              className="inline-flex items-center gap-1.5 rounded-md border border-line bg-card px-3 py-1.5 text-sm hover:bg-tint"
+              className={`inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-sm ${QUIET_BUTTON}`}
             >
               <Plus aria-hidden className="h-4 w-4" />
               Add a case

package/templates/components/evalkeeper/AnswerKey.tsx

@@ -7,17 +7,13 @@ import {
   type BoardRow,
   type CheckResult,
 } from "@crowandraven/evalkeeper";
-import {
-  addCase,
-  confirmAnswer,
-  editAnswer,
-  retireCase,
-} from "@/lib/evalkeeper/actions";
+import { addCase, editAnswer, retireCase } from "@/lib/evalkeeper/actions";
 import { Check, ChevronRight, CircleAlert, X } from "lucide-react";
 import { store } from "@/lib/evalkeeper";
 import { FieldValues, Fields } from "./Fields";
 import { AddCaseHeader } from "./AddCase";
 import { CaseMenu } from "./CaseMenu";
+import { ConfirmAnswer } from "./ConfirmAnswer";
 import { Submit } from "./Submit";
 import { Chip, type ChipTone } from "./ui";
 import { InfoTip, Tip } from "./Tip";
@@ -64,9 +60,12 @@ export async function AnswerKey({
       r.test && !r.variant ? [[r.test.runId, r.label] as const] : [],
     ),
   );
-  const outputs = (
-    await Promise.all([...modelByRun.keys()].map((id) => store.listOutputs(id)))
-  ).flat();
+  // One query where the store can: one per test is slow on a remote
+  // database.
+  const runIds = [...modelByRun.keys()];
+  const outputs = store.listOutputsFor
+    ? await store.listOutputsFor(runIds)
+    : (await Promise.all(runIds.map((id) => store.listOutputs(id)))).flat();
   const byCase = new Map<string, { model: string; output: OutputRecord }[]>();
   for (const o of outputs) {
     byCase.set(o.caseId, [
@@ -189,7 +188,11 @@ function CaseItem({
   const tags = Array.isArray(c.tags) ? c.tags : [];
 
   return (
-    <details className="group rounded-md border border-line bg-card">
+    <details
+      className="group rounded-md border border-line bg-card"
+      // Where ConfirmAnswer moves on to after confirming the case above.
+      data-pending={answer?.status === "PENDING" ? "true" : undefined}
+    >
       <summary className="flex cursor-pointer list-none flex-wrap items-center justify-between gap-2 p-3 text-sm">
         <span className="font-medium">
           <ChevronRight
@@ -312,11 +315,7 @@ function CaseItem({
         )}
 
         {answer?.status === "PENDING" && (
-          <form action={confirmAnswer}>
-            <input type="hidden" name="suite" value={suite.key} />
-            <input type="hidden" name="label" value={answer.id} />
-            <Submit>Confirm this answer</Submit>
-          </form>
+          <ConfirmAnswer suite={suite.key} labelId={answer.id} />
         )}
 
         <CaseMenu

package/templates/components/evalkeeper/ConfirmAnswer.tsx

@@ -0,0 +1,46 @@
+"use client";
+
+import { confirmAnswer } from "@/lib/evalkeeper/actions";
+import { Submit } from "./Submit";
+
+/**
+ * Confirms a case's pending answer and moves straight on: this case closes
+ * and the next one waiting for a confirmation opens, while the save and
+ * the page's refresh finish behind it. Reviewing a run of drafts shouldn't
+ * wait on the whole page each time.
+ */
+export function ConfirmAnswer({
+  suite,
+  labelId,
+}: {
+  suite: string;
+  labelId: string;
+}) {
+  return (
+    <form
+      action={confirmAnswer}
+      onSubmit={(e) => {
+        const here = e.currentTarget.closest("details");
+        if (!here) return;
+        here.open = false;
+        let next = here.nextElementSibling;
+        while (
+          next &&
+          !(
+            next instanceof HTMLDetailsElement &&
+            next.dataset.pending === "true"
+          )
+        )
+          next = next.nextElementSibling;
+        if (next instanceof HTMLDetailsElement) {
+          next.open = true;
+          next.scrollIntoView({ block: "nearest" });
+        }
+      }}
+    >
+      <input type="hidden" name="suite" value={suite} />
+      <input type="hidden" name="label" value={labelId} />
+      <Submit>Confirm this answer</Submit>
+    </form>
+  );
+}

package/templates/components/evalkeeper/RefreshWhile.tsx

@@ -1,9 +1,15 @@
 "use client";
 
 import { useRouter } from "next/navigation";
-import { useEffect } from "react";
+import { useEffect, useTransition } from "react";
 
-/** Refreshes the page every few seconds while something runs. */
+/**
+ * Refreshes the page every few seconds while something runs, waiting for
+ * each refresh to land before scheduling the next. A refresh on a timer
+ * cancels one still in flight, so when the page takes longer to render
+ * than the interval (the board against a remote database), no refresh ever
+ * lands and the page never updates (#25).
+ */
 export function RefreshWhile({
   active,
   ms = 2000,
@@ -12,14 +18,17 @@ export function RefreshWhile({
   ms?: number;
 }) {
   const router = useRouter();
+  const [refreshing, startTransition] = useTransition();
   useEffect(() => {
-    if (!active) return;
-    const t = setInterval(() => {
-      router.refresh();
+    if (!active || refreshing) return;
+    const t = setTimeout(() => {
+      startTransition(() => {
+        router.refresh();
+      });
     }, ms);
     return () => {
-      clearInterval(t);
+      clearTimeout(t);
     };
-  }, [active, ms, router]);
+  }, [active, refreshing, ms, router]);
   return null;
 }

package/templates/components/evalkeeper/RollBack.tsx

@@ -0,0 +1,62 @@
+"use client";
+
+import { useState } from "react";
+import { rollBackSettings } from "@/lib/evalkeeper/actions";
+import { Submit } from "./Submit";
+import { Tip } from "./Tip";
+
+/**
+ * Rolls a feature back to an earlier version. The button stays off until
+ * there's a note saying why: the note is kept with the version, for
+ * whoever reads the history next.
+ */
+export function RollBack({
+  suite,
+  versionId,
+  number,
+}: {
+  suite: string;
+  versionId: string;
+  number: number;
+}) {
+  const [reason, setReason] = useState("");
+  const missing = reason.trim() === "";
+  const button = (
+    <Submit variant="quiet" disabled={missing}>
+      Roll back to v{number}
+    </Submit>
+  );
+  return (
+    <form
+      action={rollBackSettings}
+      className="flex flex-wrap items-center gap-2"
+    >
+      <input type="hidden" name="suite" value={suite} />
+      <input type="hidden" name="version" value={versionId} />
+      <label htmlFor={`why-${versionId}`} className="sr-only">
+        Why roll back to v{number}?
+      </label>
+      <input
+        id={`why-${versionId}`}
+        name="reason"
+        required
+        value={reason}
+        onChange={(e) => {
+          setReason(e.target.value);
+        }}
+        placeholder="Why roll back?"
+        className="rounded-md border border-line bg-card px-2 py-1 text-xs focus:border-accent focus:outline-none"
+      />
+      {missing ? (
+        <Tip
+          text="Say why you're rolling back first. The note is kept with the version, for whoever reads the history next."
+          id={`rollback-${versionId}`}
+        >
+          {button}
+        </Tip>
+      ) : (
+        button
+      )}
+    </form>
+  );
+}

package/templates/components/evalkeeper/SettingsHistory.tsx

@@ -5,8 +5,7 @@ import {
   type HistoryEntry,
   type Settings,
 } from "@crowandraven/evalkeeper";
-import { rollBackSettings } from "@/lib/evalkeeper/actions";
-import { Submit } from "./Submit";
+import { RollBack } from "./RollBack";
 import { InfoTip, Tip } from "./Tip";
 import { Chip } from "./ui";
 import { evalsPath } from "@/lib/evalkeeper/paths";
@@ -163,24 +162,11 @@ export function HistoryEntries({
                 </Link>
               )}
               {!isLive && (
-                <form
-                  action={rollBackSettings}
-                  className="flex flex-wrap items-center gap-2"
-                >
-                  <input type="hidden" name="suite" value={suite} />
-                  <input type="hidden" name="version" value={e.to.id} />
-                  <label htmlFor={`why-${e.id}`} className="sr-only">
-                    Why roll back to v{e.to.number}?
-                  </label>
-                  <input
-                    id={`why-${e.id}`}
-                    name="reason"
-                    required
-                    placeholder="Why roll back?"
-                    className="rounded-md border border-line bg-card px-2 py-1 text-xs focus:border-accent focus:outline-none"
-                  />
-                  <Submit variant="quiet">Roll back to v{e.to.number}</Submit>
-                </form>
+                <RollBack
+                  suite={suite}
+                  versionId={e.to.id}
+                  number={e.to.number}
+                />
               )}
             </div>
           </article>

package/templates/components/evalkeeper/Submit.tsx

@@ -3,23 +3,32 @@
 import { LoaderCircle } from "lucide-react";
 import { useFormStatus } from "react-dom";
 
+/**
+ * A button that isn't the page's main action: the app's accent as an
+ * outline, so it still reads as a button in the app's colors (set
+ * `--accent` in evalkeeper.css).
+ */
+export const QUIET_BUTTON =
+  "border border-accent/40 bg-card text-accent hover:bg-accent/10";
+
 /** A submit button that spins while its form's action runs. */
 export function Submit({
   children,
   variant = "primary",
+  disabled = false,
 }: {
   children: React.ReactNode;
   variant?: "primary" | "quiet";
+  /** Off until the form is ready; wrap it in a `<Tip>` saying why. */
+  disabled?: boolean;
 }) {
   const { pending } = useFormStatus();
   return (
     <button
       type="submit"
-      disabled={pending}
+      disabled={pending || disabled}
       className={`inline-flex items-center gap-2 whitespace-nowrap rounded-md px-3 py-1.5 text-sm disabled:opacity-60 ${
-        variant === "primary"
-          ? "bg-accent text-white"
-          : "border border-line bg-card text-ink hover:bg-tint"
+        variant === "primary" ? "bg-accent text-white" : QUIET_BUTTON
       }`}
     >
       {pending && <Spinner />}