@crowandraven/evalkeeper 0.3.0 → 0.4.1

package/templates/components/evalkeeper/ConfirmAnswer.tsx

@@ -0,0 +1,46 @@
+"use client";
+
+import { confirmAnswer } from "@/lib/evalkeeper/actions";
+import { Submit } from "./Submit";
+
+/**
+ * Confirms a case's pending answer and moves straight on: this case closes
+ * and the next one waiting for a confirmation opens, while the save and
+ * the page's refresh finish behind it. Reviewing a run of drafts shouldn't
+ * wait on the whole page each time.
+ */
+export function ConfirmAnswer({
+  suite,
+  labelId,
+}: {
+  suite: string;
+  labelId: string;
+}) {
+  return (
+    <form
+      action={confirmAnswer}
+      onSubmit={(e) => {
+        const here = e.currentTarget.closest("details");
+        if (!here) return;
+        here.open = false;
+        let next = here.nextElementSibling;
+        while (
+          next &&
+          !(
+            next instanceof HTMLDetailsElement &&
+            next.dataset.pending === "true"
+          )
+        )
+          next = next.nextElementSibling;
+        if (next instanceof HTMLDetailsElement) {
+          next.open = true;
+          next.scrollIntoView({ block: "nearest" });
+        }
+      }}
+    >
+      <input type="hidden" name="suite" value={suite} />
+      <input type="hidden" name="label" value={labelId} />
+      <Submit>Confirm this answer</Submit>
+    </form>
+  );
+}

package/templates/components/evalkeeper/RefreshWhile.tsx

@@ -1,9 +1,15 @@
 "use client";
 
 import { useRouter } from "next/navigation";
-import { useEffect } from "react";
+import { useEffect, useTransition } from "react";
 
-/** Refreshes the page every few seconds while something runs. */
+/**
+ * Refreshes the page every few seconds while something runs, waiting for
+ * each refresh to land before scheduling the next. A refresh on a timer
+ * cancels one still in flight, so when the page takes longer to render
+ * than the interval (the board against a remote database), no refresh ever
+ * lands and the page never updates (#25).
+ */
 export function RefreshWhile({
   active,
   ms = 2000,
@@ -12,14 +18,17 @@ export function RefreshWhile({
   ms?: number;
 }) {
   const router = useRouter();
+  const [refreshing, startTransition] = useTransition();
   useEffect(() => {
-    if (!active) return;
-    const t = setInterval(() => {
-      router.refresh();
+    if (!active || refreshing) return;
+    const t = setTimeout(() => {
+      startTransition(() => {
+        router.refresh();
+      });
     }, ms);
     return () => {
-      clearInterval(t);
+      clearTimeout(t);
     };
-  }, [active, ms, router]);
+  }, [active, refreshing, ms, router]);
   return null;
 }

package/templates/components/evalkeeper/RollBack.tsx

@@ -0,0 +1,62 @@
+"use client";
+
+import { useState } from "react";
+import { rollBackSettings } from "@/lib/evalkeeper/actions";
+import { Submit } from "./Submit";
+import { Tip } from "./Tip";
+
+/**
+ * Rolls a feature back to an earlier version. The button stays off until
+ * there's a note saying why: the note is kept with the version, for
+ * whoever reads the history next.
+ */
+export function RollBack({
+  suite,
+  versionId,
+  number,
+}: {
+  suite: string;
+  versionId: string;
+  number: number;
+}) {
+  const [reason, setReason] = useState("");
+  const missing = reason.trim() === "";
+  const button = (
+    <Submit variant="quiet" disabled={missing}>
+      Roll back to v{number}
+    </Submit>
+  );
+  return (
+    <form
+      action={rollBackSettings}
+      className="flex flex-wrap items-center gap-2"
+    >
+      <input type="hidden" name="suite" value={suite} />
+      <input type="hidden" name="version" value={versionId} />
+      <label htmlFor={`why-${versionId}`} className="sr-only">
+        Why roll back to v{number}?
+      </label>
+      <input
+        id={`why-${versionId}`}
+        name="reason"
+        required
+        value={reason}
+        onChange={(e) => {
+          setReason(e.target.value);
+        }}
+        placeholder="Why roll back?"
+        className="rounded-md border border-line bg-card px-2 py-1 text-xs focus:border-accent focus:outline-none"
+      />
+      {missing ? (
+        <Tip
+          text="Say why you're rolling back first. The note is kept with the version, for whoever reads the history next."
+          id={`rollback-${versionId}`}
+        >
+          {button}
+        </Tip>
+      ) : (
+        button
+      )}
+    </form>
+  );
+}

package/templates/components/evalkeeper/SettingsHistory.tsx

@@ -5,8 +5,7 @@ import {
   type HistoryEntry,
   type Settings,
 } from "@crowandraven/evalkeeper";
-import { rollBackSettings } from "@/lib/evalkeeper/actions";
-import { Submit } from "./Submit";
+import { RollBack } from "./RollBack";
 import { InfoTip, Tip } from "./Tip";
 import { Chip } from "./ui";
 import { evalsPath } from "@/lib/evalkeeper/paths";
@@ -163,24 +162,11 @@ export function HistoryEntries({
                 </Link>
               )}
               {!isLive && (
-                <form
-                  action={rollBackSettings}
-                  className="flex flex-wrap items-center gap-2"
-                >
-                  <input type="hidden" name="suite" value={suite} />
-                  <input type="hidden" name="version" value={e.to.id} />
-                  <label htmlFor={`why-${e.id}`} className="sr-only">
-                    Why roll back to v{e.to.number}?
-                  </label>
-                  <input
-                    id={`why-${e.id}`}
-                    name="reason"
-                    required
-                    placeholder="Why roll back?"
-                    className="rounded-md border border-line bg-card px-2 py-1 text-xs focus:border-accent focus:outline-none"
-                  />
-                  <Submit variant="quiet">Roll back to v{e.to.number}</Submit>
-                </form>
+                <RollBack
+                  suite={suite}
+                  versionId={e.to.id}
+                  number={e.to.number}
+                />
               )}
             </div>
           </article>

package/templates/components/evalkeeper/Submit.tsx

@@ -3,23 +3,32 @@
 import { LoaderCircle } from "lucide-react";
 import { useFormStatus } from "react-dom";
 
+/**
+ * A button that isn't the page's main action: the app's accent as an
+ * outline, so it still reads as a button in the app's colors (set
+ * `--accent` in evalkeeper.css).
+ */
+export const QUIET_BUTTON =
+  "border border-accent/40 bg-card text-accent hover:bg-accent/10";
+
 /** A submit button that spins while its form's action runs. */
 export function Submit({
   children,
   variant = "primary",
+  disabled = false,
 }: {
   children: React.ReactNode;
   variant?: "primary" | "quiet";
+  /** Off until the form is ready; wrap it in a `<Tip>` saying why. */
+  disabled?: boolean;
 }) {
   const { pending } = useFormStatus();
   return (
     <button
       type="submit"
-      disabled={pending}
+      disabled={pending || disabled}
       className={`inline-flex items-center gap-2 whitespace-nowrap rounded-md px-3 py-1.5 text-sm disabled:opacity-60 ${
-        variant === "primary"
-          ? "bg-accent text-white"
-          : "border border-line bg-card text-ink hover:bg-tint"
+        variant === "primary" ? "bg-accent text-white" : QUIET_BUTTON
       }`}
     >
       {pending && <Spinner />}

package/templates/components/evalkeeper/ui.tsx

@@ -49,7 +49,9 @@ export function Score({ right, total }: { right: number; total: number }) {
         />
       </span>
       <span className="text-xs text-muted tabular-nums">
-        {right}/{total}
+        {/* Whole numbers: a failed answer counts a graded answer's average
+            parts, which can be a fraction. */}
+        {Math.round(right)}/{Math.round(total)}
       </span>
     </div>
   );

package/templates/lib/evalkeeper/actions.ts

@@ -66,6 +66,24 @@ async function evidenceFor(suiteKey: string, model: string) {
   };
 }
 
+/** What a fallback choice records: its current test, if it has one. */
+async function fallbackEvidence(suiteKey: string, model: string) {
+  const board = await runner.board(suiteKey);
+  const row = board.rows.find((r) => r.model === model && !r.variant);
+  if (!row?.canMakeFallback)
+    throw new Error("This model can't be the fallback");
+  const test = row.test && !row.test.stale ? row.test : null;
+  const failed = test?.failedAnswers ? `, ${test.failedAnswers} failed` : "";
+  return {
+    reason: `Made ${row.label} the fallback: ${test ? `${Math.round(test.right)}/${Math.round(test.total)}${failed}` : "not tested"}`,
+    evidence: {
+      runId: test?.runId ?? null,
+      liveRunId: board.liveRunId,
+      score: test ? { right: test.right, total: test.total } : null,
+    },
+  };
+}
+
 const VS: Record<string, string> = {
   BETTER: "more accurate than live",
   SAME: "about as accurate as live",
@@ -94,15 +112,11 @@ export async function makeFallback(form: FormData) {
   const admin = await requireAdmin();
   const suite = suiteFrom(form);
   const model = formText(form, "model");
-  const { row, evidence } = await evidenceFor(suite.key, model);
+  const { reason, evidence } = await fallbackEvidence(suite.key, model);
   await settings.update(
     suite.operation,
     { fallbackModel: model },
-    {
-      by: admin.name,
-      reason: `Made ${row.label} the fallback: ${evidence.score.right}/${evidence.score.total}, from another provider`,
-      evidence,
-    },
+    { by: admin.name, reason, evidence },
   );
   revalidatePath(EVALS_BASE, "layout");
 }