@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/templates/components/evalkeeper/AnswerKey.tsx

@@ -0,0 +1,360 @@
+import {
+  allPassed,
+  type CaseRecord,
+  type GlossaryTerm,
+  type OutputRecord,
+  type AnySuite,
+  type BoardRow,
+  type CheckResult,
+} from "@crowandraven/evalkeeper";
+import {
+  addCase,
+  confirmAnswer,
+  editAnswer,
+  retireCase,
+} from "@/lib/evalkeeper/actions";
+import { Check, ChevronRight, CircleAlert, X } from "lucide-react";
+import { store } from "@/lib/evalkeeper";
+import { FieldValues, Fields } from "./Fields";
+import { AddCaseHeader } from "./AddCase";
+import { CaseMenu } from "./CaseMenu";
+import { Submit } from "./Submit";
+import { Chip, type ChipTone } from "./ui";
+import { InfoTip, Tip } from "./Tip";
+
+/** A suite's cases in use, with each case's answers newest first. */
+async function loadCases(suite: string): Promise<CaseRecord[]> {
+  return (await store.listCases(suite)).map((c) => ({
+    ...c,
+    labels: [...c.labels].sort((a, b) => +b.updatedAt - +a.updatedAt),
+  }));
+}
+
+const STATUS: Record<
+  string,
+  { text: string; tone: ChipTone; term: GlossaryTerm }
+> = {
+  CONFIRMED: { text: "Confirmed", tone: "live", term: "confirmed" },
+  PROVISIONAL: { text: "Provisional", tone: "warn", term: "provisional" },
+  PENDING: { text: "Pending", tone: "warn", term: "pending" },
+};
+
+/**
+ * The answer key: every case in use, each opening to its input, its
+ * answer, and how each model's latest test did on it, with forms to add a
+ * case, edit an answer, or retire a case.
+ */
+export async function AnswerKey({
+  suite,
+  rows,
+  heading,
+  tools,
+}: {
+  suite: AnySuite;
+  rows: BoardRow[];
+  /** The section's heading, with "Add a case" across from it. */
+  heading: React.ReactNode;
+  /** More buttons beside "Add a case" (rescore). */
+  tools?: React.ReactNode;
+}) {
+  const cases = await loadCases(suite.key);
+  // Each model's latest test (at the operation's setting), by case.
+  const modelByRun = new Map(
+    rows.flatMap((r) =>
+      r.test && !r.variant ? [[r.test.runId, r.label] as const] : [],
+    ),
+  );
+  const outputs = (
+    await Promise.all([...modelByRun.keys()].map((id) => store.listOutputs(id)))
+  ).flat();
+  const byCase = new Map<string, { model: string; output: OutputRecord }[]>();
+  for (const o of outputs) {
+    byCase.set(o.caseId, [
+      ...(byCase.get(o.caseId) ?? []),
+      { model: modelByRun.get(o.runId) ?? o.runId, output: o },
+    ]);
+  }
+  const fields = suite.fields;
+
+  return (
+    <div className="space-y-4">
+      <AddCaseHeader
+        heading={heading}
+        tools={tools}
+        form={
+          fields &&
+          suite.labelFromForm && (
+            <form action={addCase} className="space-y-4">
+              <input type="hidden" name="suite" value={suite.key} />
+              <Fields fields={fields.input} prefix="new-" />
+              <p className="readable text-sm text-muted">
+                What a good answer is. You're writing it, so it's confirmed.
+              </p>
+              <Fields fields={fields.label} prefix="new-" />
+              <div className="grid gap-4 sm:grid-cols-2">
+                <div className="space-y-1">
+                  <label
+                    htmlFor="new-tags"
+                    className="block text-sm font-medium"
+                  >
+                    Tags
+                    <InfoTip term="tags" id="new-tags" />
+                  </label>
+                  <input
+                    id="new-tags"
+                    name="tags"
+                    placeholder="e.g. billing, money now"
+                    className="w-full rounded-md border border-line bg-card px-3 py-2 text-sm focus:border-accent focus:outline-none"
+                  />
+                  <p className="text-xs text-muted">
+                    What this case tests, comma-separated. The quick set picks
+                    cases to cover the most tags.
+                  </p>
+                </div>
+                <div className="space-y-1">
+                  <label
+                    htmlFor="new-split"
+                    className="block text-sm font-medium"
+                  >
+                    Set
+                    <InfoTip
+                      text="DEV: cases you tune against. TEST: cases you only check, never tune against, so they show how a change does on cases it wasn't fitted to."
+                      id="new-split"
+                    />
+                  </label>
+                  <select
+                    id="new-split"
+                    name="split"
+                    defaultValue="DEV"
+                    className="w-full rounded-md border border-line bg-card px-3 py-2 text-sm"
+                  >
+                    <option value="DEV">DEV: tune against it</option>
+                    <option value="TEST">
+                      TEST: check only, never tune against
+                    </option>
+                  </select>
+                </div>
+              </div>
+              <div className="flex gap-2">
+                <Submit>Add case</Submit>
+              </div>
+            </form>
+          )
+        }
+      />
+
+      <div className="space-y-2">
+        {cases.map((c) => (
+          <CaseItem
+            key={c.id}
+            suite={suite}
+            c={c}
+            results={byCase.get(c.id) ?? []}
+          />
+        ))}
+        {!cases.length && <p className="text-sm text-muted">No cases yet.</p>}
+      </div>
+    </div>
+  );
+}
+
+function CaseItem({
+  suite,
+  c,
+  results,
+}: {
+  suite: AnySuite;
+  c: CaseRecord;
+  results: {
+    model: string;
+    output: { scores: unknown; error: string | null };
+  }[];
+}) {
+  const answer =
+    c.labels.find((l) => l.status === "CONFIRMED") ??
+    c.labels.find((l) => l.status === "PROVISIONAL") ??
+    c.labels.find((l) => l.status === "PENDING");
+  const status = (answer && STATUS[answer.status]) ?? {
+    text: "No answer",
+    tone: "warn" as const,
+    term: "needsAnswer" as const,
+  };
+  const input = suite.input.parse(c.input);
+  const right = results.filter(
+    (r) =>
+      !r.output.error && allPassed(r.output.scores as CheckResult[] | null),
+  ).length;
+  const label = (key: string) =>
+    suite.checks.find((x) => x.key === key)?.label ?? key;
+  const tags = Array.isArray(c.tags) ? c.tags : [];
+
+  return (
+    <details className="group rounded-md border border-line bg-card">
+      <summary className="flex cursor-pointer list-none flex-wrap items-center justify-between gap-2 p-3 text-sm">
+        <span className="font-medium">
+          <ChevronRight
+            aria-hidden
+            className="mr-1.5 inline-block h-4 w-4 align-[-3px] text-muted transition-transform group-open:rotate-90"
+          />
+          {suite.title(input)}
+        </span>
+        <span className="flex items-center gap-1.5">
+          <Tip term={status.term} id={`status-${c.id}`}>
+            <Chip tone={status.tone}>{status.text}</Chip>
+          </Tip>
+          <Tip term={c.split === "TEST" ? "test" : "dev"} id={`split-${c.id}`}>
+            <Chip tone="neutral">{c.split === "TEST" ? "Test" : "Dev"}</Chip>
+          </Tip>
+          {results.length > 0 && (
+            <Tip
+              text={`Right on ${right} of ${results.length} models' latest tests.`}
+              id={`right-${c.id}`}
+              align="end"
+            >
+              <span
+                className={`ml-1 inline-flex items-center gap-1 text-xs tabular-nums ${right === results.length ? "text-good" : "text-muted"}`}
+              >
+                <Check aria-hidden className="h-3.5 w-3.5" />
+                {right}/{results.length}
+              </span>
+            </Tip>
+          )}
+        </span>
+      </summary>
+
+      <div className="relative space-y-5 px-3 pb-4 text-sm">
+        <div className="grid gap-5 md:grid-cols-2">
+          <section className="space-y-2">
+            <h4 className="text-xs font-semibold uppercase tracking-wide text-muted">
+              Input
+            </h4>
+            {suite.fields ? (
+              <FieldValues
+                fields={suite.fields.input}
+                values={c.input}
+                prefix={`${c.id}-in-`}
+              />
+            ) : (
+              <pre className="whitespace-pre-wrap text-xs">
+                {suite.describe?.(input) ?? JSON.stringify(c.input, null, 2)}
+              </pre>
+            )}
+          </section>
+          <section className="space-y-2">
+            <h4 className="text-xs font-semibold uppercase tracking-wide text-muted">
+              Answer
+            </h4>
+            {answer && suite.fields ? (
+              <FieldValues
+                fields={suite.fields.label}
+                values={answer.value}
+                prefix={`${c.id}-ans-`}
+              />
+            ) : (
+              <pre className="whitespace-pre-wrap text-xs">
+                {JSON.stringify(answer?.value ?? null, null, 2)}
+              </pre>
+            )}
+            {(tags.length > 0 || c.note) && (
+              <p className="readable text-xs text-muted">
+                {tags.length > 0 && `Tags: ${tags.join(", ")}. `}
+                {c.note}
+              </p>
+            )}
+          </section>
+        </div>
+
+        {results.length > 0 && (
+          <section className="space-y-2">
+            <h4 className="text-xs font-semibold uppercase tracking-wide text-muted">
+              Each model's latest test
+            </h4>
+            <ul className="space-y-1">
+              {results.map(({ model, output }) => {
+                const checks = (output.scores as CheckResult[] | null) ?? null;
+                const failed = checks?.filter((x) => x.pass === false) ?? [];
+                return (
+                  <li key={model} className="flex items-start gap-1.5">
+                    {output.error ? (
+                      <CircleAlert
+                        aria-label="No answer"
+                        className="mt-0.5 h-4 w-4 shrink-0 text-bad"
+                      />
+                    ) : failed.length ? (
+                      <X
+                        aria-label="Wrong"
+                        className="mt-0.5 h-4 w-4 shrink-0 text-warn"
+                      />
+                    ) : (
+                      <Check
+                        aria-label="Right"
+                        className="mt-0.5 h-4 w-4 shrink-0 text-good"
+                      />
+                    )}
+                    <span>
+                      <span className="font-medium">{model}</span>
+                      {output.error && (
+                        <span className="text-muted">: {output.error}</span>
+                      )}
+                      {failed.map((x) => (
+                        <span key={x.key} className="text-muted">
+                          {" "}
+                          · {label(x.key)}
+                          {x.note ? `: ${x.note}` : ""}
+                        </span>
+                      ))}
+                    </span>
+                  </li>
+                );
+              })}
+            </ul>
+          </section>
+        )}
+
+        {answer?.status === "PENDING" && (
+          <form action={confirmAnswer}>
+            <input type="hidden" name="suite" value={suite.key} />
+            <input type="hidden" name="label" value={answer.id} />
+            <Submit>Confirm this answer</Submit>
+          </form>
+        )}
+
+        <CaseMenu
+          suite={suite.key}
+          caseId={c.id}
+          retire={retireCase}
+          startEditing={!answer}
+          editLabel={
+            suite.fields && suite.labelFromForm
+              ? answer
+                ? "Edit answer"
+                : "Write the answer"
+              : null
+          }
+          edit={
+            suite.fields && (
+              <form
+                key={answer?.updatedAt.toISOString() ?? "none"}
+                action={editAnswer}
+                className="space-y-4"
+              >
+                <input type="hidden" name="suite" value={suite.key} />
+                <input type="hidden" name="case" value={c.id} />
+                <Fields
+                  fields={suite.fields.label}
+                  values={answer?.value}
+                  prefix={`${c.id}-`}
+                />
+                <p className="readable text-xs text-muted">
+                  Saves a new confirmed answer and keeps the old one in the
+                  history. Tests are rescored for free.
+                </p>
+                <Submit>Save answer</Submit>
+              </form>
+            )
+          }
+        />
+      </div>
+    </details>
+  );
+}

package/templates/components/evalkeeper/CaseMenu.tsx

@@ -0,0 +1,122 @@
+"use client";
+
+import { useEffect, useRef, useState } from "react";
+import { EllipsisVertical } from "lucide-react";
+
+/**
+ * A case's menu, in the top right of the open case: edit its answer (the
+ * form opens below the menu's row) and retire it. Closes on a click
+ * outside or Escape.
+ */
+export function CaseMenu({
+  suite,
+  caseId,
+  editLabel,
+  edit,
+  retire,
+  startEditing = false,
+}: {
+  suite: string;
+  caseId: string;
+  /** "Edit answer" or "Write the answer"; null when there's no form. */
+  editLabel: string | null;
+  /** The answer form, shown when someone picks editLabel. */
+  edit: React.ReactNode;
+  /** Your server action that retires a case (gets suite and case). */
+  retire: (form: FormData) => Promise<void>;
+  /** Open the form from the start, for a case with no answer yet. */
+  startEditing?: boolean;
+}) {
+  const [open, setOpen] = useState(false);
+  const [editing, setEditing] = useState(startEditing);
+  const ref = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (!open) return;
+    const close = (e: MouseEvent | KeyboardEvent) => {
+      if (
+        e instanceof KeyboardEvent
+          ? e.key === "Escape"
+          : !ref.current?.contains(e.target as Node)
+      )
+        setOpen(false);
+    };
+    document.addEventListener("mousedown", close);
+    document.addEventListener("keydown", close);
+    return () => {
+      document.removeEventListener("mousedown", close);
+      document.removeEventListener("keydown", close);
+    };
+  }, [open]);
+
+  const item =
+    "block w-full rounded px-3 py-1.5 text-left text-sm hover:bg-tint";
+
+  return (
+    <>
+      <div ref={ref} className="absolute right-3 top-0 z-10">
+        <button
+          type="button"
+          aria-label="Case actions"
+          title="Case actions"
+          aria-haspopup="menu"
+          aria-expanded={open}
+          onClick={() => {
+            setOpen(!open);
+          }}
+          className="inline-flex h-8 w-8 items-center justify-center rounded-md text-muted hover:bg-tint hover:text-ink"
+        >
+          <EllipsisVertical aria-hidden className="h-4 w-4" />
+        </button>
+        {open && (
+          <div
+            role="menu"
+            className="absolute right-0 mt-1 w-44 rounded-md border border-line bg-card p-1 shadow-lg"
+          >
+            {editLabel && (
+              <button
+                type="button"
+                role="menuitem"
+                className={item}
+                onClick={() => {
+                  setEditing(true);
+                  setOpen(false);
+                }}
+              >
+                {editLabel}
+              </button>
+            )}
+            <form action={retire}>
+              <input type="hidden" name="suite" value={suite} />
+              <input type="hidden" name="case" value={caseId} />
+              <button
+                type="submit"
+                role="menuitem"
+                className={`${item} text-bad`}
+              >
+                Retire case
+              </button>
+            </form>
+          </div>
+        )}
+      </div>
+      {editing && (
+        <div className="space-y-3 rounded-md bg-tint p-3">
+          <div className="flex items-center justify-between gap-2">
+            <h4 className="text-sm font-medium">{editLabel}</h4>
+            <button
+              type="button"
+              onClick={() => {
+                setEditing(false);
+              }}
+              className="text-sm text-muted hover:text-ink"
+            >
+              Cancel
+            </button>
+          </div>
+          {edit}
+        </div>
+      )}
+    </>
+  );
+}

package/templates/components/evalkeeper/Fields.tsx

@@ -0,0 +1,110 @@
+import { asText, fieldValue, type FormField } from "@crowandraven/evalkeeper";
+import { InfoTip } from "./Tip";
+
+/**
+ * A suite's form fields, rendered from its field specs: label, input, and
+ * help in one wrapper. `values` fills them from a stored input or answer.
+ */
+export function Fields({
+  fields,
+  values,
+  prefix = "",
+}: {
+  fields: FormField[];
+  values?: unknown;
+  /** Keeps ids unique when a page has several forms. */
+  prefix?: string;
+}) {
+  return (
+    <>
+      {fields.map((f) => {
+        const id = `${prefix}${f.name}`;
+        const value = values === undefined ? "" : fieldValue(f, values);
+        const box =
+          "w-full rounded-md border border-line bg-card px-3 py-2 text-sm focus:border-accent focus:outline-none";
+        return (
+          <div key={f.name} className="space-y-1">
+            <label htmlFor={id} className="block text-sm font-medium">
+              {f.label}
+            </label>
+            {f.kind === "select" ? (
+              <select
+                id={id}
+                name={f.name}
+                defaultValue={value || f.options?.[0]}
+                className={box}
+              >
+                {f.options?.map((o) => (
+                  <option key={o} value={o}>
+                    {o.replace(/_/g, " ")}
+                  </option>
+                ))}
+              </select>
+            ) : f.kind === "text" ? (
+              <input
+                id={id}
+                name={f.name}
+                defaultValue={value}
+                required={f.required}
+                className={box}
+              />
+            ) : (
+              <textarea
+                id={id}
+                name={f.name}
+                defaultValue={value}
+                required={f.required}
+                rows={f.kind === "lines" ? 4 : 5}
+                className={box}
+              />
+            )}
+            <p className="readable text-xs text-muted">{f.help}</p>
+          </div>
+        );
+      })}
+    </>
+  );
+}
+
+/** A stored input or answer, read with the suite's field labels. */
+export function FieldValues({
+  fields,
+  values,
+  prefix = "",
+}: {
+  fields: FormField[];
+  values: unknown;
+  /** Keeps tooltip ids unique when a page shows several. */
+  prefix?: string;
+}) {
+  return (
+    <dl className="space-y-2">
+      {fields.map((f) => {
+        const v = (values as Record<string, unknown> | null)?.[f.name];
+        return (
+          <div key={f.name}>
+            <dt className="text-xs text-muted">
+              {f.label}
+              <InfoTip text={f.help} id={`${prefix}${f.name}-read`} />
+            </dt>
+            <dd className="readable whitespace-pre-wrap">
+              {Array.isArray(v) ? (
+                v.length ? (
+                  <ul className="list-disc pl-5">
+                    {v.map((x, i) => (
+                      <li key={i}>{asText(x)}</li>
+                    ))}
+                  </ul>
+                ) : (
+                  <span className="text-muted">None</span>
+                )
+              ) : (
+                (asText(v) || "—").replace(/_/g, " ")
+              )}
+            </dd>
+          </div>
+        );
+      })}
+    </dl>
+  );
+}

package/templates/components/evalkeeper/RefreshWhile.tsx

@@ -0,0 +1,25 @@
+"use client";
+
+import { useRouter } from "next/navigation";
+import { useEffect } from "react";
+
+/** Refreshes the page every few seconds while something runs. */
+export function RefreshWhile({
+  active,
+  ms = 2000,
+}: {
+  active: boolean;
+  ms?: number;
+}) {
+  const router = useRouter();
+  useEffect(() => {
+    if (!active) return;
+    const t = setInterval(() => {
+      router.refresh();
+    }, ms);
+    return () => {
+      clearInterval(t);
+    };
+  }, [active, ms, router]);
+  return null;
+}