@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/templates/components/evalkeeper/SettingsHistory.tsx

@@ -0,0 +1,191 @@
+import Link from "next/link";
+import {
+  describeChange,
+  type ChangeKind,
+  type HistoryEntry,
+  type Settings,
+} from "@crowandraven/evalkeeper";
+import { rollBackSettings } from "@/lib/evalkeeper/actions";
+import { Submit } from "./Submit";
+import { InfoTip, Tip } from "./Tip";
+import { Chip } from "./ui";
+import { evalsPath } from "@/lib/evalkeeper/paths";
+
+const KIND: Record<
+  ChangeKind,
+  { text: string; tone: "live" | "fallback" | "warn" | "neutral"; help: string }
+> = {
+  CHANGE: {
+    text: "Changed",
+    tone: "neutral",
+    help: "A person changed a setting.",
+  },
+  PROMOTE: {
+    text: "Promoted",
+    tone: "live",
+    help: "Made live or the fallback from a test, with the test as evidence.",
+  },
+  ROLLBACK: {
+    text: "Rolled back",
+    tone: "warn",
+    help: "An earlier version made live again.",
+  },
+  DEPLOY: {
+    text: "Deployed",
+    tone: "fallback",
+    help: "The instructions changed in code under the live settings.",
+  },
+};
+
+type Evidence = { runId?: string | null; live?: string | null };
+
+/**
+ * A feature's settings history: every switch, newest first, with what
+ * changed, who, why, and the evidence; any earlier version can be made
+ * live again, with a reason.
+ */
+export function SettingsHistory({
+  suite,
+  entries,
+  liveId,
+  label,
+  defaults,
+}: {
+  suite: string;
+  entries: HistoryEntry[];
+  liveId: string | null;
+  label: (model: string) => string;
+  /** The settings in code, shown as v1 until the first change. */
+  defaults: Settings;
+}) {
+  return (
+    <section className="space-y-3" aria-labelledby="history">
+      <h2 id="history" className="text-lg font-semibold">
+        Versions
+        <InfoTip term="versions" />
+      </h2>
+      <HistoryEntries
+        suite={suite}
+        entries={entries}
+        liveId={liveId}
+        label={label}
+        defaults={defaults}
+      />
+    </section>
+  );
+}
+
+/**
+ * The versions in a settings history, newest first, each with a rollback
+ * unless it's live. For a page's history section or a card's history toggle.
+ */
+export function HistoryEntries({
+  suite,
+  entries,
+  liveId,
+  label,
+  defaults,
+  plain = false,
+}: {
+  suite: string;
+  entries: HistoryEntry[];
+  liveId: string | null;
+  label: (model: string) => string;
+  /** The settings in code, shown as v1 until the first change. */
+  defaults: Settings;
+  /** No tint behind each version, for a list already on a tint. */
+  plain?: boolean;
+}) {
+  const box = `space-y-2 text-sm ${plain ? "py-2" : "rounded-md border border-line bg-card p-3"}`;
+  if (entries.length === 0)
+    return (
+      <article aria-label="Version 1: Defaults" className={box}>
+        <span className="flex flex-wrap items-center gap-2">
+          <span className="font-medium">v1</span>
+          <Chip tone="neutral">Defaults</Chip>
+          <Chip tone="live">Live now</Chip>
+        </span>
+        <p>
+          {label(defaults.model)} live
+          {defaults.fallbackModel
+            ? ` · fallback ${label(defaults.fallbackModel)}`
+            : " · no fallback"}
+        </p>
+        <p className="readable text-muted">
+          The settings in code. The first change records them as v1, so
+          there&apos;s always something to roll back to.
+        </p>
+      </article>
+    );
+  return (
+    <div className="space-y-2">
+      {entries.map((e) => {
+        const kind = KIND[e.kind];
+        const ev = (e.evidence ?? {}) as Evidence;
+        const testId = ev.runId ?? ev.live ?? null;
+        const isLive = e.to.id === liveId;
+        return (
+          <article
+            key={e.id}
+            aria-label={`Version ${e.to.number}: ${kind.text}`}
+            className={box}
+          >
+            <div className="flex flex-wrap items-center justify-between gap-2">
+              <span className="flex flex-wrap items-center gap-2">
+                <span className="font-medium">v{e.to.number}</span>
+                <Tip text={kind.help} id={`kind-${e.id}`}>
+                  <Chip tone={kind.tone}>{kind.text}</Chip>
+                </Tip>
+                {isLive && <Chip tone="live">Live now</Chip>}
+              </span>
+              <span className="text-xs text-muted">
+                {e.by ?? "System"} · {e.createdAt.toLocaleString()}
+              </span>
+            </div>
+            <ul className="readable list-disc pl-5">
+              {(e.kind === "ROLLBACK"
+                ? [`Back to v${e.to.number}: ${label(e.to.model)} live`]
+                : e.kind === "DEPLOY"
+                  ? ["Same models, new instructions"]
+                  : describeChange(e.from, e.to, label)
+              ).map((line) => (
+                <li key={line}>{line}</li>
+              ))}
+            </ul>
+            {e.reason && <p className="readable text-muted">{e.reason}</p>}
+            <div className="flex flex-wrap items-center gap-3">
+              {testId && (
+                <Link
+                  href={evalsPath(suite, "runs", testId)}
+                  className="text-xs underline"
+                >
+                  The test behind it
+                </Link>
+              )}
+              {!isLive && (
+                <form
+                  action={rollBackSettings}
+                  className="flex flex-wrap items-center gap-2"
+                >
+                  <input type="hidden" name="suite" value={suite} />
+                  <input type="hidden" name="version" value={e.to.id} />
+                  <label htmlFor={`why-${e.id}`} className="sr-only">
+                    Why roll back to v{e.to.number}?
+                  </label>
+                  <input
+                    id={`why-${e.id}`}
+                    name="reason"
+                    required
+                    placeholder="Why roll back?"
+                    className="rounded-md border border-line bg-card px-2 py-1 text-xs focus:border-accent focus:outline-none"
+                  />
+                  <Submit variant="quiet">Roll back to v{e.to.number}</Submit>
+                </form>
+              )}
+            </div>
+          </article>
+        );
+      })}
+    </div>
+  );
+}

package/templates/components/evalkeeper/Submit.tsx

@@ -0,0 +1,58 @@
+"use client";
+
+import { LoaderCircle } from "lucide-react";
+import { useFormStatus } from "react-dom";
+
+/** A submit button that spins while its form's action runs. */
+export function Submit({
+  children,
+  variant = "primary",
+}: {
+  children: React.ReactNode;
+  variant?: "primary" | "quiet";
+}) {
+  const { pending } = useFormStatus();
+  return (
+    <button
+      type="submit"
+      disabled={pending}
+      className={`inline-flex items-center gap-2 whitespace-nowrap rounded-md px-3 py-1.5 text-sm disabled:opacity-60 ${
+        variant === "primary"
+          ? "bg-accent text-white"
+          : "border border-line bg-card text-ink hover:bg-tint"
+      }`}
+    >
+      {pending && <Spinner />}
+      {children}
+    </button>
+  );
+}
+
+/**
+ * An icon-only submit button. `label` is its accessible name; wrap it in a
+ * `<Tip control>` for the tooltip. The icon turns into the loader while the
+ * action runs.
+ */
+export function IconSubmit({
+  label,
+  icon,
+}: {
+  label: string;
+  icon: React.ReactNode;
+}) {
+  const { pending } = useFormStatus();
+  return (
+    <button
+      type="submit"
+      aria-label={label}
+      disabled={pending}
+      className="inline-flex h-8 w-8 items-center justify-center rounded-md border border-line bg-card text-ink hover:bg-tint disabled:opacity-60"
+    >
+      {pending ? <Spinner /> : icon}
+    </button>
+  );
+}
+
+export function Spinner() {
+  return <LoaderCircle aria-hidden className="h-3.5 w-3.5 animate-spin" />;
+}

package/templates/components/evalkeeper/Thumbs.tsx

@@ -0,0 +1,117 @@
+"use client";
+
+import { useState, useTransition } from "react";
+import { ThumbsDown, ThumbsUp } from "lucide-react";
+import { formText } from "@crowandraven/evalkeeper";
+
+type Rating = "UP" | "DOWN";
+
+/**
+ * Thumbs up or down on one AI answer, with "what was wrong?" after a
+ * thumbs down. Clicking the same thumb again takes the rating back.
+ * `action` is your server action; it gets targetId, rating, and reason (or
+ * clear=1), and looks the answer up itself.
+ */
+export function Thumbs({
+  targetId,
+  initial = null,
+  action,
+}: {
+  targetId: string;
+  initial?: Rating | null;
+  action: (form: FormData) => Promise<void>;
+}) {
+  const [rating, setRating] = useState<Rating | null>(initial);
+  const [asking, setAsking] = useState(false);
+  const [sent, setSent] = useState(false);
+  const [pending, start] = useTransition();
+
+  const send = (fields: Record<string, string>, after: () => void) => {
+    const form = new FormData();
+    form.set("targetId", targetId);
+    for (const [k, v] of Object.entries(fields)) form.set(k, v);
+    start(async () => {
+      await action(form);
+      after();
+    });
+  };
+
+  const thumb = (value: Rating) => {
+    const on = rating === value;
+    const label = on
+      ? "Take back your rating"
+      : value === "UP"
+        ? "Good answer"
+        : "Not a good answer";
+    return (
+      <button
+        type="button"
+        aria-label={label}
+        title={label}
+        aria-pressed={on}
+        disabled={pending}
+        onClick={() => {
+          send(on ? { rating: value, clear: "1" } : { rating: value }, () => {
+            setRating(on ? null : value);
+            setAsking(!on && value === "DOWN");
+            setSent(false);
+          });
+        }}
+        className={`inline-flex h-8 w-8 items-center justify-center rounded-md border text-sm disabled:opacity-60 ${
+          on
+            ? "border-accent bg-tint text-accent"
+            : "border-line bg-card text-muted hover:text-ink"
+        }`}
+      >
+        {value === "UP" ? (
+          <ThumbsUp aria-hidden className="h-4 w-4" />
+        ) : (
+          <ThumbsDown aria-hidden className="h-4 w-4" />
+        )}
+      </button>
+    );
+  };
+
+  return (
+    <div className="space-y-2">
+      <div className="flex items-center gap-1">
+        {thumb("UP")}
+        {thumb("DOWN")}
+        {sent && (
+          <span className="ml-2 text-sm text-muted">Thanks: that helps.</span>
+        )}
+      </div>
+      {asking && (
+        <form
+          className="flex flex-wrap items-end gap-2"
+          onSubmit={(e) => {
+            e.preventDefault();
+            const reason = formText(new FormData(e.currentTarget), "reason");
+            send({ rating: "DOWN", reason }, () => {
+              setAsking(false);
+              setSent(true);
+            });
+          }}
+        >
+          <label htmlFor={`why-${targetId}`} className="sr-only">
+            What was wrong?
+          </label>
+          <input
+            id={`why-${targetId}`}
+            name="reason"
+            maxLength={1000}
+            placeholder="What was wrong? (optional)"
+            className="min-w-0 flex-1 rounded-md border border-line bg-card px-3 py-1.5 text-sm focus:border-accent focus:outline-none"
+          />
+          <button
+            type="submit"
+            disabled={pending}
+            className="rounded-md border border-line bg-card px-3 py-1.5 text-sm hover:bg-tint disabled:opacity-60"
+          >
+            Send
+          </button>
+        </form>
+      )}
+    </div>
+  );
+}

package/templates/components/evalkeeper/Tip.tsx

@@ -0,0 +1,98 @@
+import { Info } from "lucide-react";
+import { GLOSSARY, type GlossaryTerm } from "@crowandraven/evalkeeper";
+
+/**
+ * Tooltips that explain how things work. They open on hover and on
+ * keyboard focus, and screen readers read the text as the element's
+ * description. No JavaScript: CSS shows them. The named group (group/tip)
+ * keeps a tooltip from opening when an outer "group" (an open case) is
+ * hovered.
+ *
+ *   <InfoTip term="score" />           an (i) after a label
+ *   <Tip term="live"><Chip …/></Tip>   explains the thing it wraps
+ *   <Tip text="…" control><button …/></Tip>   a button's tooltip
+ */
+
+type Props = {
+  /** A glossary entry, or your own text with `text`. */
+  term?: GlossaryTerm;
+  text?: string;
+  /** Unique per page when the same term shows more than once. */
+  id?: string;
+  /**
+   * Where it opens. "bottom" inside a scrolling container (a table), whose
+   * top edge would cut it off.
+   */
+  side?: "top" | "bottom";
+  /** "end" near a container's right edge, so it opens leftward. */
+  align?: "center" | "end";
+};
+
+function body(props: Props) {
+  return props.text ?? (props.term ? GLOSSARY[props.term] : "");
+}
+
+function Bubble({
+  id,
+  text,
+  side = "top",
+  align = "center",
+}: {
+  id: string;
+  text: string;
+  side?: "top" | "bottom" | undefined;
+  align?: "center" | "end" | undefined;
+}) {
+  return (
+    <span
+      role="tooltip"
+      id={id}
+      className={`pointer-events-none invisible absolute ${side === "top" ? "bottom-full mb-2" : "top-full mt-2"} ${align === "end" ? "right-0" : "left-1/2 -translate-x-1/2"} z-20 w-64 rounded-md border border-line bg-card px-3 py-2 text-left text-xs font-normal normal-case leading-snug tracking-normal text-ink opacity-0 shadow-lg transition-opacity group-focus-within/tip:visible group-focus-within/tip:opacity-100 group-hover/tip:visible group-hover/tip:opacity-100`}
+    >
+      {text}
+    </span>
+  );
+}
+
+/** A small (i) button that explains the label before it. */
+export function InfoTip(props: Props) {
+  const text = body(props);
+  const id = `tip-${props.id ?? props.term ?? "info"}`;
+  return (
+    <span className="group/tip relative ml-1 inline-flex align-middle">
+      <button
+        type="button"
+        aria-describedby={id}
+        aria-label="What's this?"
+        className="inline-flex h-4 w-4 items-center justify-center rounded-full text-muted hover:text-ink"
+      >
+        <Info aria-hidden className="h-3.5 w-3.5" />
+      </button>
+      <Bubble id={id} text={text} side={props.side} align={props.align} />
+    </span>
+  );
+}
+
+/** Explains whatever it wraps (a chip, a number) on hover and focus. */
+export function Tip(
+  props: Props & {
+    children: React.ReactNode;
+    /**
+     * The child is a control that takes focus itself (a button), so the
+     * wrapper isn't a second tab stop.
+     */
+    control?: boolean;
+  },
+) {
+  const text = body(props);
+  const id = `tip-${props.id ?? props.term ?? "info"}`;
+  return (
+    <span
+      className="group/tip relative inline-flex"
+      {...(props.control ? {} : { tabIndex: 0, "aria-describedby": id })}
+    >
+      {props.children}
+      <Bubble id={id} text={text} side={props.side} align={props.align} />
+    </span>
+  );
+}

package/templates/components/evalkeeper/WeeklyCheck.tsx

@@ -0,0 +1,124 @@
+import Link from "next/link";
+import { money, type WeeklySummary } from "@crowandraven/evalkeeper";
+import { runWeeklyNow } from "@/lib/evalkeeper/actions";
+import { RefreshWhile } from "./RefreshWhile";
+import { Submit } from "./Submit";
+import { InfoTip } from "./Tip";
+import { Card, Chip, pct } from "./ui";
+import { evalsPath } from "@/lib/evalkeeper/paths";
+
+/**
+ * The weekly check on the evals hub: when it last ran, what it found, and
+ * a button to run it now. It only reports; a person decides what to change.
+ */
+export function WeeklyCheck({ weekly }: { weekly: WeeklySummary }) {
+  return (
+    <section aria-labelledby="weekly" className="space-y-3">
+      <RefreshWhile active={weekly.running > 0} />
+      <div className="flex flex-wrap items-center justify-between gap-4">
+        <h2 id="weekly" className="text-lg font-semibold">
+          Weekly check
+          <InfoTip term="weeklyCheck" />
+        </h2>
+        <form action={runWeeklyNow}>
+          <Submit variant="quiet">Run it now</Submit>
+        </form>
+      </div>
+      <Card className="space-y-4">
+        <p className="readable text-sm text-muted">
+          {weekly.running > 0
+            ? `Running now: ${weekly.running} quick set${weekly.running === 1 ? "" : "s"} left. `
+            : weekly.lastAt
+              ? `Last run ${weekly.lastAt.toLocaleDateString("en-US", { month: "long", day: "numeric" })}, ${money(weekly.costUsd)}. `
+              : "Not run yet. "}
+          Every Monday, each live model answers its suite&apos;s quick set
+          again, so a quiet change by its provider shows up here. It also flags
+          models retiring soon and features people rate poorly.
+        </p>
+
+        {weekly.clear && (
+          <p className="readable text-sm text-good">
+            All clear: no drift, no retirements within 60 days, and no feature
+            with a high thumbs-down share.
+          </p>
+        )}
+
+        {!weekly.clear && weekly.lastAt && (
+          <ul className="readable space-y-2 text-sm">
+            {weekly.drift.map((d) => (
+              <li
+                key={`drift-${d.suite}`}
+                className="flex flex-wrap items-baseline gap-2"
+              >
+                <Chip tone="serious">Drift</Chip>
+                <span>
+                  <Link
+                    href={evalsPath(d.suite)}
+                    className="font-medium underline"
+                  >
+                    {d.label}
+                  </Link>{" "}
+                  on {d.model} fell from {pct(d.previous)} to {pct(d.latest)} on
+                  its quick set. Test every model there to see if it&apos;s
+                  real.
+                </span>
+              </li>
+            ))}
+            {weekly.failed.map((f) => (
+              <li
+                key={`failed-${f.suite}`}
+                className="flex flex-wrap items-baseline gap-2"
+              >
+                <Chip tone="serious">Didn&apos;t finish</Chip>
+                <span>
+                  <Link
+                    href={evalsPath(f.suite)}
+                    className="font-medium underline"
+                  >
+                    {f.label}
+                  </Link>
+                  : <span className="text-muted">{f.error}</span>
+                </span>
+              </li>
+            ))}
+            {weekly.retiring.map((r) => (
+              <li
+                key={`retiring-${r.suite}-${r.role}`}
+                className="flex flex-wrap items-baseline gap-2"
+              >
+                <Chip tone="warn">Retiring</Chip>
+                <span>
+                  {r.model}, the {r.role} model for{" "}
+                  <Link
+                    href={evalsPath(r.suite)}
+                    className="font-medium underline"
+                  >
+                    {r.label}
+                  </Link>
+                  , retires {r.when}
+                  {r.days < 0 ? " (already)" : ` (in ${r.days} days)`}. Pick
+                  another on its suite page.
+                </span>
+              </li>
+            ))}
+            {weekly.thumbs.map((t) => (
+              <li
+                key={`thumbs-${t.operation}`}
+                className="flex flex-wrap items-baseline gap-2"
+              >
+                <Chip tone="warn">Thumbs down</Chip>
+                <span>
+                  {t.operation}: {t.down} of {t.up + t.down} ratings this week.{" "}
+                  <Link href={evalsPath("feedback")} className="underline">
+                    Read the reasons
+                  </Link>
+                  .
+                </span>
+              </li>
+            ))}
+          </ul>
+        )}
+      </Card>
+    </section>
+  );
+}

package/templates/components/evalkeeper/evalkeeper.css

@@ -0,0 +1,71 @@
+/*
+ * Evalkeeper's colors and the "readable" line length, for the eval pages.
+ * Import it into your global CSS right after Tailwind:
+ *   @import "tailwindcss";
+ *   @import "../components/evalkeeper/evalkeeper.css";
+ * Change the colors to your app's; the pages use only these names (bg-card,
+ * text-muted, border-line, bg-tint, text-accent, text-good, text-warn,
+ * text-bad). Dark mode follows the system.
+ */
+
+:root {
+  --bg: #f7f7f5;
+  --card: #ffffff;
+  --ink: #1c1d1f;
+  --muted: #6b6f76;
+  --line: #e4e4e0;
+  --tint: #f0f0ec;
+  --accent: #3b5bdb;
+  --good: #2f9e44;
+  --warn: #e8890c;
+  --bad: #e03131;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --bg: #141517;
+    --card: #1c1d20;
+    --ink: #ececea;
+    --muted: #9a9ea6;
+    --line: #2c2e33;
+    --tint: #232529;
+    --accent: #748ffc;
+    --good: #51cf66;
+    --warn: #fcc419;
+    --bad: #ff6b6b;
+  }
+}
+
+@theme inline {
+  --color-bg: var(--bg);
+  --color-card: var(--card);
+  --color-ink: var(--ink);
+  --color-muted: var(--muted);
+  --color-line: var(--line);
+  --color-tint: var(--tint);
+  --color-accent: var(--accent);
+  --color-good: var(--good);
+  --color-warn: var(--warn);
+  --color-bad: var(--bad);
+}
+
+/* Anything you can press shows it. Tailwind's reset leaves buttons with
+   the default arrow. */
+@layer base {
+  button:not(:disabled),
+  [role="button"],
+  summary,
+  select,
+  label[for],
+  input[type="checkbox"],
+  input[type="radio"] {
+    cursor: pointer;
+  }
+  button:disabled {
+    cursor: not-allowed;
+  }
+}
+
+/* Prose never runs the full width of a wide page. */
+.readable {
+  max-width: 65ch;
+}