@crowandraven/evalkeeper 0.0.0-stage → 0.2.0

package/templates/app/evals/feedback/page.tsx

@@ -0,0 +1,219 @@
+import Link from "next/link";
+import {
+  asText,
+  thumbsSummary,
+  whyNotACase,
+  type AnySuite,
+  type FeedbackRecord,
+} from "@crowandraven/evalkeeper";
+import { FieldValues } from "@/components/evalkeeper/Fields";
+import { Submit } from "@/components/evalkeeper/Submit";
+import { InfoTip, Tip } from "@/components/evalkeeper/Tip";
+import { BackLink, Card, Chip, pct } from "@/components/evalkeeper/ui";
+import { ThumbsDown, ThumbsUp } from "lucide-react";
+import { feedbackStore, requireAdmin, SUITES } from "@/lib/evalkeeper";
+import { daysAgo } from "@/lib/evalkeeper/time";
+import { dismissFeedback, makeCase } from "@/lib/evalkeeper/actions";
+import { EVALS_BASE } from "@/lib/evalkeeper/paths";
+
+export const dynamic = "force-dynamic";
+
+/**
+ * What people thought of the AI's answers: thumbs per feature this week,
+ * and each rating waiting for a look, to make into a case or dismiss.
+ */
+export default async function FeedbackPage() {
+  await requireAdmin();
+  const all = await feedbackStore.listFeedback();
+  const summary = thumbsSummary(all, daysAgo(7));
+  const open = all
+    .filter((f) => f.status === "OPEN")
+    .sort((a, b) => Number(b.rating === "DOWN") - Number(a.rating === "DOWN"));
+  const suiteFor = (op: string) => SUITES.find((s) => s.operation === op);
+  const label = (op: string) => suiteFor(op)?.label ?? op;
+
+  return (
+    <div className="space-y-8">
+      <div className="space-y-2">
+        <BackLink href={EVALS_BASE}>Evals</BackLink>
+        <h1 className="text-2xl font-semibold">Feedback</h1>
+        <p className="readable text-muted">
+          Thumbs up and down from people using the product. A rating on a kept
+          input can become a draft case, so the next model is tested on the
+          answer people didn&apos;t like.
+        </p>
+      </div>
+
+      <section className="space-y-3">
+        <h2 className="text-lg font-semibold">
+          This week
+          <InfoTip term="thumbs" />
+        </h2>
+        {summary.length === 0 ? (
+          <p className="text-sm text-muted">No ratings this week.</p>
+        ) : (
+          <Card className="overflow-x-auto p-0 lg:overflow-visible">
+            <table className="w-full text-sm">
+              <thead className="text-left text-muted">
+                <tr className="border-b border-line">
+                  <th className="p-3 font-normal">Feature</th>
+                  <th className="p-3 font-normal">Thumbs up</th>
+                  <th className="p-3 font-normal">Thumbs down</th>
+                  <th className="p-3 font-normal">Share down</th>
+                  <th className="p-3 font-normal">Waiting</th>
+                </tr>
+              </thead>
+              <tbody>
+                {summary.map((s) => (
+                  <tr
+                    key={s.operation}
+                    className="border-b border-line last:border-0"
+                  >
+                    <td className="p-3">
+                      <span className="mr-2 font-medium">
+                        {label(s.operation)}
+                      </span>
+                      {s.alert && (
+                        <Tip
+                          text="At least 5 ratings, and at least 20% of them thumbs down: worth a look."
+                          id={`alert-${s.operation}`}
+                          side="bottom"
+                        >
+                          <Chip tone="warn">Needs a look</Chip>
+                        </Tip>
+                      )}
+                    </td>
+                    <td className="p-3 tabular-nums">{s.up}</td>
+                    <td className="p-3 tabular-nums">{s.down}</td>
+                    <td className="p-3 tabular-nums">
+                      {s.downShare === null ? "—" : pct(s.downShare)}
+                    </td>
+                    <td className="p-3 tabular-nums">{s.open}</td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </Card>
+        )}
+      </section>
+
+      <section className="space-y-3">
+        <h2 className="text-lg font-semibold">
+          Waiting for a look ({open.length})
+        </h2>
+        {open.length === 0 && (
+          <p className="text-sm text-muted">
+            Nothing waiting. Rate an answer on the{" "}
+            <Link href="/try" className="underline">
+              try it
+            </Link>{" "}
+            page to see one here.
+          </p>
+        )}
+        <div className="space-y-3">
+          {open.map((f) => (
+            <Rating
+              key={f.id}
+              fields={suiteFor(f.operation)?.fields}
+              f={f}
+              feature={label(f.operation)}
+              why={whyNotACase(f, suiteFor(f.operation))}
+            />
+          ))}
+        </div>
+      </section>
+    </div>
+  );
+}
+
+function Rating({
+  f,
+  fields,
+  feature,
+  why,
+}: {
+  f: FeedbackRecord;
+  fields: AnySuite["fields"];
+  feature: string;
+  why: string | null;
+}) {
+  return (
+    <article
+      aria-label={`Rating: ${feature}${f.reason ? `, ${f.reason}` : ""}`}
+    >
+      <Card className="space-y-3 text-sm">
+        <div className="flex flex-wrap items-center justify-between gap-2">
+          <span className="flex items-center gap-2">
+            <Chip tone={f.rating === "DOWN" ? "serious" : "live"}>
+              {f.rating === "DOWN" ? (
+                <>
+                  <ThumbsDown aria-hidden className="h-3.5 w-3.5" /> Thumbs down
+                </>
+              ) : (
+                <>
+                  <ThumbsUp aria-hidden className="h-3.5 w-3.5" /> Thumbs up
+                </>
+              )}
+            </Chip>
+            <span className="font-medium">{feature}</span>
+          </span>
+          <span className="text-xs text-muted">
+            {f.updatedAt.toLocaleString()}
+          </span>
+        </div>
+        {f.reason && <p className="readable">&ldquo;{f.reason}&rdquo;</p>}
+        <div className="grid gap-4 md:grid-cols-2">
+          <div className="space-y-1">
+            <p className="text-xs font-semibold uppercase tracking-wide text-muted">
+              Input
+            </p>
+            {f.input === null ? (
+              <p className="text-muted">Not kept</p>
+            ) : fields ? (
+              <FieldValues
+                fields={fields.input}
+                values={f.input}
+                prefix={`${f.id}-in-`}
+              />
+            ) : (
+              <p className="whitespace-pre-wrap font-mono text-xs">
+                {asText(f.input)}
+              </p>
+            )}
+          </div>
+          <div className="space-y-1">
+            <p className="text-xs font-semibold uppercase tracking-wide text-muted">
+              Answer
+            </p>
+            {fields && f.output !== null ? (
+              <FieldValues
+                fields={fields.label}
+                values={f.output}
+                prefix={`${f.id}-out-`}
+              />
+            ) : (
+              <p className="whitespace-pre-wrap font-mono text-xs">
+                {asText(f.output) || f.excerpt || "—"}
+              </p>
+            )}
+          </div>
+        </div>
+        <div className="flex flex-wrap items-center gap-2">
+          {why === null ? (
+            <form action={makeCase} className="flex items-center gap-1">
+              <input type="hidden" name="feedback" value={f.id} />
+              <Submit>Make a case</Submit>
+              <InfoTip term="makeCase" id={`make-${f.id}`} />
+            </form>
+          ) : (
+            <p className="readable text-xs text-muted">{why}</p>
+          )}
+          <form action={dismissFeedback}>
+            <input type="hidden" name="feedback" value={f.id} />
+            <Submit variant="quiet">Dismiss</Submit>
+          </form>
+        </div>
+      </Card>
+    </article>
+  );
+}

package/templates/app/evals/page.tsx

@@ -0,0 +1,313 @@
+import Link from "next/link";
+import {
+  attention,
+  perAnswer,
+  seconds,
+  suiteHealth,
+  thumbsSummary,
+  type GlossaryTerm,
+  type HistoryEntry,
+  type Settings,
+  type SettingsVersion,
+} from "@crowandraven/evalkeeper";
+import { Card, StatusBadge, pct } from "@/components/evalkeeper/ui";
+import {
+  feedbackStore,
+  MODELS,
+  requireAdmin,
+  runner,
+  settings,
+  SUITES,
+} from "@/lib/evalkeeper";
+import { daysAgo } from "@/lib/evalkeeper/time";
+import { loadWeekly } from "@/lib/evalkeeper/weekly";
+import { WeeklyCheck } from "@/components/evalkeeper/WeeklyCheck";
+import { InfoTip } from "@/components/evalkeeper/Tip";
+import { evalsPath } from "@/lib/evalkeeper/paths";
+import { HistoryEntries } from "@/components/evalkeeper/SettingsHistory";
+import { ChevronDown } from "lucide-react";
+
+export const dynamic = "force-dynamic";
+
+/** The evals hub: where each suite stands, and what needs attention. */
+export default async function EvalsHub() {
+  await requireAdmin();
+  const [boards, weekly] = await Promise.all([
+    Promise.all(SUITES.map((s) => runner.board(s.key))),
+    loadWeekly(),
+  ]);
+  const histories = await Promise.all(
+    SUITES.map((s) =>
+      Promise.all([
+        settings.history(s.operation),
+        settings.current(s.operation),
+      ]),
+    ),
+  );
+  const thumbs = thumbsSummary(await feedbackStore.listFeedback(), daysAgo(7));
+  const down = thumbs.reduce((t, s) => t + s.down, 0);
+  const waiting = thumbs.reduce((t, s) => t + s.open, 0);
+  const flagged = thumbs.filter((s) => s.alert).length;
+  const needs = attention(
+    boards.map((b) => ({
+      key: b.suite.key,
+      label: b.suite.label,
+      rows: b.rows,
+      toReview: b.provisional,
+    })),
+  );
+  return (
+    <div className="space-y-8">
+      <div className="readable space-y-2">
+        <h1 className="text-2xl font-semibold">Evals</h1>
+        <p className="text-muted">
+          Each suite tests one AI feature against answers people confirmed. Test
+          models on it, then pick the live one on evidence.
+        </p>
+      </div>
+
+      <div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-4">
+        <Stat
+          title="Most room to improve"
+          term="improve"
+          value={needs.improve ? pct(needs.improve.best) : "—"}
+          detail={
+            !needs.improve
+              ? "Test a suite to see"
+              : needs.improve.best >= 1
+                ? "Every suite has a model scoring 100%"
+                : `${needs.improve.label}, best on ${needs.improve.model}`
+          }
+        />
+        <Stat
+          title="Slowest live model"
+          term="slowest"
+          value={needs.slowest ? seconds(needs.slowest.ms) : "—"}
+          detail={
+            needs.slowest
+              ? `${needs.slowest.label} on ${needs.slowest.model}`
+              : "—"
+          }
+        />
+        <Stat
+          title="Costliest live model"
+          term="costliest"
+          value={needs.costliest ? perAnswer(needs.costliest.usd) : "—"}
+          detail={
+            needs.costliest
+              ? `${needs.costliest.label} on ${needs.costliest.model}`
+              : "—"
+          }
+        />
+        <Link href={evalsPath("feedback")}>
+          <Stat
+            title="Thumbs down this week"
+            term="thumbs"
+            value={String(down)}
+            detail={
+              flagged
+                ? `${flagged} feature${flagged === 1 ? "" : "s"} need a look · ${waiting} waiting`
+                : `${waiting} waiting for a look`
+            }
+          />
+        </Link>
+      </div>
+
+      <WeeklyCheck weekly={weekly} />
+
+      <section aria-labelledby="suites" className="space-y-3">
+        <h2 id="suites" className="text-lg font-semibold">
+          Suites
+          <InfoTip term="suites" />
+        </h2>
+        <div className="space-y-4">
+          {boards.map((b, i) => {
+            const h = suiteHealth(
+              b.rows,
+              b.suite.checks,
+              b.suite.priority,
+              b.cases,
+              b.provisional,
+              MODELS,
+            );
+            const [history, current] = histories[i] ?? [[], null];
+            return (
+              <SuiteCard
+                key={b.suite.key}
+                board={b}
+                health={h}
+                history={history}
+                current={current}
+              />
+            );
+          })}
+        </div>
+      </section>
+    </div>
+  );
+}
+
+function Stat({
+  title,
+  term,
+  value,
+  detail,
+}: {
+  title: string;
+  term: GlossaryTerm;
+  value: string;
+  detail: string;
+}) {
+  return (
+    <Card className="space-y-1">
+      <p className="text-sm text-muted">
+        {title}
+        <InfoTip term={term} />
+      </p>
+      <p className="text-2xl font-semibold tabular-nums">{value}</p>
+      <p className="text-sm text-muted">{detail}</p>
+    </Card>
+  );
+}
+
+type Board = Awaited<ReturnType<typeof runner.board>>;
+
+const label = (model: string) => MODELS[model]?.label ?? model;
+const day = (d: Date) => d.toLocaleDateString();
+/** Green at 95% and up, amber from 80%, red below. */
+const scoreTone = (rate: number) =>
+  rate >= 0.95 ? "text-good" : rate >= 0.8 ? "text-warn" : "text-bad";
+
+/**
+ * One feature on the hub: its live version and models, what the live
+ * model costs and how fast it is, its latest test, what needs a look, and
+ * its settings history.
+ */
+function SuiteCard({
+  board: b,
+  health: h,
+  history,
+  current,
+}: {
+  board: Board;
+  health: ReturnType<typeof suiteHealth>;
+  history: HistoryEntry[];
+  current: { settings: Settings; version: SettingsVersion | null } | null;
+}) {
+  const live = b.rows.find((r) => r.live && !r.variant);
+  const t = live?.test;
+  const version = current?.version ?? null;
+  const since = version
+    ? (history.find((e) => e.to.id === version.id)?.createdAt ??
+      version.createdAt)
+    : null;
+  const s = current?.settings;
+  const price = s ? MODELS[s.model] : undefined;
+  const versions = Math.max(1, ...history.map((e) => e.to.number));
+  return (
+    <Card className="relative space-y-4 transition-colors hover:border-muted">
+      <div className="flex flex-wrap items-start justify-between gap-4">
+        <div className="min-w-0 space-y-2">
+          <div className="flex flex-wrap items-center gap-2">
+            <h2 className="font-semibold">
+              {/* Covers the whole card; the versions toggle sits above it. */}
+              <Link
+                href={evalsPath(b.suite.key)}
+                className="after:absolute after:inset-0 after:rounded-lg"
+              >
+                {b.suite.label}
+              </Link>
+            </h2>
+            <StatusBadge status={h.status} />
+          </div>
+          <p className="readable text-sm text-muted">{b.suite.description}</p>
+        </div>
+        {t && (
+          <div className="shrink-0 space-y-0.5 text-right text-sm">
+            <p className="font-semibold tabular-nums">
+              {perAnswer(t.costPerAnswer)} / answer
+            </p>
+            <p className="text-xs text-muted">{seconds(t.avgMs)} avg</p>
+            {price?.inputPer1M !== undefined &&
+              price.outputPer1M !== undefined && (
+                <p className="text-xs text-muted">
+                  ${price.inputPer1M} / ${price.outputPer1M} per 1M
+                </p>
+              )}
+          </div>
+        )}
+      </div>
+
+      {s && (
+        <div className="space-y-0.5 text-sm">
+          <p>
+            <span className="font-medium">v{version?.number ?? 1}</span>{" "}
+            <span className="text-muted">
+              · {since ? `live since ${day(since)}` : "defaults from code"}
+            </span>
+          </p>
+          <p className="text-xs text-muted">
+            {label(s.model)}
+            {s.fallbackModel
+              ? ` · fallback ${label(s.fallbackModel)}`
+              : " · no fallback"}
+            {s.reasoningEffort ? ` · reasoning ${s.reasoningEffort}` : ""}
+          </p>
+        </div>
+      )}
+
+      <p className="text-sm">
+        {t ? (
+          <>
+            Score{" "}
+            <span className={`font-semibold ${scoreTone(t.right / t.total)}`}>
+              {pct(t.right / t.total)}
+            </span>
+          </>
+        ) : (
+          "Live model not tested yet"
+        )}{" "}
+        · {b.cases} cases · {h.testedModels} models tested ·{" "}
+        <Link
+          href={evalsPath(b.suite.key)}
+          className="relative text-accent hover:underline"
+        >
+          Compare models
+        </Link>
+      </p>
+
+      {h.details.length > 0 && (
+        <ul className="readable list-disc space-y-1 pl-5 text-sm text-muted">
+          {h.details.slice(0, 3).map((d) => (
+            <li key={d}>{d}</li>
+          ))}
+        </ul>
+      )}
+
+      {s && (
+        <details className="group/history relative rounded-md bg-tint">
+          <summary className="flex list-none items-center justify-between gap-2 p-3 text-sm font-medium">
+            Versions
+            <span className="flex items-center gap-1 text-xs font-normal text-muted">
+              {versions} version{versions === 1 ? "" : "s"}
+              <ChevronDown
+                aria-hidden
+                className="h-4 w-4 transition-transform group-open/history:rotate-180"
+              />
+            </span>
+          </summary>
+          <div className="px-3 pb-3">
+            <HistoryEntries
+              suite={b.suite.key}
+              entries={history}
+              liveId={version?.id ?? null}
+              label={label}
+              defaults={s}
+              plain
+            />
+          </div>
+        </details>
+      )}
+    </Card>
+  );
+}

package/templates/components/evalkeeper/AddCase.tsx

@@ -0,0 +1,62 @@
+"use client";
+
+import { useState } from "react";
+import { Plus } from "lucide-react";
+import { Card } from "./ui";
+
+/**
+ * A section's header row with an "Add a case" button at the top right,
+ * beside `tools`; the button opens `form` below the header.
+ */
+export function AddCaseHeader({
+  heading,
+  tools,
+  form,
+}: {
+  heading: React.ReactNode;
+  tools?: React.ReactNode;
+  /** The add-a-case form; null hides the button. */
+  form: React.ReactNode;
+}) {
+  const [open, setOpen] = useState(false);
+  return (
+    <>
+      <div className="flex flex-wrap items-center justify-between gap-4">
+        {heading}
+        <div className="flex items-center gap-2">
+          {form && (
+            <button
+              type="button"
+              aria-expanded={open}
+              onClick={() => {
+                setOpen(!open);
+              }}
+              className="inline-flex items-center gap-1.5 rounded-md border border-line bg-card px-3 py-1.5 text-sm hover:bg-tint"
+            >
+              <Plus aria-hidden className="h-4 w-4" />
+              Add a case
+            </button>
+          )}
+          {tools}
+        </div>
+      </div>
+      {open && form && (
+        <Card className="space-y-4">
+          <div className="flex items-center justify-between gap-2">
+            <h3 className="font-medium">New case</h3>
+            <button
+              type="button"
+              onClick={() => {
+                setOpen(false);
+              }}
+              className="text-sm text-muted hover:text-ink"
+            >
+              Cancel
+            </button>
+          </div>
+          {form}
+        </Card>
+      )}
+    </>
+  );
+}