@vercel/agent-eval-playground 0.0.1

package/README.md

@@ -0,0 +1,50 @@
+# @vercel/agent-eval-playground
+
+A web-based dashboard for browsing agent-eval experiment results. Ships as a standalone Next.js app — no build step required.
+
+## Usage
+
+```bash
+# Run from your eval project root (where results/ and evals/ live)
+npx @vercel/agent-eval-playground
+
+# With options
+npx @vercel/agent-eval-playground --results-dir ./results --evals-dir ./evals --port 3001
+```
+
+### CLI Options
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--results-dir <dir>` | `./results` | Path to experiment results directory |
+| `--evals-dir <dir>` | `./evals` | Path to eval fixtures directory |
+| `--port, -p <port>` | `3000` | HTTP server port |
+| `--help, -h` | | Show help |
+
+## Pages
+
+| Route | Description |
+|-------|-------------|
+| `/` | Dashboard with stats, recent experiments, eval previews |
+| `/experiments` | Full experiment list |
+| `/experiments/[name]/[timestamp]` | Experiment detail with per-eval breakdown and run results |
+| `/evals` | All eval fixtures |
+| `/evals/[name]` | Eval detail with prompt, files, and content |
+| `/compare` | Side-by-side comparison of two experiment runs |
+| `/transcript/[experiment]/[timestamp]/[evalName]/[run]` | Full agent transcript viewer |
+
+## Architecture
+
+- **Server Components** for all data fetching (`lib/data.ts` reads `fs` directly)
+- **Client Components** only for interactivity (`ComparePage` dropdowns, `TranscriptViewer` collapsibles)
+- **No API routes** — all data is read server-side, no client-side fetching
+- **No database** — filesystem is the source of truth
+- **`force-dynamic`** on all pages — data can change between requests
+
+### How it works
+
+The CLI (`bin.mjs`) resolves the bundled `next` binary, sets `RESULTS_DIR` and `EVALS_DIR` as environment variables, and spawns `next dev` from the package directory. Server Components in `lib/data.ts` read directly from the filesystem.
+
+## Publishing
+
+Published alongside `@vercel/agent-eval` via changesets. Run `npx changeset`, select `@vercel/agent-eval-playground`, and merge to main. The release workflow handles the rest via OIDC.

package/app/compare/page.tsx

@@ -0,0 +1,40 @@
+import { ComparePage } from "@/components/ComparePage";
+import { listExperiments, getExperimentDetail } from "@/lib/data";
+
+export const dynamic = "force-dynamic";
+
+function formatTimestamp(ts: string): string {
+  try {
+    const isoString = ts.replace(/T(\d{2})-(\d{2})-(\d{2})/, "T$1:$2:$3");
+    const date = new Date(isoString);
+    if (isNaN(date.getTime())) return ts;
+    return date.toLocaleString();
+  } catch {
+    return ts;
+  }
+}
+
+export default async function CompareRoute() {
+  const { items: experiments } = listExperiments();
+
+  // Build options and details map server-side to avoid hydration mismatch
+  // (toLocaleString differs between Node.js and browser)
+  const options = experiments.flatMap((exp) =>
+    exp.timestamps.map((ts) => ({
+      value: `${exp.name}|||${ts}`,
+      label: `${exp.name} / ${formatTimestamp(ts)}`,
+    }))
+  );
+
+  const detailsMap: Record<string, ReturnType<typeof getExperimentDetail>> = {};
+  for (const exp of experiments) {
+    for (const ts of exp.timestamps) {
+      const detail = getExperimentDetail(exp.name, ts);
+      if (detail) {
+        detailsMap[`${exp.name}|||${ts}`] = detail;
+      }
+    }
+  }
+
+  return <ComparePage options={options} detailsMap={detailsMap} />;
+}

package/app/evals/[name]/page.tsx

@@ -0,0 +1,22 @@
+import { notFound } from "next/navigation";
+import { EvalDetail } from "@/components/EvalDetail";
+import { getEvalDetail } from "@/lib/data";
+
+export const dynamic = "force-dynamic";
+
+export default async function EvalDetailRoute({
+  params,
+}: {
+  params: Promise<{ name: string }>;
+}) {
+  const { name } = await params;
+  const decodedName = decodeURIComponent(name);
+
+  const data = getEvalDetail(decodedName);
+
+  if (!data) {
+    notFound();
+  }
+
+  return <EvalDetail data={data} />;
+}

package/app/evals/page.tsx

@@ -0,0 +1,18 @@
+import { EvalsPage } from "@/components/EvalsPage";
+import { listEvals } from "@/lib/data";
+
+export const dynamic = "force-dynamic";
+
+const LIMIT = 21; // 7 rows of 3
+
+export default async function EvalsRoute({
+  searchParams,
+}: {
+  searchParams: Promise<{ all?: string }>;
+}) {
+  const { all } = await searchParams;
+  const showAll = all !== undefined;
+  const { items: evals, total } = listEvals(showAll ? undefined : LIMIT);
+
+  return <EvalsPage evals={evals} total={total} showAll={showAll} />;
+}

package/app/experiments/[name]/[timestamp]/page.tsx

@@ -0,0 +1,23 @@
+import { notFound } from "next/navigation";
+import { ExperimentDetail } from "@/components/ExperimentDetail";
+import { getExperimentDetail } from "@/lib/data";
+
+export const dynamic = "force-dynamic";
+
+export default async function ExperimentPage({
+  params,
+}: {
+  params: Promise<{ name: string; timestamp: string }>;
+}) {
+  const { name, timestamp } = await params;
+  const decodedName = decodeURIComponent(name);
+  const decodedTimestamp = decodeURIComponent(timestamp);
+
+  const data = getExperimentDetail(decodedName, decodedTimestamp);
+
+  if (!data) {
+    notFound();
+  }
+
+  return <ExperimentDetail data={data} />;
+}

package/app/experiments/page.tsx

@@ -0,0 +1,28 @@
+import { ExperimentList } from "@/components/ExperimentList";
+import { listExperiments } from "@/lib/data";
+
+export const dynamic = "force-dynamic";
+
+const LIMIT = 20;
+
+export default async function ExperimentsPage({
+  searchParams,
+}: {
+  searchParams: Promise<{ all?: string }>;
+}) {
+  const { all } = await searchParams;
+  const showAll = all !== undefined;
+  const { items: experiments, total } = listExperiments(showAll ? undefined : LIMIT);
+
+  return (
+    <div className="space-y-6">
+      <div>
+        <h1 className="text-2xl font-bold tracking-tight">Experiments</h1>
+        <p className="text-muted-foreground mt-1">
+          Browse and inspect your agent evaluation results.
+        </p>
+      </div>
+      <ExperimentList experiments={experiments} total={total} showAll={showAll} />
+    </div>
+  );
+}

package/app/globals.css

@@ -0,0 +1,126 @@
+@import "tailwindcss";
+@import "tw-animate-css";
+@import "shadcn/tailwind.css";
+
+@custom-variant dark (&:is(.dark *));
+
+@theme inline {
+  --color-background: var(--background);
+  --color-foreground: var(--foreground);
+  --font-sans: var(--font-sans);
+  --font-mono: var(--font-geist-mono);
+  --color-sidebar-ring: var(--sidebar-ring);
+  --color-sidebar-border: var(--sidebar-border);
+  --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+  --color-sidebar-accent: var(--sidebar-accent);
+  --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+  --color-sidebar-primary: var(--sidebar-primary);
+  --color-sidebar-foreground: var(--sidebar-foreground);
+  --color-sidebar: var(--sidebar);
+  --color-chart-5: var(--chart-5);
+  --color-chart-4: var(--chart-4);
+  --color-chart-3: var(--chart-3);
+  --color-chart-2: var(--chart-2);
+  --color-chart-1: var(--chart-1);
+  --color-ring: var(--ring);
+  --color-input: var(--input);
+  --color-border: var(--border);
+  --color-destructive: var(--destructive);
+  --color-accent-foreground: var(--accent-foreground);
+  --color-accent: var(--accent);
+  --color-muted-foreground: var(--muted-foreground);
+  --color-muted: var(--muted);
+  --color-secondary-foreground: var(--secondary-foreground);
+  --color-secondary: var(--secondary);
+  --color-primary-foreground: var(--primary-foreground);
+  --color-primary: var(--primary);
+  --color-popover-foreground: var(--popover-foreground);
+  --color-popover: var(--popover);
+  --color-card-foreground: var(--card-foreground);
+  --color-card: var(--card);
+  --radius-sm: calc(var(--radius) - 4px);
+  --radius-md: calc(var(--radius) - 2px);
+  --radius-lg: var(--radius);
+  --radius-xl: calc(var(--radius) + 4px);
+  --radius-2xl: calc(var(--radius) + 8px);
+  --radius-3xl: calc(var(--radius) + 12px);
+  --radius-4xl: calc(var(--radius) + 16px);
+}
+
+:root {
+  --background: oklch(1 0 0);
+  --foreground: oklch(0.141 0.005 285.823);
+  --card: oklch(1 0 0);
+  --card-foreground: oklch(0.141 0.005 285.823);
+  --popover: oklch(1 0 0);
+  --popover-foreground: oklch(0.141 0.005 285.823);
+  --primary: oklch(0.648 0.2 131.684);
+  --primary-foreground: oklch(0.986 0.031 120.757);
+  --secondary: oklch(0.967 0.001 286.375);
+  --secondary-foreground: oklch(0.21 0.006 285.885);
+  --muted: oklch(0.967 0.001 286.375);
+  --muted-foreground: oklch(0.552 0.016 285.938);
+  --accent: oklch(0.967 0.001 286.375);
+  --accent-foreground: oklch(0.21 0.006 285.885);
+  --destructive: oklch(0.577 0.245 27.325);
+  --border: oklch(0.92 0.004 286.32);
+  --input: oklch(0.92 0.004 286.32);
+  --ring: oklch(0.705 0.015 286.067);
+  --chart-1: oklch(0.871 0.15 154.449);
+  --chart-2: oklch(0.723 0.219 149.579);
+  --chart-3: oklch(0.627 0.194 149.214);
+  --chart-4: oklch(0.527 0.154 150.069);
+  --chart-5: oklch(0.448 0.119 151.328);
+  --radius: 0.45rem;
+  --sidebar: oklch(0.985 0 0);
+  --sidebar-foreground: oklch(0.141 0.005 285.823);
+  --sidebar-primary: oklch(0.648 0.2 131.684);
+  --sidebar-primary-foreground: oklch(0.986 0.031 120.757);
+  --sidebar-accent: oklch(0.967 0.001 286.375);
+  --sidebar-accent-foreground: oklch(0.21 0.006 285.885);
+  --sidebar-border: oklch(0.92 0.004 286.32);
+  --sidebar-ring: oklch(0.705 0.015 286.067);
+}
+
+.dark {
+  --background: oklch(0.141 0.005 285.823);
+  --foreground: oklch(0.985 0 0);
+  --card: oklch(0.21 0.006 285.885);
+  --card-foreground: oklch(0.985 0 0);
+  --popover: oklch(0.21 0.006 285.885);
+  --popover-foreground: oklch(0.985 0 0);
+  --primary: oklch(0.648 0.2 131.684);
+  --primary-foreground: oklch(0.986 0.031 120.757);
+  --secondary: oklch(0.274 0.006 286.033);
+  --secondary-foreground: oklch(0.985 0 0);
+  --muted: oklch(0.274 0.006 286.033);
+  --muted-foreground: oklch(0.705 0.015 286.067);
+  --accent: oklch(0.274 0.006 286.033);
+  --accent-foreground: oklch(0.985 0 0);
+  --destructive: oklch(0.704 0.191 22.216);
+  --border: oklch(1 0 0 / 10%);
+  --input: oklch(1 0 0 / 15%);
+  --ring: oklch(0.552 0.016 285.938);
+  --chart-1: oklch(0.871 0.15 154.449);
+  --chart-2: oklch(0.723 0.219 149.579);
+  --chart-3: oklch(0.627 0.194 149.214);
+  --chart-4: oklch(0.527 0.154 150.069);
+  --chart-5: oklch(0.448 0.119 151.328);
+  --sidebar: oklch(0.21 0.006 285.885);
+  --sidebar-foreground: oklch(0.985 0 0);
+  --sidebar-primary: oklch(0.768 0.233 130.85);
+  --sidebar-primary-foreground: oklch(0.986 0.031 120.757);
+  --sidebar-accent: oklch(0.274 0.006 286.033);
+  --sidebar-accent-foreground: oklch(0.985 0 0);
+  --sidebar-border: oklch(1 0 0 / 10%);
+  --sidebar-ring: oklch(0.552 0.016 285.938);
+}
+
+@layer base {
+  * {
+    @apply border-border outline-ring/50;
+  }
+  body {
+    @apply bg-background text-foreground;
+  }
+}

package/app/layout.tsx

@@ -0,0 +1,102 @@
+import type { Metadata } from "next";
+import { Geist, Geist_Mono, Figtree } from "next/font/google";
+import Link from "next/link";
+import "./globals.css";
+
+const figtree = Figtree({ subsets: ["latin"], variable: "--font-sans" });
+
+const geistSans = Geist({
+  variable: "--font-geist-sans",
+  subsets: ["latin"],
+});
+
+const geistMono = Geist_Mono({
+  variable: "--font-geist-mono",
+  subsets: ["latin"],
+});
+
+export const metadata: Metadata = {
+  title: "Agent Eval Playground",
+  description: "Browse experiment results, inspect agent transcripts, and compare runs",
+};
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en" className={`dark ${figtree.variable}`}>
+      <body className={`${geistSans.variable} ${geistMono.variable} antialiased`}>
+        <div className="min-h-screen bg-background">
+          <header className="sticky top-0 z-50 bg-background">
+            <div className="flex h-14 items-center justify-between px-4 gap-6">
+              <div className="flex items-center gap-2">
+                <Link href="https://vercel.com" title="Made with love by Vercel">
+                  <svg
+                    data-testid="geist-icon"
+                    height="18"
+                    strokeLinejoin="round"
+                    viewBox="0 0 16 16"
+                    width="18"
+                    style={{ color: "currentcolor" }}
+                  >
+                    <path
+                      fillRule="evenodd"
+                      clipRule="evenodd"
+                      d="M8 1L16 15H0L8 1Z"
+                      fill="currentColor"
+                    />
+                  </svg>
+                </Link>
+                <span className="text-(--ds-gray-500)">
+                  <svg
+                    data-testid="geist-icon"
+                    height="16"
+                    strokeLinejoin="round"
+                    viewBox="0 0 16 16"
+                    width="16"
+                    style={{ color: "currentcolor" }}
+                  >
+                    <path
+                      fillRule="evenodd"
+                      clipRule="evenodd"
+                      d="M4.01526 15.3939L4.3107 14.7046L10.3107 0.704556L10.6061 0.0151978L11.9849 0.606077L11.6894 1.29544L5.68942 15.2954L5.39398 15.9848L4.01526 15.3939Z"
+                      fill="currentColor"
+                    />
+                  </svg>
+                </span>
+                <Link href="/">
+                  <span className="font-medium tracking-tight text-lg">agent-eval</span>
+                </Link>
+              </div>
+              <nav className="flex items-baseline gap-4">
+                <Link
+                  href="/experiments"
+                  className="text-sm text-muted-foreground hover:text-foreground transition-colors"
+                >
+                  Experiments
+                </Link>
+                <Link
+                  href="/compare"
+                  className="text-sm text-muted-foreground hover:text-foreground transition-colors"
+                >
+                  Compare
+                </Link>
+                <Link
+                  href="/evals"
+                  className="text-sm text-muted-foreground hover:text-foreground transition-colors"
+                >
+                  Evals
+                </Link>
+              </nav>
+            </div>
+          </header>
+          <main className="mx-auto max-w-5xl px-6 py-8 text-sm">
+            {children}
+          </main>
+        </div>
+      </body>
+    </html>
+  );
+}

package/app/page.tsx

@@ -0,0 +1,179 @@
+import Link from "next/link";
+import { listExperiments, listEvals } from "@/lib/data";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+
+export const dynamic = "force-dynamic";
+
+export default async function HomePage() {
+  const { items: experiments, total: totalExperiments } = listExperiments(6);
+  const { items: evals, total: totalEvals } = listEvals(6);
+
+  const totalRuns = experiments.reduce((sum, e) => sum + e.timestamps.length, 0);
+
+  // Compute aggregate pass rate across all latest experiment runs
+  const totalPassed = experiments.reduce((sum, e) => sum + (e.latestPassedRuns ?? 0), 0);
+  const totalAttempted = experiments.reduce((sum, e) => sum + (e.latestTotalRuns ?? 0), 0);
+  const overallPassRate = totalAttempted > 0 ? (totalPassed / totalAttempted) * 100 : 0;
+
+  return (
+    <div className="space-y-10">
+      {/* Hero */}
+      <div>
+        <h1 className="text-2xl font-bold tracking-tight">Overview</h1>
+        <p className="text-muted-foreground mt-1">
+          Playground for your agent evals to view experiments, evals, and compare runs.
+        </p>
+      </div>
+
+      {/* Stats */}
+      <div className="grid grid-cols-1 sm:grid-cols-4 gap-4">
+        <Card>
+          <CardContent className="py-4 px-4">
+            <div className="text-xs text-muted-foreground">Experiments</div>
+            <div className="text-2xl font-bold mt-1">{totalExperiments}</div>
+          </CardContent>
+        </Card>
+        <Card>
+          <CardContent className="py-4 px-4">
+            <div className="text-xs text-muted-foreground">Total Runs</div>
+            <div className="text-2xl font-bold mt-1">{totalRuns}</div>
+          </CardContent>
+        </Card>
+        <Card>
+          <CardContent className="py-4 px-4">
+            <div className="text-xs text-muted-foreground">Eval Fixtures</div>
+            <div className="text-2xl font-bold mt-1">{totalEvals}</div>
+          </CardContent>
+        </Card>
+        <Card>
+          <CardContent className="py-4 px-4">
+            <div className="text-xs text-muted-foreground">Latest Pass Rate</div>
+            <div className="text-2xl font-bold mt-1">
+              {totalAttempted > 0 ? `${overallPassRate.toFixed(0)}%` : "—"}
+            </div>
+          </CardContent>
+        </Card>
+      </div>
+
+      {/* Recent Experiments */}
+      <section className="space-y-3">
+        <div className="flex items-center justify-between">
+          <h2 className="text-lg font-semibold tracking-tight">Recent Experiments</h2>
+          <Link href="/experiments" className="text-sm text-muted-foreground hover:text-foreground transition-colors">
+            View all →
+          </Link>
+        </div>
+        {experiments.length === 0 ? (
+          <Card>
+            <CardContent className="py-8 text-center text-muted-foreground">
+              No experiments yet. Run <code className="text-foreground bg-muted px-1.5 py-0.5 rounded text-xs">agent-eval</code> to get started.
+            </CardContent>
+          </Card>
+        ) : (
+          <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
+            {experiments.slice(0, 6).map((exp) => (
+              <Link
+                key={exp.name}
+                href={exp.latestTimestamp
+                  ? `/experiments/${encodeURIComponent(exp.name)}/${encodeURIComponent(exp.latestTimestamp)}`
+                  : "/experiments"}
+                className="block cursor-pointer"
+              >
+                <Card className="transition-colors hover:bg-muted">
+                  <CardHeader className="pb-2">
+                    <CardTitle className="text-sm">{exp.name}</CardTitle>
+                  </CardHeader>
+                  <CardContent>
+                    <div className="flex items-center gap-2">
+                      {exp.latestPassRate !== undefined ? (
+                        <Badge
+                          variant={
+                            exp.latestPassRate === 100
+                              ? "default"
+                              : exp.latestPassRate >= 50
+                                ? "secondary"
+                                : "destructive"
+                          }
+                        >
+                          {exp.latestPassRate.toFixed(0)}%
+                        </Badge>
+                      ) : (
+                        <Badge variant="outline">—</Badge>
+                      )}
+                      <span className="text-xs text-muted-foreground">
+                        {exp.timestamps.length} run{exp.timestamps.length !== 1 ? "s" : ""}
+                      </span>
+                    </div>
+                  </CardContent>
+                </Card>
+              </Link>
+            ))}
+          </div>
+        )}
+      </section>
+
+      {/* Evals Preview */}
+      <section className="space-y-3">
+        <div className="flex items-center justify-between">
+          <h2 className="text-lg font-semibold tracking-tight">Eval Fixtures</h2>
+          <Link href="/evals" className="text-sm text-muted-foreground hover:text-foreground transition-colors">
+            View all →
+          </Link>
+        </div>
+        {evals.length === 0 ? (
+          <Card>
+            <CardContent className="py-8 text-center text-muted-foreground">
+              No evals found. Create evals in your <code className="text-foreground bg-muted px-1.5 py-0.5 rounded text-xs">evals/</code> directory.
+            </CardContent>
+          </Card>
+        ) : (
+          <div className="grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 gap-4">
+            {evals.slice(0, 6).map((evalInfo) => (
+              <Link
+                key={evalInfo.name}
+                href={`/evals/${encodeURIComponent(evalInfo.name)}`}
+                className="block cursor-pointer"
+              >
+                <Card className="transition-colors hover:bg-muted">
+                  <CardHeader className="pb-2">
+                    <CardTitle className="text-sm">{evalInfo.name}</CardTitle>
+                  </CardHeader>
+                  <CardContent>
+                    {evalInfo.prompt ? (
+                      <p className="text-xs text-muted-foreground line-clamp-2">
+                        {evalInfo.prompt.slice(0, 150)}
+                        {evalInfo.prompt.length > 150 ? "..." : ""}
+                      </p>
+                    ) : (
+                      <p className="text-xs text-muted-foreground">No prompt</p>
+                    )}
+                  </CardContent>
+                </Card>
+              </Link>
+            ))}
+          </div>
+        )}
+      </section>
+
+      {/* Compare CTA */}
+      <section className="space-y-3">
+        <div className="flex items-center justify-between">
+          <h2 className="text-lg font-semibold tracking-tight">Compare</h2>
+        </div>
+        <Link href="/compare" className="block cursor-pointer">
+          <Card className="transition-colors hover:bg-muted">
+            <CardContent className="py-8 text-center">
+              <p className="mx-auto max-w-sm text-sm text-muted-foreground">
+                Compare two experiment runs side-by-side to see pass rate deltas, duration changes, and per-eval breakdowns.
+              </p>
+              <p className="text-xs text-foreground mt-2">
+                Open Compare →
+              </p>
+            </CardContent>
+          </Card>
+        </Link>
+      </section>
+    </div>
+  );
+}

package/app/transcript/[experiment]/[timestamp]/[evalName]/[run]/page.tsx

@@ -0,0 +1,43 @@
+import { notFound } from "next/navigation";
+import { TranscriptPage } from "@/components/TranscriptPage";
+import { getTranscript } from "@/lib/data";
+
+export const dynamic = "force-dynamic";
+
+export default async function TranscriptRoute({
+  params,
+}: {
+  params: Promise<{
+    experiment: string;
+    timestamp: string;
+    evalName: string;
+    run: string;
+  }>;
+}) {
+  const { experiment, timestamp, evalName, run } = await params;
+  const decodedExperiment = decodeURIComponent(experiment);
+  const decodedTimestamp = decodeURIComponent(timestamp);
+  const decodedEvalName = decodeURIComponent(evalName);
+  const decodedRun = decodeURIComponent(run);
+
+  const transcript = getTranscript(
+    decodedExperiment,
+    decodedTimestamp,
+    decodedEvalName,
+    decodedRun
+  );
+
+  if (!transcript) {
+    notFound();
+  }
+
+  return (
+    <TranscriptPage
+      experiment={decodedExperiment}
+      timestamp={decodedTimestamp}
+      evalName={decodedEvalName}
+      run={decodedRun}
+      transcript={transcript}
+    />
+  );
+}