@vercel/agent-eval-playground 0.0.1

package/bin.mjs

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+
+import { spawn } from "child_process";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import { createRequire } from "module";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+
+// Parse CLI arguments
+const args = process.argv.slice(2);
+let resultsDir = "./results";
+let evalsDir = "./evals";
+let port = "3000";
+let watch = false;
+
+for (let i = 0; i < args.length; i++) {
+  switch (args[i]) {
+    case "--results-dir":
+      resultsDir = args[++i];
+      break;
+    case "--evals-dir":
+      evalsDir = args[++i];
+      break;
+    case "--port":
+    case "-p":
+      port = args[++i];
+      break;
+    case "--watch":
+      watch = true;
+      break;
+    case "--help":
+    case "-h":
+      console.log(`
+Usage: agent-eval-playground [options]
+
+Options:
+  --results-dir <dir>  Path to results directory (default: ./results)
+  --evals-dir <dir>    Path to evals directory (default: ./evals)
+  --port, -p <port>    HTTP server port (default: 3000)
+  --watch              Enable live mode — watch results for changes
+  --help, -h           Show this help message
+`);
+      process.exit(0);
+  }
+}
+
+// Set environment variables for the Next.js app
+process.env.RESULTS_DIR = resolve(resultsDir);
+process.env.EVALS_DIR = resolve(evalsDir);
+if (watch) {
+  process.env.WATCH = "true";
+}
+
+// Find the next binary from this package's dependencies
+let nextBin;
+try {
+  const nextPkgPath = require.resolve("next/package.json");
+  nextBin = resolve(dirname(nextPkgPath), "dist", "bin", "next");
+} catch {
+  console.error(
+    'Error: "next" package not found. Make sure dependencies are installed.'
+  );
+  process.exit(1);
+}
+
+console.log(`Agent Eval Playground`);
+console.log(`  Results: ${process.env.RESULTS_DIR}`);
+console.log(`  Evals:   ${process.env.EVALS_DIR}`);
+console.log(`  Port:    ${port}`);
+if (watch) console.log(`  Watch:   enabled`);
+console.log();
+
+// Run next dev from the package directory
+const child = spawn(process.execPath, [nextBin, "dev", "-p", port], {
+  cwd: __dirname,
+  stdio: "inherit",
+  env: process.env,
+});
+
+child.on("exit", (code) => process.exit(code ?? 0));
+
+// Forward signals
+process.on("SIGINT", () => child.kill("SIGINT"));
+process.on("SIGTERM", () => child.kill("SIGTERM"));

package/components/ComparePage.tsx

@@ -0,0 +1,312 @@
+"use client";
+
+import { useState } from "react";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeader,
+  TableRow,
+} from "@/components/ui/table";
+import { Separator } from "@/components/ui/separator";
+
+interface EvalDetail {
+  name: string;
+  totalRuns: number;
+  passedRuns: number;
+  passRate: number;
+  meanDuration: number;
+}
+
+interface ExperimentDetailData {
+  name: string;
+  timestamp: string;
+  evals: EvalDetail[];
+}
+
+interface SelectOption {
+  value: string;
+  label: string;
+}
+
+interface ComparePageProps {
+  options: SelectOption[];
+  detailsMap: Record<string, ExperimentDetailData | null>;
+}
+
+export function ComparePage({ options, detailsMap }: ComparePageProps) {
+
+  // Pre-select the two most recent runs if available
+  const defaultLeft = options.length >= 1 ? options[0].value : "";
+  const defaultRight = options.length >= 2 ? options[1].value : "";
+
+  const [leftValue, setLeftValue] = useState(defaultLeft);
+  const [rightValue, setRightValue] = useState(defaultRight);
+
+  // Look up detail data directly from the pre-loaded map
+  const leftData = leftValue ? (detailsMap[leftValue] ?? null) : null;
+  const rightData = rightValue ? (detailsMap[rightValue] ?? null) : null;
+
+  // Merge eval names from both sides
+  const allEvalNames = new Set<string>();
+  leftData?.evals.forEach((e) => allEvalNames.add(e.name));
+  rightData?.evals.forEach((e) => allEvalNames.add(e.name));
+  const evalNames = Array.from(allEvalNames).sort();
+
+  const leftMap = new Map(leftData?.evals.map((e) => [e.name, e]) ?? []);
+  const rightMap = new Map(rightData?.evals.map((e) => [e.name, e]) ?? []);
+
+  return (
+    <div className="space-y-6">
+      <div>
+        <h1 className="text-2xl font-bold tracking-tight">Compare</h1>
+        <p className="text-muted-foreground mt-1">
+          Compare two experiment runs side-by-side.
+        </p>
+      </div>
+
+      {/* Selection */}
+      <div className="grid grid-cols-1 md:grid-cols-2 gap-4">
+        <Card>
+          <CardHeader className="pb-3">
+            <CardTitle className="text-sm text-muted-foreground">
+              Left
+            </CardTitle>
+          </CardHeader>
+          <CardContent>
+            <Select
+              value={leftValue || undefined}
+              onValueChange={setLeftValue}
+            >
+              <SelectTrigger>
+                <SelectValue placeholder="Select experiment run..." />
+              </SelectTrigger>
+              <SelectContent>
+                {options.map((opt) => (
+                  <SelectItem key={opt.value} value={opt.value}>
+                    {opt.label}
+                  </SelectItem>
+                ))}
+              </SelectContent>
+            </Select>
+          </CardContent>
+        </Card>
+
+        <Card>
+          <CardHeader className="pb-3">
+            <CardTitle className="text-sm text-muted-foreground">
+              Right
+            </CardTitle>
+          </CardHeader>
+          <CardContent>
+            <Select
+              value={rightValue || undefined}
+              onValueChange={setRightValue}
+            >
+              <SelectTrigger>
+                <SelectValue placeholder="Select experiment run..." />
+              </SelectTrigger>
+              <SelectContent>
+                {options.map((opt) => (
+                  <SelectItem key={opt.value} value={opt.value}>
+                    {opt.label}
+                  </SelectItem>
+                ))}
+              </SelectContent>
+            </Select>
+          </CardContent>
+        </Card>
+      </div>
+
+      {/* Comparison table */}
+      {leftData && rightData && evalNames.length > 0 && (
+        <>
+          <Separator />
+
+          {/* Summary comparison */}
+          <div className="grid grid-cols-1 md:grid-cols-3 gap-4">
+            <ComparisonStat
+              label="Overall Pass Rate"
+              left={avgPassRate(leftData.evals)}
+              right={avgPassRate(rightData.evals)}
+              format={(v) => `${v.toFixed(0)}%`}
+              higherIsBetter
+            />
+            <ComparisonStat
+              label="Avg Duration"
+              left={avgDuration(leftData.evals)}
+              right={avgDuration(rightData.evals)}
+              format={(v) => `${v.toFixed(1)}s`}
+              higherIsBetter={false}
+            />
+            <ComparisonStat
+              label="Evals Passed"
+              left={leftData.evals.filter((e) => e.passedRuns === e.totalRuns).length}
+              right={rightData.evals.filter((e) => e.passedRuns === e.totalRuns).length}
+              format={(v) => `${v}`}
+              higherIsBetter
+            />
+          </div>
+
+          {/* Per-eval comparison table */}
+          <Card>
+            <CardHeader>
+              <CardTitle className="text-base">Per-Eval Comparison</CardTitle>
+            </CardHeader>
+            <CardContent>
+              <Table>
+                <TableHeader>
+                  <TableRow>
+                    <TableHead>Eval</TableHead>
+                    <TableHead className="text-center">Left Pass Rate</TableHead>
+                    <TableHead className="text-center">Right Pass Rate</TableHead>
+                    <TableHead className="text-center">Delta</TableHead>
+                    <TableHead className="text-center">Left Duration</TableHead>
+                    <TableHead className="text-center">Right Duration</TableHead>
+                  </TableRow>
+                </TableHeader>
+                <TableBody>
+                  {evalNames.map((name) => {
+                    const left = leftMap.get(name);
+                    const right = rightMap.get(name);
+                    const leftRate = left?.passRate ?? 0;
+                    const rightRate = right?.passRate ?? 0;
+                    const delta = rightRate - leftRate;
+
+                    return (
+                      <TableRow key={name}>
+                        <TableCell className="font-medium">{name}</TableCell>
+                        <TableCell className="text-center">
+                          {left ? (
+                            <Badge
+                              variant={
+                                left.passRate === 100 ? "default" : "destructive"
+                              }
+                            >
+                              {left.passRate.toFixed(0)}%
+                            </Badge>
+                          ) : (
+                            <span className="text-muted-foreground">--</span>
+                          )}
+                        </TableCell>
+                        <TableCell className="text-center">
+                          {right ? (
+                            <Badge
+                              variant={
+                                right.passRate === 100 ? "default" : "destructive"
+                              }
+                            >
+                              {right.passRate.toFixed(0)}%
+                            </Badge>
+                          ) : (
+                            <span className="text-muted-foreground">--</span>
+                          )}
+                        </TableCell>
+                        <TableCell className="text-center">
+                          {left && right ? (
+                            <span
+                              className={
+                                delta > 0
+                                  ? "text-green-500"
+                                  : delta < 0
+                                    ? "text-red-500"
+                                    : "text-muted-foreground"
+                              }
+                            >
+                              {delta > 0 ? "+" : ""}
+                              {delta.toFixed(0)}%
+                            </span>
+                          ) : (
+                            <span className="text-muted-foreground">--</span>
+                          )}
+                        </TableCell>
+                        <TableCell className="text-center text-sm text-muted-foreground">
+                          {left ? `${left.meanDuration.toFixed(1)}s` : "--"}
+                        </TableCell>
+                        <TableCell className="text-center text-sm text-muted-foreground">
+                          {right ? `${right.meanDuration.toFixed(1)}s` : "--"}
+                        </TableCell>
+                      </TableRow>
+                    );
+                  })}
+                </TableBody>
+              </Table>
+            </CardContent>
+          </Card>
+        </>
+      )}
+
+      {/* Prompt to select both */}
+      {(!leftData || !rightData) && (
+        <Card>
+          <CardContent className="py-12 text-center">
+            <p className="text-muted-foreground">
+              Select two experiment runs above to compare them.
+            </p>
+          </CardContent>
+        </Card>
+      )}
+    </div>
+  );
+}
+
+function ComparisonStat({
+  label,
+  left,
+  right,
+  format,
+  higherIsBetter,
+}: {
+  label: string;
+  left: number;
+  right: number;
+  format: (v: number) => string;
+  higherIsBetter: boolean;
+}) {
+  const delta = right - left;
+  const improved = higherIsBetter ? delta > 0 : delta < 0;
+  const regressed = higherIsBetter ? delta < 0 : delta > 0;
+
+  return (
+    <Card>
+      <CardContent className="py-4 px-4">
+        <div className="text-xs text-muted-foreground">{label}</div>
+        <div className="flex items-baseline gap-4 mt-1">
+          <span className="text-lg font-medium">{format(left)}</span>
+          <span className="text-muted-foreground">→</span>
+          <span className="text-lg font-medium">{format(right)}</span>
+          {delta !== 0 && (
+            <Badge
+              variant={improved ? "default" : regressed ? "destructive" : "secondary"}
+              className="text-xs"
+            >
+              {delta > 0 ? "+" : ""}
+              {format(Math.abs(delta))}
+            </Badge>
+          )}
+        </div>
+      </CardContent>
+    </Card>
+  );
+}
+
+function avgPassRate(evals: EvalDetail[]): number {
+  if (evals.length === 0) return 0;
+  return evals.reduce((s, e) => s + e.passRate, 0) / evals.length;
+}
+
+function avgDuration(evals: EvalDetail[]): number {
+  if (evals.length === 0) return 0;
+  return evals.reduce((s, e) => s + e.meanDuration, 0) / evals.length;
+}
+

package/components/EvalDetail.tsx

@@ -0,0 +1,114 @@
+import Link from "next/link";
+import { Badge } from "@/components/ui/badge";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import { ScrollArea } from "@/components/ui/scroll-area";
+
+interface EvalDetailData {
+  name: string;
+  prompt: string;
+  files: string[];
+  fileContents?: Record<string, string>;
+}
+
+interface EvalDetailProps {
+  data: EvalDetailData;
+}
+
+export function EvalDetail({ data }: EvalDetailProps) {
+  const evalFile = data.fileContents?.["EVAL.ts"] || data.fileContents?.["EVAL.tsx"];
+  const evalFileName = data.fileContents?.["EVAL.ts"] ? "EVAL.ts" : "EVAL.tsx";
+  const packageJson = data.fileContents?.["package.json"];
+
+  return (
+    <div className="space-y-6">
+      {/* Breadcrumb */}
+      <div className="flex items-center gap-2 text-sm text-muted-foreground">
+        <Link href="/evals" className="cursor-pointer hover:underline underline-offset-4">
+          Evals
+        </Link>
+        <span>/</span>
+        <span>{data.name}</span>
+      </div>
+
+      <div>
+        <h1 className="text-2xl font-bold tracking-tight">{data.name}</h1>
+      </div>
+
+      <Tabs defaultValue="prompt" className="w-full">
+        <TabsList>
+          <TabsTrigger value="prompt">PROMPT.md</TabsTrigger>
+          {evalFile && <TabsTrigger value="eval">{evalFileName}</TabsTrigger>}
+          {packageJson && <TabsTrigger value="package">package.json</TabsTrigger>}
+          <TabsTrigger value="files">Files</TabsTrigger>
+        </TabsList>
+
+        <TabsContent value="prompt" className="mt-4">
+          <ScrollArea className="h-[calc(100vh-300px)]">
+            <pre className="text-sm font-mono whitespace-pre-wrap rounded-lg bg-muted p-4">
+              {data.prompt || "No PROMPT.md found."}
+            </pre>
+          </ScrollArea>
+        </TabsContent>
+
+        {evalFile && (
+          <TabsContent value="eval" className="mt-4">
+            <ScrollArea className="h-[calc(100vh-300px)]">
+              <pre className="text-sm font-mono whitespace-pre-wrap rounded-lg bg-muted p-4 overflow-x-auto">
+                {evalFile}
+              </pre>
+            </ScrollArea>
+          </TabsContent>
+        )}
+
+        {packageJson && (
+          <TabsContent value="package" className="mt-4">
+            <ScrollArea className="h-[calc(100vh-300px)]">
+              <pre className="text-sm font-mono whitespace-pre-wrap rounded-lg bg-muted p-4 overflow-x-auto">
+                {packageJson}
+              </pre>
+            </ScrollArea>
+          </TabsContent>
+        )}
+
+        <TabsContent value="files" className="mt-4">
+          <div className="text-sm text-muted-foreground mb-3">
+            {data.files.length} files
+          </div>
+          <div className="space-y-1">
+            {data.files.map((file) => (
+              <div
+                key={file}
+                className="flex items-center gap-2 text-sm font-mono bg-muted rounded-lg px-3 py-1.5"
+              >
+                <FileIcon filename={file} />
+                <span>{file}</span>
+              </div>
+            ))}
+          </div>
+        </TabsContent>
+      </Tabs>
+    </div>
+  );
+}
+
+function FileIcon({ filename }: { filename: string }) {
+  const ext = filename.split(".").pop()?.toLowerCase();
+
+  const iconMap: Record<string, string> = {
+    ts: "TS",
+    tsx: "TX",
+    js: "JS",
+    json: "{}",
+    md: "MD",
+    css: "CS",
+    html: "HT",
+  };
+
+  const label = iconMap[ext || ""] || "F";
+
+  return (
+    <Badge variant="outline" className="text-[10px] font-mono px-1 py-0 h-5 w-6 flex items-center justify-center">
+      {label}
+    </Badge>
+  );
+}

package/components/EvalsPage.tsx

@@ -0,0 +1,80 @@
+import Link from "next/link";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import { ShowMore } from "@/components/ShowMore";
+
+interface EvalInfo {
+  name: string;
+  prompt: string;
+  files: string[];
+}
+
+interface EvalsPageProps {
+  evals: EvalInfo[];
+  total: number;
+  showAll: boolean;
+}
+
+export function EvalsPage({ evals, total, showAll }: EvalsPageProps) {
+  return (
+    <div className="space-y-6">
+      <div>
+        <h1 className="text-2xl font-bold tracking-tight">Evals</h1>
+        <p className="text-muted-foreground mt-1">
+          Browse all eval fixtures in your project.
+        </p>
+      </div>
+
+      {evals.length === 0 ? (
+        <Card>
+          <CardContent className="py-12 text-center">
+            <p className="text-muted-foreground text-lg">No evals found</p>
+            <p className="text-muted-foreground text-sm mt-2">
+              Create evals in your{" "}
+              <code className="text-foreground bg-muted px-1.5 py-0.5 rounded text-xs">
+                evals/
+              </code>{" "}
+              directory.
+            </p>
+          </CardContent>
+        </Card>
+      ) : (
+        <ShowMore total={total} showAllHref={showAll ? undefined : "/evals?all"} className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
+          {evals.map((evalInfo) => (
+            <Link key={evalInfo.name} href={`/evals/${encodeURIComponent(evalInfo.name)}`} className="block cursor-pointer">
+            <Card className="transition-colors hover:bg-muted">
+              <CardHeader>
+                <CardTitle className="text-base">{evalInfo.name}</CardTitle>
+              </CardHeader>
+              <CardContent>
+                {evalInfo.prompt && (
+                  <p className="text-sm text-muted-foreground line-clamp-3 mb-3">
+                    {evalInfo.prompt.slice(0, 200)}
+                    {evalInfo.prompt.length > 200 ? "..." : ""}
+                  </p>
+                )}
+                <div className="flex flex-wrap gap-1">
+                  {evalInfo.files.slice(0, 5).map((file) => (
+                    <Badge
+                      key={file}
+                      variant="secondary"
+                      className="text-xs font-mono"
+                    >
+                      {file}
+                    </Badge>
+                  ))}
+                  {evalInfo.files.length > 5 && (
+                    <Badge variant="outline" className="text-xs">
+                      +{evalInfo.files.length - 5} more
+                    </Badge>
+                  )}
+                </div>
+              </CardContent>
+            </Card>
+            </Link>
+          ))}
+        </ShowMore>
+      )}
+    </div>
+  );
+}