@vercel/agent-eval-playground 0.0.1

package/components/ui/separator.tsx

@@ -0,0 +1,28 @@
+"use client"
+
+import * as React from "react"
+import { Separator as SeparatorPrimitive } from "radix-ui"
+
+import { cn } from "@/lib/utils"
+
+function Separator({
+  className,
+  orientation = "horizontal",
+  decorative = true,
+  ...props
+}: React.ComponentProps<typeof SeparatorPrimitive.Root>) {
+  return (
+    <SeparatorPrimitive.Root
+      data-slot="separator"
+      decorative={decorative}
+      orientation={orientation}
+      className={cn(
+        "bg-border shrink-0 data-horizontal:h-px data-horizontal:w-full data-vertical:w-px data-vertical:self-stretch",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+export { Separator }

package/components/ui/table.tsx

@@ -0,0 +1,116 @@
+"use client"
+
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Table({ className, ...props }: React.ComponentProps<"table">) {
+  return (
+    <div
+      data-slot="table-container"
+      className="relative w-full overflow-x-auto"
+    >
+      <table
+        data-slot="table"
+        className={cn("w-full caption-bottom text-sm", className)}
+        {...props}
+      />
+    </div>
+  )
+}
+
+function TableHeader({ className, ...props }: React.ComponentProps<"thead">) {
+  return (
+    <thead
+      data-slot="table-header"
+      className={cn("[&_tr]:border-b", className)}
+      {...props}
+    />
+  )
+}
+
+function TableBody({ className, ...props }: React.ComponentProps<"tbody">) {
+  return (
+    <tbody
+      data-slot="table-body"
+      className={cn("[&_tr:last-child]:border-0", className)}
+      {...props}
+    />
+  )
+}
+
+function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
+  return (
+    <tfoot
+      data-slot="table-footer"
+      className={cn(
+        "bg-muted/50 border-t font-medium [&>tr]:last:border-b-0",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
+  return (
+    <tr
+      data-slot="table-row"
+      className={cn(
+        "hover:bg-muted/50 data-[state=selected]:bg-muted border-b transition-colors",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function TableHead({ className, ...props }: React.ComponentProps<"th">) {
+  return (
+    <th
+      data-slot="table-head"
+      className={cn(
+        "text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function TableCell({ className, ...props }: React.ComponentProps<"td">) {
+  return (
+    <td
+      data-slot="table-cell"
+      className={cn(
+        "p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function TableCaption({
+  className,
+  ...props
+}: React.ComponentProps<"caption">) {
+  return (
+    <caption
+      data-slot="table-caption"
+      className={cn("text-muted-foreground mt-4 text-sm", className)}
+      {...props}
+    />
+  )
+}
+
+export {
+  Table,
+  TableHeader,
+  TableBody,
+  TableFooter,
+  TableHead,
+  TableRow,
+  TableCell,
+  TableCaption,
+}

package/components/ui/tabs.tsx

@@ -0,0 +1,91 @@
+"use client"
+
+import * as React from "react"
+import { cva, type VariantProps } from "class-variance-authority"
+import { Tabs as TabsPrimitive } from "radix-ui"
+
+import { cn } from "@/lib/utils"
+
+function Tabs({
+  className,
+  orientation = "horizontal",
+  ...props
+}: React.ComponentProps<typeof TabsPrimitive.Root>) {
+  return (
+    <TabsPrimitive.Root
+      data-slot="tabs"
+      data-orientation={orientation}
+      orientation={orientation}
+      className={cn(
+        "group/tabs flex gap-2 data-[orientation=horizontal]:flex-col",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+const tabsListVariants = cva(
+  "rounded-lg p-[3px] group-data-[orientation=horizontal]/tabs:h-9 data-[variant=line]:rounded-none group/tabs-list text-muted-foreground inline-flex w-fit items-center justify-center group-data-[orientation=vertical]/tabs:h-fit group-data-[orientation=vertical]/tabs:flex-col",
+  {
+    variants: {
+      variant: {
+        default: "bg-muted",
+        line: "gap-1 bg-transparent",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+)
+
+function TabsList({
+  className,
+  variant = "default",
+  ...props
+}: React.ComponentProps<typeof TabsPrimitive.List> &
+  VariantProps<typeof tabsListVariants>) {
+  return (
+    <TabsPrimitive.List
+      data-slot="tabs-list"
+      data-variant={variant}
+      className={cn(tabsListVariants({ variant }), className)}
+      {...props}
+    />
+  )
+}
+
+function TabsTrigger({
+  className,
+  ...props
+}: React.ComponentProps<typeof TabsPrimitive.Trigger>) {
+  return (
+    <TabsPrimitive.Trigger
+      data-slot="tabs-trigger"
+      className={cn(
+        "cursor-pointer focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:outline-ring text-foreground/60 hover:text-foreground dark:text-muted-foreground dark:hover:text-foreground relative inline-flex h-[calc(100%-1px)] flex-1 items-center justify-center gap-1.5 rounded-md border border-transparent px-2 py-1 text-sm font-medium whitespace-nowrap transition-all group-data-[orientation=vertical]/tabs:w-full group-data-[orientation=vertical]/tabs:justify-start focus-visible:ring-[3px] focus-visible:outline-1 disabled:pointer-events-none disabled:opacity-50 group-data-[variant=default]/tabs-list:data-[state=active]:shadow-sm group-data-[variant=line]/tabs-list:data-[state=active]:shadow-none [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
+        "group-data-[variant=line]/tabs-list:bg-transparent group-data-[variant=line]/tabs-list:data-[state=active]:bg-transparent dark:group-data-[variant=line]/tabs-list:data-[state=active]:border-transparent dark:group-data-[variant=line]/tabs-list:data-[state=active]:bg-transparent",
+        "data-[state=active]:bg-background dark:data-[state=active]:text-foreground dark:data-[state=active]:border-input dark:data-[state=active]:bg-input/30 data-[state=active]:text-foreground",
+        "after:bg-foreground after:absolute after:opacity-0 after:transition-opacity group-data-[orientation=horizontal]/tabs:after:inset-x-0 group-data-[orientation=horizontal]/tabs:after:bottom-[-5px] group-data-[orientation=horizontal]/tabs:after:h-0.5 group-data-[orientation=vertical]/tabs:after:inset-y-0 group-data-[orientation=vertical]/tabs:after:-right-1 group-data-[orientation=vertical]/tabs:after:w-0.5 group-data-[variant=line]/tabs-list:data-[state=active]:after:opacity-100",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+function TabsContent({
+  className,
+  ...props
+}: React.ComponentProps<typeof TabsPrimitive.Content>) {
+  return (
+    <TabsPrimitive.Content
+      data-slot="tabs-content"
+      className={cn("flex-1 outline-none", className)}
+      {...props}
+    />
+  )
+}
+
+export { Tabs, TabsList, TabsTrigger, TabsContent, tabsListVariants }

package/components/ui/textarea.tsx

@@ -0,0 +1,18 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
+  return (
+    <textarea
+      data-slot="textarea"
+      className={cn(
+        "border-input bg-input/20 dark:bg-input/30 focus-visible:border-ring focus-visible:ring-ring/30 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:aria-invalid:border-destructive/50 resize-none rounded-md border px-2 py-2 text-sm transition-colors focus-visible:ring-2 aria-invalid:ring-2 md:text-xs/relaxed placeholder:text-muted-foreground flex field-sizing-content min-h-16 w-full outline-none disabled:cursor-not-allowed disabled:opacity-50",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+export { Textarea }

package/components/ui/tooltip.tsx

@@ -0,0 +1,57 @@
+"use client"
+
+import * as React from "react"
+import { Tooltip as TooltipPrimitive } from "radix-ui"
+
+import { cn } from "@/lib/utils"
+
+function TooltipProvider({
+  delayDuration = 0,
+  ...props
+}: React.ComponentProps<typeof TooltipPrimitive.Provider>) {
+  return (
+    <TooltipPrimitive.Provider
+      data-slot="tooltip-provider"
+      delayDuration={delayDuration}
+      {...props}
+    />
+  )
+}
+
+function Tooltip({
+  ...props
+}: React.ComponentProps<typeof TooltipPrimitive.Root>) {
+  return <TooltipPrimitive.Root data-slot="tooltip" {...props} />
+}
+
+function TooltipTrigger({
+  ...props
+}: React.ComponentProps<typeof TooltipPrimitive.Trigger>) {
+  return <TooltipPrimitive.Trigger data-slot="tooltip-trigger" {...props} />
+}
+
+function TooltipContent({
+  className,
+  sideOffset = 0,
+  children,
+  ...props
+}: React.ComponentProps<typeof TooltipPrimitive.Content>) {
+  return (
+    <TooltipPrimitive.Portal>
+      <TooltipPrimitive.Content
+        data-slot="tooltip-content"
+        sideOffset={sideOffset}
+        className={cn(
+          "bg-foreground text-background animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-md px-3 py-1.5 text-xs text-balance",
+          className
+        )}
+        {...props}
+      >
+        {children}
+        <TooltipPrimitive.Arrow className="bg-foreground fill-foreground z-50 size-2.5 translate-y-[calc(-50%_-_2px)] rotate-45 rounded-[2px]" />
+      </TooltipPrimitive.Content>
+    </TooltipPrimitive.Portal>
+  )
+}
+
+export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }

package/components.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "radix-mira",
+  "rsc": true,
+  "tsx": true,
+  "tailwind": {
+    "config": "",
+    "css": "app/globals.css",
+    "baseColor": "zinc",
+    "cssVariables": true,
+    "prefix": ""
+  },
+  "iconLibrary": "remixicon",
+  "rtl": false,
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "menuColor": "inverted",
+  "menuAccent": "subtle",
+  "registries": {}
+}

package/lib/data.ts

@@ -0,0 +1,297 @@
+/**
+ * Server-side data access for the playground.
+ * Reads JSON files from the results/ and evals/ directories.
+ * Directory paths are provided via RESULTS_DIR and EVALS_DIR env vars.
+ */
+
+import { readdirSync, readFileSync, existsSync } from "fs";
+import { join, resolve } from "path";
+
+function getResultsDir(): string {
+  return resolve(process.env.RESULTS_DIR || "./results");
+}
+
+function getEvalsDir(): string {
+  return resolve(process.env.EVALS_DIR || "./evals");
+}
+
+/** List experiments from the results directory. Pass limit to cap expensive per-item reads. */
+export function listExperiments(limit?: number) {
+  const resultsDir = getResultsDir();
+
+  if (!existsSync(resultsDir)) {
+    return { items: [], total: 0 };
+  }
+
+  const entries = readdirSync(resultsDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+
+  const total = entries.length;
+  const toProcess = limit ? entries.slice(0, limit) : entries;
+
+  const items = toProcess.map((name) => {
+    const expDir = join(resultsDir, name);
+    const timestamps = readdirSync(expDir, { withFileTypes: true })
+      .filter((e) => e.isDirectory())
+      .map((e) => e.name)
+      .sort()
+      .reverse();
+
+    let latestPassRate: number | undefined;
+    let latestTotalRuns = 0;
+    let latestPassedRuns = 0;
+
+    if (timestamps.length > 0) {
+      const latestDir = join(expDir, timestamps[0]);
+      const evalDirs = readdirSync(latestDir, { withFileTypes: true }).filter(
+        (e) => e.isDirectory()
+      );
+
+      for (const evalDir of evalDirs) {
+        const summaryPath = join(latestDir, evalDir.name, "summary.json");
+        if (existsSync(summaryPath)) {
+          try {
+            const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+            latestTotalRuns += summary.totalRuns ?? 0;
+            latestPassedRuns += summary.passedRuns ?? 0;
+          } catch {
+            // Skip invalid summary files
+          }
+        }
+      }
+
+      if (latestTotalRuns > 0) {
+        latestPassRate = (latestPassedRuns / latestTotalRuns) * 100;
+      }
+    }
+
+    return {
+      name,
+      timestamps,
+      latestTimestamp: timestamps[0] ?? null,
+      latestPassRate,
+      latestTotalRuns,
+      latestPassedRuns,
+    };
+  });
+
+  return { items, total };
+}
+
+/** Get timestamps for a specific experiment */
+export function getExperiment(name: string) {
+  const expDir = join(getResultsDir(), name);
+
+  if (!existsSync(expDir)) {
+    return null;
+  }
+
+  const timestamps = readdirSync(expDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name)
+    .sort()
+    .reverse();
+
+  return { name, timestamps, latestTimestamp: timestamps[0] ?? null };
+}
+
+/** Get full experiment detail for a specific timestamp */
+export function getExperimentDetail(name: string, timestamp: string) {
+  const runDir = join(getResultsDir(), name, timestamp);
+
+  if (!existsSync(runDir)) {
+    return null;
+  }
+
+  const evalDirs = readdirSync(runDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+
+  const evals = evalDirs.map((evalName) => {
+    const evalDir = join(runDir, evalName);
+    const summaryPath = join(evalDir, "summary.json");
+
+    let summary = {
+      totalRuns: 0,
+      passedRuns: 0,
+      passRate: "0%",
+      meanDuration: 0,
+    };
+    if (existsSync(summaryPath)) {
+      try {
+        summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
+      } catch {
+        // Use defaults
+      }
+    }
+
+    // List run directories
+    const runDirs = readdirSync(evalDir, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && e.name.startsWith("run-"))
+      .map((e) => e.name)
+      .sort();
+
+    // Read each run's result.json
+    const runs = runDirs.map((runDirName) => {
+      const resultPath = join(evalDir, runDirName, "result.json");
+      let result = null;
+      if (existsSync(resultPath)) {
+        try {
+          result = JSON.parse(readFileSync(resultPath, "utf-8"));
+        } catch {
+          // Skip
+        }
+      }
+      return { name: runDirName, result };
+    });
+
+    return {
+      name: evalName,
+      totalRuns: summary.totalRuns,
+      passedRuns: summary.passedRuns,
+      passRate:
+        typeof summary.passRate === "string"
+          ? parseFloat(summary.passRate)
+          : summary.passRate,
+      meanDuration: summary.meanDuration,
+      runs,
+    };
+  });
+
+  return { name, timestamp, evals };
+}
+
+/** Get result for a specific run */
+export function getRunResult(
+  experiment: string,
+  timestamp: string,
+  evalName: string,
+  run: string
+) {
+  const resultPath = join(
+    getResultsDir(),
+    experiment,
+    timestamp,
+    evalName,
+    run,
+    "result.json"
+  );
+
+  if (!existsSync(resultPath)) {
+    return null;
+  }
+
+  try {
+    return { result: JSON.parse(readFileSync(resultPath, "utf-8")) };
+  } catch {
+    return null;
+  }
+}
+
+/** Get parsed transcript for a specific run */
+export function getTranscript(
+  experiment: string,
+  timestamp: string,
+  evalName: string,
+  run: string
+) {
+  const transcriptPath = join(
+    getResultsDir(),
+    experiment,
+    timestamp,
+    evalName,
+    run,
+    "transcript.json"
+  );
+
+  if (!existsSync(transcriptPath)) {
+    return null;
+  }
+
+  try {
+    return JSON.parse(readFileSync(transcriptPath, "utf-8"));
+  } catch {
+    return null;
+  }
+}
+
+/** List evals from the evals directory. Pass limit to cap per-item reads. */
+export function listEvals(limit?: number) {
+  const evalsDir = getEvalsDir();
+
+  if (!existsSync(evalsDir)) {
+    return { items: [], total: 0 };
+  }
+
+  const entries = readdirSync(evalsDir, { withFileTypes: true })
+    .filter((e) => e.isDirectory())
+    .map((e) => e.name);
+
+  const total = entries.length;
+  const toProcess = limit ? entries.slice(0, limit) : entries;
+
+  const items = toProcess.map((name) => {
+    const evalDir = join(evalsDir, name);
+    const promptPath = join(evalDir, "PROMPT.md");
+    let prompt = "";
+    if (existsSync(promptPath)) {
+      prompt = readFileSync(promptPath, "utf-8");
+    }
+
+    const files = readdirSync(evalDir, { withFileTypes: true })
+      .filter((e) => e.isFile())
+      .map((e) => e.name);
+
+    return { name, prompt, files };
+  });
+
+  return { items, total };
+}
+
+/** Get detail for a specific eval */
+export function getEvalDetail(name: string) {
+  const evalDir = join(getEvalsDir(), name);
+
+  if (!existsSync(evalDir)) {
+    return null;
+  }
+
+  const promptPath = join(evalDir, "PROMPT.md");
+  let prompt = "";
+  if (existsSync(promptPath)) {
+    prompt = readFileSync(promptPath, "utf-8");
+  }
+
+  // Recursively list files
+  const files: string[] = [];
+  function walk(dir: string, prefix: string) {
+    const entries = readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (entry.name === "node_modules" || entry.name === ".git") continue;
+      const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        walk(join(dir, entry.name), relativePath);
+      } else {
+        files.push(relativePath);
+      }
+    }
+  }
+  walk(evalDir, "");
+
+  // Read file contents for key files
+  const fileContents: Record<string, string> = {};
+  const keyFiles = ["PROMPT.md", "EVAL.ts", "EVAL.tsx", "package.json"];
+  for (const file of keyFiles) {
+    const filePath = join(evalDir, file);
+    if (existsSync(filePath)) {
+      try {
+        fileContents[file] = readFileSync(filePath, "utf-8");
+      } catch {
+        // Skip unreadable files
+      }
+    }
+  }
+
+  return { name, prompt, files, fileContents };
+}