@darkrishabh/bench-ai 1.0.0

package/src/app/api/suite/route.ts

@@ -0,0 +1,314 @@
+import { NextRequest, NextResponse } from "next/server";
+import {
+  ClaudeProvider,
+  OllamaProvider,
+  MinimaxProvider,
+  OpenAICompatibleProvider,
+  createClaudeCLIProvider,
+  createCodexProvider,
+  parseSuiteConfig,
+  runSuite,
+} from "@darkrishabh/bench-ai";
+import type { SuiteConfig, SuiteResult, Provider } from "@darkrishabh/bench-ai";
+import type { LLMInstance } from "@/types";
+import { PRESET_BASE_URLS } from "@/types";
+import type { SuiteJudgeMeta } from "@/lib/suite-judge-meta";
+
+/** Long suite runs: raise on Vercel via plan limits (Pro supports up to 800s). */
+export const maxDuration = 300;
+export const dynamic = "force-dynamic";
+
+function buildProvider(instance: LLMInstance): Provider | null {
+  try {
+    switch (instance.provider) {
+      case "claude": {
+        const apiKey = instance.apiKey?.trim() || process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) return null;
+        return new ClaudeProvider(apiKey, instance.model, {
+          maxTokens: instance.maxTokens,
+          temperature: instance.temperature,
+        });
+      }
+      case "ollama":
+        return new OllamaProvider(
+          instance.baseUrl || process.env.OLLAMA_BASE_URL || "http://localhost:11434",
+          instance.model,
+          { temperature: instance.temperature }
+        );
+      case "minimax": {
+        const apiKey = instance.apiKey?.trim() || process.env.MINIMAX_API_KEY;
+        const groupId = instance.groupId?.trim() || process.env.MINIMAX_GROUP_ID;
+        if (!apiKey || !groupId) return null;
+        return new MinimaxProvider(apiKey, groupId, instance.model, {
+          maxTokens: instance.maxTokens,
+          temperature: instance.temperature,
+        });
+      }
+      case "claude-cli":
+        return createClaudeCLIProvider(instance.model, { timeoutMs: 120_000 });
+      case "codex":
+        return createCodexProvider(instance.model, { timeoutMs: 120_000 });
+      default: {
+        const apiKey = instance.apiKey?.trim() || process.env[`${instance.provider.toUpperCase().replace(/-/g, "_")}_API_KEY`] || "";
+        const baseUrl =
+          instance.baseUrl?.trim() ||
+          PRESET_BASE_URLS[instance.provider as keyof typeof PRESET_BASE_URLS] ||
+          "";
+        if (!baseUrl) return null;
+        return new OpenAICompatibleProvider(instance.provider, baseUrl, apiKey, instance.model, {
+          maxTokens: instance.maxTokens,
+          temperature: instance.temperature,
+        });
+      }
+    }
+  } catch {
+    return null;
+  }
+}
+
+function countLlmRubricAssertions(config: SuiteConfig): number {
+  let n = 0;
+  for (const t of config.tests) {
+    for (const a of t.assert ?? []) {
+      if (a.type === "llm-rubric") n++;
+    }
+  }
+  return n;
+}
+
+function buildSuiteJudgeMeta(
+  judgeBody: {
+    mode?: string;
+    ollamaBaseUrl?: string;
+    ollamaModel?: string;
+  } | undefined,
+  judgeProvider: Provider | undefined,
+  rubricCount: number
+): SuiteJudgeMeta {
+  const mode = judgeBody?.mode ?? "auto";
+
+  if (!judgeBody || mode === "none") {
+    return {
+      rubricAssertionCount: rubricCount,
+      willEvaluateRubrics: false,
+      judgeMode: mode,
+      judgeBackend: "off",
+      summary:
+        rubricCount > 0
+          ? `Judge is off (mode: none). ${rubricCount} llm-rubric assertion(s) did not call an LLM — enable a judge under Settings → Judge.`
+          : "No llm-rubric assertions in this suite; judge settings are unused for this run.",
+    };
+  }
+
+  if (mode === "ollama") {
+    const url = judgeBody.ollamaBaseUrl?.trim() || process.env.OLLAMA_BASE_URL || "http://localhost:11434";
+    const model = judgeBody.ollamaModel?.trim() || "llama3.2";
+    const label = `ollama/${model}`;
+    const will = rubricCount > 0;
+    return {
+      rubricAssertionCount: rubricCount,
+      willEvaluateRubrics: will,
+      judgeMode: mode,
+      judgeBackend: "ollama",
+      judgeLabel: label,
+      summary: will
+        ? `Rubric judge is Ollama at ${url} (${model}). Each llm-rubric triggers a judge request — see run log lines "→ Judge LLM".`
+        : `Judge is set to Ollama (${model}) but this suite has no llm-rubric assertions.`,
+    };
+  }
+
+  if (judgeProvider) {
+    const label = `${judgeProvider.name}/${judgeProvider.model}`;
+    const will = rubricCount > 0;
+    return {
+      rubricAssertionCount: rubricCount,
+      willEvaluateRubrics: will,
+      judgeMode: mode,
+      judgeBackend: "claude",
+      judgeLabel: label,
+      summary: will
+        ? `Rubric judge is Claude (${label}). Each llm-rubric triggers a judge request — see run log lines "→ Judge LLM".`
+        : `Claude judge is configured (${label}) but this suite has no llm-rubric assertions.`,
+    };
+  }
+
+  return {
+    rubricAssertionCount: rubricCount,
+    willEvaluateRubrics: false,
+    judgeMode: mode,
+    judgeBackend: "off",
+    summary:
+      rubricCount > 0
+        ? `No Anthropic API key available for the judge (${mode} mode). Add your key under Settings → Secrets for the judge variable, or set ANTHROPIC_API_KEY on the server. llm-rubric did not call Claude.`
+        : "No llm-rubric assertions; Claude judge key is still missing if you add rubrics later.",
+  };
+}
+
+function buildJudgeProvider(judge?: {
+  mode?: string;
+  anthropicApiKey?: string;
+  claudeModel?: string;
+  ollamaBaseUrl?: string;
+  ollamaModel?: string;
+}): Provider | undefined {
+  if (!judge || judge.mode === "none") return undefined;
+
+  if (judge.mode === "ollama") {
+    return new OllamaProvider(
+      judge.ollamaBaseUrl?.trim() || process.env.OLLAMA_BASE_URL || "http://localhost:11434",
+      (judge.ollamaModel || "llama3.2").trim(),
+      {}
+    );
+  }
+
+  const fromBody = judge.anthropicApiKey?.trim();
+  const fromEnv = process.env.ANTHROPIC_API_KEY?.trim();
+  const key = fromBody || (judge.mode === "auto" || judge.mode === "claude" ? fromEnv : undefined);
+  if (!key) return undefined;
+
+  const model = judge.claudeModel?.trim() || "claude-3-5-haiku-20241022";
+  return new ClaudeProvider(key, model);
+}
+
+type JudgeBody = {
+  mode?: string;
+  anthropicApiKey?: string;
+  claudeModel?: string;
+  ollamaBaseUrl?: string;
+  ollamaModel?: string;
+};
+
+type SuitePostBody = {
+  yaml?: string;
+  instances?: LLMInstance[];
+  judge?: JudgeBody;
+  stream?: boolean;
+};
+
+type PreparedSuite = {
+  config: SuiteConfig;
+  providers: Provider[];
+  judgeProvider: Provider | undefined;
+  judgeMeta: SuiteJudgeMeta;
+  rubricCount: number;
+};
+
+function prepareSuite(body: SuitePostBody): PreparedSuite | NextResponse {
+  if (!body.yaml?.trim()) {
+    return NextResponse.json({ error: "yaml is required" }, { status: 400 });
+  }
+
+  let config: SuiteConfig;
+  try {
+    config = parseSuiteConfig(body.yaml);
+  } catch (err) {
+    return NextResponse.json(
+      { error: `Invalid suite config: ${err instanceof Error ? err.message : String(err)}` },
+      { status: 400 }
+    );
+  }
+
+  const enabled = (body.instances ?? []).filter((i) => i.enabled);
+  if (enabled.length === 0) {
+    return NextResponse.json({ error: "no enabled instances" }, { status: 400 });
+  }
+
+  const providers = enabled.map(buildProvider).filter((p): p is Provider => p !== null);
+  if (providers.length === 0) {
+    return NextResponse.json({ error: "no providers could be constructed — check credentials" }, { status: 400 });
+  }
+
+  const rubricCount = countLlmRubricAssertions(config);
+  const judgeProvider = buildJudgeProvider(body.judge);
+  const judgeMeta = buildSuiteJudgeMeta(body.judge, judgeProvider, rubricCount);
+
+  return { config, providers, judgeProvider, judgeMeta, rubricCount };
+}
+
+async function executeSuite(
+  prepared: PreparedSuite,
+  emitLine?: (line: string) => void
+): Promise<{ result: SuiteResult; runLog: string[]; judgeMeta: SuiteJudgeMeta }> {
+  const { config, providers, judgeProvider, judgeMeta, rubricCount } = prepared;
+  const runLog: string[] = [];
+  const log = (message: string) => {
+    const line = `[${new Date().toISOString().slice(11, 23)}] ${message}`;
+    runLog.push(line);
+    emitLine?.(line);
+  };
+
+  if (judgeMeta.willEvaluateRubrics && judgeMeta.judgeLabel) {
+    log(`Rubric judge ACTIVE (${judgeMeta.judgeBackend}): ${judgeMeta.judgeLabel} · ${rubricCount} llm-rubric step(s)`);
+  } else if (rubricCount > 0) {
+    log(`Rubric judge INACTIVE — ${rubricCount} llm-rubric step(s) will not call an LLM (see summary below)`);
+  } else {
+    log(`No llm-rubric assertions in suite — judge not used`);
+  }
+
+  const result = await runSuite({
+    config,
+    providers,
+    judgeProvider,
+    onLog: log,
+  });
+
+  return { result, runLog, judgeMeta };
+}
+
+function sseChunk(obj: unknown): Uint8Array {
+  return new TextEncoder().encode(`data: ${JSON.stringify(obj)}\n\n`);
+}
+
+export async function POST(req: NextRequest) {
+  const body = (await req.json()) as SuitePostBody;
+  const prepared = prepareSuite(body);
+  if (prepared instanceof NextResponse) return prepared;
+
+  if (body.stream === true) {
+    const stream = new ReadableStream({
+      async start(controller) {
+        try {
+          const out = await executeSuite(prepared, (line) => {
+            controller.enqueue(sseChunk({ type: "log", line }));
+          });
+          controller.enqueue(
+            sseChunk({
+              type: "done",
+              result: out.result,
+              runLog: out.runLog,
+              judgeMeta: out.judgeMeta,
+            })
+          );
+        } catch (e) {
+          controller.enqueue(
+            sseChunk({
+              type: "error",
+              message: e instanceof Error ? e.message : String(e),
+            })
+          );
+        } finally {
+          controller.close();
+        }
+      },
+    });
+
+    return new Response(stream, {
+      headers: {
+        "Content-Type": "text/event-stream; charset=utf-8",
+        "Cache-Control": "no-cache, no-transform",
+        Connection: "keep-alive",
+        "X-Accel-Buffering": "no",
+      },
+    });
+  }
+
+  try {
+    const out = await executeSuite(prepared);
+    return NextResponse.json({ ...out.result, runLog: out.runLog, judgeMeta: out.judgeMeta });
+  } catch (e) {
+    return NextResponse.json(
+      { error: e instanceof Error ? e.message : String(e) },
+      { status: 500 }
+    );
+  }
+}

package/src/app/globals.css

@@ -0,0 +1,215 @@
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+
+:root {
+  /* Canvas & surfaces — Bench AI light shell */
+  --bg:              #f9f9f9;
+  --bg-gradient:     #f9f9f9;
+  --surface:         #ffffff;
+  --surface-subtle:  #fafafa;
+  --surface-muted:   #f4f4f5;
+  --surface-hover:   #f0f0f1;
+
+  /* Borders */
+  --border:          rgba(15, 23, 42, 0.07);
+  --border-strong:   rgba(15, 23, 42, 0.11);
+  --border-focus:    rgba(15, 23, 42, 0.18);
+
+  /* Text */
+  --text-1:          #0a0a0a;
+  --text-2:          #525252;
+  --text-3:          #a3a3a3;
+
+  /* Brand / actions */
+  --accent:          #2563eb;
+  --accent-hover:    #1d4ed8;
+  --accent-muted:    #3b82f6;
+  --accent-subtle:   #eff6ff;
+  --accent-text:     #1d4ed8;
+  --ring:            #2563eb;
+  --ring-subtle:     rgba(37, 99, 235, 0.2);
+
+  /* Quick comparison metric hues */
+  --chart-latency:   #16a34a;
+  --chart-tokens:    #7c3aed;
+  --chart-cost:      #b45309;
+
+  /* Response code panels */
+  --code-bg:         #faf6f0;
+  --code-border:     rgba(120, 83, 50, 0.14);
+
+  /* Semantic */
+  --green:           #047857;
+  --green-subtle:    #ecfdf5;
+  --red:             #b91c1c;
+  --red-subtle:      #fef2f2;
+  --amber:           #b45309;
+  --amber-subtle:    #fffbeb;
+
+  /* Provider accents (muted for headers) */
+  --claude:          #5b21b6;
+  --claude-subtle:   #f5f3ff;
+  --claude-border:   #e9d5ff;
+  --ollama:          #047857;
+  --ollama-subtle:   #ecfdf5;
+  --ollama-border:   #a7f3d0;
+  --minimax:         #c2410c;
+  --minimax-subtle:  #fff7ed;
+  --minimax-border:  #fed7aa;
+  --openai:          #0d9488;
+  --openai-subtle:   #f0fdfa;
+  --openai-border:   #99f6e4;
+  --groq:            #c2410c;
+  --groq-subtle:     #fff7ed;
+  --groq-border:     #fed7aa;
+  --openrouter:      #4f46e5;
+  --openrouter-subtle: #eef2ff;
+  --openrouter-border: #c7d2fe;
+  --nvidia-nim:      #4d7c0f;
+  --nvidia-nim-subtle: #f7fee7;
+  --nvidia-nim-border: #d9f99d;
+  --together:        #1d4ed8;
+  --together-subtle: #eff6ff;
+  --together-border: #bfdbfe;
+  --perplexity:      #0f766e;
+  --perplexity-subtle: #f0fdfa;
+  --perplexity-border: #99f6e4;
+  --custom:          #64748b;
+  --custom-subtle:   #f8fafc;
+  --custom-border:   #e2e8f0;
+
+  /* Elevation */
+  --shadow-xs:  0 1px 2px rgba(15, 23, 42, 0.035);
+  --shadow-sm:  0 1px 3px rgba(15, 23, 42, 0.05);
+  --shadow-md:  0 4px 20px rgba(15, 23, 42, 0.06), 0 1px 3px rgba(15, 23, 42, 0.04);
+  --shadow-lg:  0 24px 48px rgba(15, 23, 42, 0.1), 0 8px 16px rgba(15, 23, 42, 0.05);
+  --shadow-drawer: -8px 0 40px rgba(15, 23, 42, 0.08);
+
+  --r-sm:  6px;
+  --r-md:  10px;
+  --r-lg:  14px;
+  --r-xl:  18px;
+  --r-2xl: 22px;
+
+  --font: var(--font-inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif);
+  --font-mono: "SF Mono", "JetBrains Mono", "Fira Code", ui-monospace, monospace;
+}
+
+html { font-size: 15px; }
+
+body {
+  font-family: var(--font);
+  background: var(--bg);
+  color: var(--text-1);
+  line-height: 1.55;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+/* Focus — consistent, accessible */
+button:focus-visible,
+a:focus-visible {
+  outline: 2px solid var(--ring);
+  outline-offset: 2px;
+}
+
+input:focus-visible,
+textarea:focus-visible,
+select:focus-visible {
+  outline: none;
+  border-color: var(--ring) !important;
+  box-shadow: 0 0 0 3px var(--ring-subtle);
+}
+
+/* ── Markdown ─────────────────────────────────────────────────── */
+.md {
+  font-size: 0.925rem;
+  line-height: 1.72;
+  color: var(--text-1);
+  word-break: break-word;
+}
+.md > *:last-child { margin-bottom: 0; }
+
+.md p { margin-bottom: 0.85em; }
+
+.md h1, .md h2, .md h3, .md h4, .md h5, .md h6 {
+  font-weight: 600;
+  line-height: 1.35;
+  margin: 1.35em 0 0.5em;
+  color: var(--text-1);
+  letter-spacing: -0.02em;
+}
+.md h1 { font-size: 1.2em; }
+.md h2 { font-size: 1.08em; }
+.md h3 { font-size: 1em; }
+.md h4, .md h5, .md h6 { font-size: 0.92em; }
+.md h1:first-child,
+.md h2:first-child,
+.md h3:first-child { margin-top: 0; }
+
+.md ul, .md ol { padding-left: 1.45em; margin-bottom: 0.85em; }
+.md li { margin-bottom: 0.35em; line-height: 1.65; }
+.md li:last-child { margin-bottom: 0; }
+.md li > ul, .md li > ol { margin-top: 0.28em; margin-bottom: 0; }
+
+.md code {
+  font-family: var(--font-mono);
+  font-size: 0.84em;
+  background: color-mix(in srgb, var(--code-bg) 75%, var(--surface));
+  border: 1px solid var(--code-border);
+  border-radius: 5px;
+  padding: 0.12em 0.38em;
+  color: var(--text-1);
+}
+
+.md pre {
+  background: var(--code-bg);
+  border: 1px solid var(--code-border);
+  border-radius: var(--r-md);
+  padding: 1rem 1.1rem;
+  overflow-x: auto;
+  margin-bottom: 0.85em;
+}
+.md pre code {
+  background: none;
+  border: none;
+  padding: 0;
+  font-size: 0.82em;
+  line-height: 1.65;
+  color: var(--text-1);
+}
+
+.md blockquote {
+  border-left: 3px solid var(--border-strong);
+  padding: 0.15em 0 0.15em 1em;
+  color: var(--text-2);
+  margin: 0.85em 0;
+  font-style: italic;
+}
+
+.md table {
+  width: 100%;
+  border-collapse: collapse;
+  margin-bottom: 0.85em;
+  font-size: 0.88em;
+}
+.md thead tr { background: var(--surface-subtle); }
+.md th {
+  padding: 0.55rem 0.8rem;
+  text-align: left;
+  font-weight: 600;
+  border-bottom: 1px solid var(--border-strong);
+  white-space: nowrap;
+  color: var(--text-1);
+}
+.md td {
+  padding: 0.5rem 0.8rem;
+  border-bottom: 1px solid var(--border);
+}
+.md tr:last-child td { border-bottom: none; }
+.md tbody tr:hover { background: var(--surface-hover); }
+
+.md a { color: var(--accent); text-decoration: none; font-weight: 500; }
+.md a:hover { text-decoration: underline; }
+.md strong { font-weight: 600; }
+.md em { font-style: italic; color: var(--text-2); }
+.md hr { border: none; border-top: 1px solid var(--border); margin: 1.25em 0; }

package/src/app/icon.svg

@@ -0,0 +1,14 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" role="img" aria-label="Bench AI">
+  <defs>
+    <linearGradient id="bg" x1="4" y1="2" x2="28" y2="30" gradientUnits="userSpaceOnUse">
+      <stop stop-color="#1e40af"/>
+      <stop offset="1" stop-color="#172554"/>
+    </linearGradient>
+  </defs>
+  <rect width="32" height="32" rx="8" fill="url(#bg)"/>
+  <!-- Side-by-side panels = compare -->
+  <rect x="7" y="9" width="7" height="14" rx="2" fill="#ffffff" fill-opacity="0.95"/>
+  <rect x="18" y="9" width="7" height="14" rx="2" fill="#ffffff" fill-opacity="0.78"/>
+  <!-- Subtle divider -->
+  <rect x="15" y="8" width="2" height="16" rx="1" fill="#ffffff" fill-opacity="0.35"/>
+</svg>

package/src/app/layout.tsx

@@ -0,0 +1,44 @@
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+import { BRAND_NAME, BRAND_TAGLINE } from "../lib/brand";
+
+const inter = Inter({
+  subsets: ["latin"],
+  variable: "--font-inter",
+  display: "swap",
+});
+
+const siteUrl = "https://bench-ai-web.vercel.app";
+
+export const metadata: Metadata = {
+  metadataBase: new URL(siteUrl),
+  title: BRAND_NAME,
+  description: BRAND_TAGLINE,
+  openGraph: {
+    title: BRAND_NAME,
+    description: BRAND_TAGLINE,
+    url: "/",
+    siteName: BRAND_NAME,
+    type: "website",
+  },
+  twitter: {
+    card: "summary_large_image",
+    title: BRAND_NAME,
+    description: BRAND_TAGLINE,
+  },
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en" className={inter.variable}>
+      <body style={{ fontFamily: "var(--font-inter, var(--font))" }}>
+        {children}
+      </body>
+    </html>
+  );
+}

package/src/app/opengraph-image.tsx

@@ -0,0 +1,73 @@
+import { ImageResponse } from "next/og";
+import { BRAND_NAME, BRAND_TAGLINE } from "../lib/brand";
+
+export const alt = `${BRAND_NAME} — ${BRAND_TAGLINE}`;
+export const size = { width: 1200, height: 630 };
+export const contentType = "image/png";
+
+/** Social preview card; copy matches `../lib/brand.ts` (not static PNG). */
+export default function OpenGraphImage() {
+  return new ImageResponse(
+    (
+      <div
+        style={{
+          height: "100%",
+          width: "100%",
+          display: "flex",
+          flexDirection: "column",
+          alignItems: "flex-start",
+          justifyContent: "center",
+          backgroundColor: "#f9f9f9",
+          backgroundImage:
+            "linear-gradient(135deg, #f9f9f9 0%, #ffffff 45%, #eff6ff 100%)",
+          padding: 72,
+        }}
+      >
+        <div
+          style={{
+            width: 120,
+            height: 6,
+            borderRadius: 3,
+            backgroundColor: "#2563eb",
+            marginBottom: 36,
+          }}
+        />
+        <div
+          style={{
+            display: "flex",
+            flexDirection: "column",
+            gap: 20,
+          }}
+        >
+          <div
+            style={{
+              fontSize: 76,
+              fontWeight: 700,
+              color: "#0a0a0a",
+              letterSpacing: "-0.03em",
+              lineHeight: 1.05,
+              fontFamily:
+                'ui-sans-serif, system-ui, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
+            }}
+          >
+            {BRAND_NAME}
+          </div>
+          <div
+            style={{
+              fontSize: 30,
+              fontWeight: 500,
+              color: "#525252",
+              maxWidth: 920,
+              lineHeight: 1.35,
+              fontFamily:
+                'ui-sans-serif, system-ui, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
+            }}
+          >
+            {BRAND_TAGLINE}
+          </div>
+        </div>
+      </div>
+    ),
+    { ...size }
+  );
+}