@darkrishabh/bench-ai 1.0.0

package/dist/engine/suite-loader.d.ts

@@ -0,0 +1,11 @@
+import type { SuiteConfig } from "./types.js";
+/**
+ * Interpolate {{varName}} placeholders in a template string.
+ */
+export declare function interpolate(template: string, vars: Record<string, string>): string;
+/**
+ * Parse a YAML string into a validated SuiteConfig.
+ * Throws a descriptive error if required fields are missing.
+ */
+export declare function parseSuiteConfig(yaml: string): SuiteConfig;
+//# sourceMappingURL=suite-loader.d.ts.map

package/dist/engine/suite-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite-loader.d.ts","sourceRoot":"","sources":["../../src/engine/suite-loader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAuB,MAAM,YAAY,CAAC;AAEnE;;GAEG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAElF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAiC1D"}

package/dist/engine/suite-loader.js

@@ -0,0 +1,75 @@
+import { load } from "js-yaml";
+/**
+ * Interpolate {{varName}} placeholders in a template string.
+ */
+export function interpolate(template, vars) {
+    return template.replace(/\{\{(\w+)\}\}/g, (_, key) => vars[key] ?? `{{${key}}}`);
+}
+/**
+ * Parse a YAML string into a validated SuiteConfig.
+ * Throws a descriptive error if required fields are missing.
+ */
+export function parseSuiteConfig(yaml) {
+    const raw = load(yaml);
+    if (!raw || typeof raw !== "object") {
+        throw new Error("Suite config must be a YAML object");
+    }
+    if (!Array.isArray(raw.prompts) || raw.prompts.length === 0) {
+        throw new Error("Suite config must have at least one entry under `prompts`");
+    }
+    if (!Array.isArray(raw.tests) || raw.tests.length === 0) {
+        throw new Error("Suite config must have at least one entry under `tests`");
+    }
+    const prompts = raw.prompts.map((p, i) => {
+        if (typeof p !== "string")
+            throw new Error(`prompts[${i}] must be a string`);
+        return p;
+    });
+    const tests = raw.tests.map((t, i) => {
+        if (typeof t !== "object" || t === null) {
+            throw new Error(`tests[${i}] must be an object`);
+        }
+        const tc = t;
+        const vars = (tc.vars ?? {});
+        const assert = tc.assert
+            ? validateAssertions(tc.assert, i)
+            : undefined;
+        return { vars, assert };
+    });
+    return { prompts, tests };
+}
+function validateAssertions(raw, testIdx) {
+    return raw.map((a, i) => {
+        if (typeof a !== "object" || a === null) {
+            throw new Error(`tests[${testIdx}].assert[${i}] must be an object`);
+        }
+        const obj = a;
+        const type = obj.type;
+        switch (type) {
+            case "contains":
+            case "not-contains":
+                if (typeof obj.value !== "string") {
+                    throw new Error(`tests[${testIdx}].assert[${i}]: "${type}" requires a string "value"`);
+                }
+                return { type, value: obj.value };
+            case "llm-rubric":
+                if (typeof obj.value !== "string") {
+                    throw new Error(`tests[${testIdx}].assert[${i}]: "llm-rubric" requires a string "value" (the criterion)`);
+                }
+                return { type: "llm-rubric", value: obj.value };
+            case "latency":
+                if (typeof obj.threshold !== "number") {
+                    throw new Error(`tests[${testIdx}].assert[${i}]: "latency" requires a numeric "threshold" (ms)`);
+                }
+                return { type: "latency", threshold: obj.threshold };
+            case "cost":
+                if (typeof obj.threshold !== "number") {
+                    throw new Error(`tests[${testIdx}].assert[${i}]: "cost" requires a numeric "threshold" (USD)`);
+                }
+                return { type: "cost", threshold: obj.threshold };
+            default:
+                throw new Error(`tests[${testIdx}].assert[${i}]: unknown assertion type "${type}"`);
+        }
+    });
+}
+//# sourceMappingURL=suite-loader.js.map

package/dist/engine/suite-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite-loader.js","sourceRoot":"","sources":["../../src/engine/suite-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAG/B;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,IAA4B;IACxE,OAAO,QAAQ,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC;AACnF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAA4B,CAAC;IAElD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAa,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,CAAS,EAAE,EAAE;QAClE,IAAI,OAAO,CAAC,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC;QAC7E,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAgB,GAAG,CAAC,KAAmB,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,CAAS,EAAE,EAAE;QAC/E,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,EAAE,GAAG,CAA4B,CAAC;QACxC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAA2B,CAAC;QACvD,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM;YACtB,CAAC,CAAC,kBAAkB,CAAC,EAAE,CAAC,MAAmB,EAAE,CAAC,CAAC;YAC/C,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAc,EAAE,OAAe;IACzD,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,CAAS,EAAE,EAAE;QACvC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,YAAY,CAAC,qBAAqB,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,GAAG,GAAG,CAA4B,CAAC;QACzC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAc,CAAC;QAEhC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU,CAAC;YAChB,KAAK,cAAc;gBACjB,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,YAAY,CAAC,OAAO,IAAI,6BAA6B,CAAC,CAAC;gBACzF,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAe,CAAC;YAEjD,KAAK,YAAY;gBACf,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;oBAClC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,YAAY,CAAC,2DAA2D,CAAC,CAAC;gBAC5G,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAe,CAAC;YAE/D,KAAK,SAAS;gBACZ,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;oBACtC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,YAAY,CAAC,kDAAkD,CAAC,CAAC;gBACnG,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAe,CAAC;YAEpE,KAAK,MAAM;gBACT,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;oBACtC,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,YAAY,CAAC,gDAAgD,CAAC,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAe,CAAC;YAEjE;gBACE,MAAM,IAAI,KAAK,CAAC,SAAS,OAAO,YAAY,CAAC,8BAA8B,IAAI,GAAG,CAAC,CAAC;QACxF,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/engine/types.d.ts

@@ -0,0 +1,104 @@
+/** Built-in provider names. The type is widened to `string` so that
+ *  OpenAI-compatible providers (openai, groq, openrouter, etc.) can also
+ *  flow through ProviderResult without a cast. */
+export type ProviderName = string;
+export interface ProviderConfig {
+    claude?: {
+        apiKey: string;
+        model?: string;
+    };
+    ollama?: {
+        baseUrl?: string;
+        model?: string;
+    };
+    minimax?: {
+        apiKey: string;
+        groupId: string;
+        model?: string;
+    };
+}
+export interface ProviderResult {
+    provider: ProviderName;
+    model: string;
+    output: string;
+    latencyMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    error?: string;
+}
+export interface DiffResult {
+    prompt: string;
+    results: ProviderResult[];
+    ranAt: string;
+}
+export interface RunOptions {
+    prompt: string;
+    providers: ProviderName[];
+    config: ProviderConfig;
+    runs?: number;
+}
+export type Assertion = {
+    type: "contains";
+    value: string;
+} | {
+    type: "not-contains";
+    value: string;
+} | {
+    type: "llm-rubric";
+    value: string;
+} | {
+    type: "latency";
+    threshold: number;
+} | {
+    type: "cost";
+    threshold: number;
+};
+export interface TestCase {
+    vars?: Record<string, string>;
+    assert?: Assertion[];
+}
+export interface SuiteConfig {
+    /** Prompt templates — use {{varName}} for interpolation */
+    prompts: string[];
+    tests: TestCase[];
+}
+export interface AssertionResult {
+    type: string;
+    pass: boolean;
+    /** 0–1; deterministic assertions are 0 or 1; llm-rubric is 0 or 1 */
+    score: number;
+    reason?: string;
+    /** Set for llm-rubric: the criterion text from the suite YAML */
+    rubricCriterion?: string;
+}
+export interface ProviderTestResult extends ProviderResult {
+    assertions: AssertionResult[];
+    /** true only if every assertion passed */
+    pass: boolean;
+    /** fraction of assertions that passed (0–1) */
+    score: number;
+}
+export interface TestCaseResult {
+    /** Interpolated prompt */
+    prompt: string;
+    vars: Record<string, string>;
+    providerResults: ProviderTestResult[];
+    ranAt: string;
+}
+export interface ProviderSummary {
+    provider: ProviderName;
+    model: string;
+    passed: number;
+    failed: number;
+    total: number;
+    /** fraction of test cases passed (0–1) */
+    score: number;
+    avgLatencyMs: number;
+    totalCostUsd: number;
+}
+export interface SuiteResult {
+    cases: TestCaseResult[];
+    summary: ProviderSummary[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/engine/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/engine/types.ts"],"names":[],"mappings":"AAAA;;kDAEkD;AAClD,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC;AAElC,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,OAAO,CAAC,EAAE;QACR,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAID,MAAM,MAAM,SAAS,GACjB;IAAE,IAAI,EAAE,UAAU,CAAC;IAAK,KAAK,EAAE,MAAM,CAAA;CAAE,GACvC;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACvC;IAAE,IAAI,EAAE,YAAY,CAAC;IAAG,KAAK,EAAE,MAAM,CAAA;CAAE,GACvC;IAAE,IAAI,EAAE,SAAS,CAAC;IAAM,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAS,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAEhD,MAAM,WAAW,QAAQ;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,2DAA2D;IAC3D,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,qEAAqE;IACrE,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,kBAAmB,SAAQ,cAAc;IACxD,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,0CAA0C;IAC1C,IAAI,EAAE,OAAO,CAAC;IACd,+CAA+C;IAC/C,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,eAAe,EAAE,kBAAkB,EAAE,CAAC;IACtC,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,0CAA0C;IAC1C,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,OAAO,EAAE,eAAe,EAAE,CAAC;CAC5B"}

package/dist/engine/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/engine/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/engine/types.ts"],"names":[],"mappings":""}

package/next-env.d.ts

@@ -0,0 +1,6 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+/// <reference path="./.next/types/routes.d.ts" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

package/next.config.ts

@@ -0,0 +1,26 @@
+import type { NextConfig } from "next";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const nextConfig: NextConfig = {
+  outputFileTracingRoot: __dirname,
+  webpack(config, { isServer }) {
+    config.resolve.alias = {
+      ...config.resolve.alias,
+      // Same package: resolve engine via emitted JS (src uses .js specifiers).
+      "@darkrishabh/bench-ai": path.resolve(__dirname, "dist/engine/index.js"),
+    };
+    if (!isServer) {
+      config.resolve.fallback = {
+        ...config.resolve.fallback,
+        child_process: false,
+        util: false,
+      };
+    }
+    return config;
+  },
+};
+
+export default nextConfig;

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@darkrishabh/bench-ai",
+  "version": "1.0.0",
+  "description": "Bench AI — compare LLM outputs across models: CLI, web UI, and YAML eval suites.",
+  "type": "module",
+  "bin": {
+    "bench-ai": "./dist/cli/index.js"
+  },
+  "main": "./dist/engine/index.js",
+  "types": "./dist/engine/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/engine/index.js",
+      "types": "./dist/engine/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "src/app",
+    "src/components",
+    "src/lib",
+    "src/types.ts",
+    "public",
+    "next.config.ts",
+    "next-env.d.ts",
+    "vercel.json",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/darkrishabh/bench-ai.git"
+  },
+  "bugs": {
+    "url": "https://github.com/darkrishabh/bench-ai/issues"
+  },
+  "homepage": "https://github.com/darkrishabh/bench-ai#readme",
+  "scripts": {
+    "predev": "npm run build:engine",
+    "build:engine": "tsc -p tsconfig.build.json",
+    "dev": "concurrently -k -n engine,web \"tsc -p tsconfig.build.json --watch\" \"next dev\"",
+    "build": "npm run build:engine && next build",
+    "start": "next start",
+    "type-check": "npm run build:engine && tsc -p tsconfig.json --noEmit",
+    "clean": "rm -rf dist .next"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.36.3",
+    "commander": "^12.1.0",
+    "ink": "^5.0.1",
+    "js-yaml": "^4.1.1",
+    "next": "^15.5.15",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-markdown": "^9.0.1",
+    "remark-gfm": "^4.0.0"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.10.5",
+    "@types/react": "^18.3.12",
+    "@types/react-dom": "^18.3.1",
+    "concurrently": "^9.1.2",
+    "typescript": "^5.7.3"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "packageManager": "npm@10.9.2"
+}

package/public/icon.svg

@@ -0,0 +1,14 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" role="img" aria-label="Bench AI">
+  <defs>
+    <linearGradient id="bg" x1="4" y1="2" x2="28" y2="30" gradientUnits="userSpaceOnUse">
+      <stop stop-color="#1e40af"/>
+      <stop offset="1" stop-color="#172554"/>
+    </linearGradient>
+  </defs>
+  <rect width="32" height="32" rx="8" fill="url(#bg)"/>
+  <!-- Side-by-side panels = compare -->
+  <rect x="7" y="9" width="7" height="14" rx="2" fill="#ffffff" fill-opacity="0.95"/>
+  <rect x="18" y="9" width="7" height="14" rx="2" fill="#ffffff" fill-opacity="0.78"/>
+  <!-- Subtle divider -->
+  <rect x="15" y="8" width="2" height="16" rx="1" fill="#ffffff" fill-opacity="0.35"/>
+</svg>

package/src/app/api/diff/route.ts

@@ -0,0 +1,135 @@
+import { NextRequest, NextResponse } from "next/server";
+import {
+  ClaudeProvider,
+  OllamaProvider,
+  MinimaxProvider,
+  OpenAICompatibleProvider,
+  createClaudeCLIProvider,
+  createCodexProvider,
+} from "@darkrishabh/bench-ai";
+import type { ProviderResult } from "@darkrishabh/bench-ai";
+import type { LLMInstance, WebProviderResult } from "@/types";
+import { PRESET_BASE_URLS } from "@/types";
+
+export async function POST(req: NextRequest) {
+  const body = (await req.json()) as {
+    prompt?: string;
+    instances?: LLMInstance[];
+  };
+
+  if (!body.prompt?.trim()) {
+    return NextResponse.json({ error: "prompt is required" }, { status: 400 });
+  }
+
+  const enabled = (body.instances ?? []).filter((i) => i.enabled);
+  if (enabled.length === 0) {
+    return NextResponse.json({ error: "no enabled instances" }, { status: 400 });
+  }
+
+  const results: WebProviderResult[] = await Promise.all(
+    enabled.map(async (instance): Promise<WebProviderResult> => {
+      const label = `${instance.provider} / ${instance.model}`;
+      let base: ProviderResult;
+
+      try {
+        switch (instance.provider) {
+          // ── Native providers ────────────────────────────────────────────
+          case "claude": {
+            const apiKey = instance.apiKey?.trim() || process.env.ANTHROPIC_API_KEY;
+            if (!apiKey) throw new Error("No API key — add one in Configure or set ANTHROPIC_API_KEY");
+            const p = new ClaudeProvider(apiKey, instance.model, {
+              maxTokens: instance.maxTokens,
+              temperature: instance.temperature,
+            });
+            base = await p.complete(body.prompt!);
+            break;
+          }
+
+          case "ollama": {
+            const p = new OllamaProvider(
+              instance.baseUrl || process.env.OLLAMA_BASE_URL || "http://localhost:11434",
+              instance.model,
+              { temperature: instance.temperature }
+            );
+            base = await p.complete(body.prompt!);
+            break;
+          }
+
+          case "claude-cli": {
+            const p = createClaudeCLIProvider(instance.model, { timeoutMs: 120_000 });
+            base = await p.complete(body.prompt!);
+            break;
+          }
+
+          case "codex": {
+            const p = createCodexProvider(instance.model, { timeoutMs: 120_000 });
+            base = await p.complete(body.prompt!);
+            break;
+          }
+
+          case "minimax": {
+            const apiKey = instance.apiKey?.trim() || process.env.MINIMAX_API_KEY;
+            const groupId = instance.groupId?.trim() || process.env.MINIMAX_GROUP_ID;
+            if (!apiKey || !groupId) throw new Error("Missing Minimax credentials — add API Key and Group ID in Configure");
+            const p = new MinimaxProvider(apiKey, groupId, instance.model, {
+              maxTokens: instance.maxTokens,
+              temperature: instance.temperature,
+            });
+            base = await p.complete(body.prompt!);
+            break;
+          }
+
+          // ── OpenAI-compatible providers ─────────────────────────────────
+          default: {
+            const apiKey = instance.apiKey?.trim() || process.env[`${instance.provider.toUpperCase().replace(/-/g, "_")}_API_KEY`] || "";
+            const baseUrl =
+              instance.baseUrl?.trim() ||
+              PRESET_BASE_URLS[instance.provider as keyof typeof PRESET_BASE_URLS] ||
+              "";
+
+            if (!baseUrl) throw new Error(`No base URL configured for "${instance.provider}"`);
+
+            // OpenRouter recommends an HTTP-Referer header
+            const extraHeaders: Record<string, string> =
+              instance.provider === "openrouter"
+                ? { "HTTP-Referer": "https://github.com/darkrishabh/bench-ai", "X-Title": "Bench AI" }
+                : {};
+
+            const p = new OpenAICompatibleProvider(
+              instance.provider,
+              baseUrl,
+              apiKey,
+              instance.model,
+              {
+                maxTokens: instance.maxTokens,
+                temperature: instance.temperature,
+                extraHeaders,
+              }
+            );
+            base = await p.complete(body.prompt!);
+            break;
+          }
+        }
+      } catch (err) {
+        base = {
+          provider: instance.provider,
+          model: instance.model,
+          output: "",
+          latencyMs: 0,
+          inputTokens: 0,
+          outputTokens: 0,
+          costUsd: 0,
+          error: err instanceof Error ? err.message : String(err),
+        };
+      }
+
+      return { ...base, instanceId: instance.id, label };
+    })
+  );
+
+  return NextResponse.json({
+    prompt: body.prompt,
+    results,
+    ranAt: new Date().toISOString(),
+  });
+}

package/src/app/api/models/route.ts

@@ -0,0 +1,96 @@
+import { NextRequest, NextResponse } from "next/server";
+import { PRESET_MODELS } from "@/types";
+import { filterOpenAiChatModelIds } from "@/lib/openai-model-list";
+
+/**
+ * GET /api/models?provider=ollama&baseUrl=http://localhost:11434
+ *
+ * Proxies model-listing APIs so the browser doesn't have to deal with CORS
+ * or mixed-content issues when the target is a local server.
+ */
+export async function GET(req: NextRequest) {
+  const { searchParams } = req.nextUrl;
+  const provider = searchParams.get("provider");
+  const baseUrl = (searchParams.get("baseUrl") ?? "").replace(/\/$/, "");
+
+  try {
+    switch (provider) {
+      case "ollama": {
+        const url = `${baseUrl || "http://localhost:11434"}/api/tags`;
+        const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
+        if (!res.ok) throw new Error(`Ollama ${res.status}`);
+        const data = (await res.json()) as { models?: Array<{ name: string }> };
+        const models = (data.models ?? []).map((m) => m.name).sort();
+        return NextResponse.json({ models });
+      }
+
+      default:
+        return NextResponse.json({ error: `No dynamic model list for "${provider}"` }, { status: 400 });
+    }
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : String(err) },
+      { status: 502 }
+    );
+  }
+}
+
+const OPENAI_FALLBACK = PRESET_MODELS.openai;
+
+/**
+ * POST /api/models — list OpenAI chat models (requires API key from body or OPENAI_API_KEY on server).
+ */
+export async function POST(req: NextRequest) {
+  const body = (await req.json()) as {
+    provider?: string;
+    apiKey?: string;
+    baseUrl?: string;
+  };
+
+  if (body.provider !== "openai") {
+    return NextResponse.json({ error: "Only provider \"openai\" is supported for POST" }, { status: 400 });
+  }
+
+  const key = body.apiKey?.trim() || process.env.OPENAI_API_KEY?.trim();
+  const base = (body.baseUrl?.trim() || "https://api.openai.com/v1").replace(/\/$/, "");
+
+  if (!key) {
+    return NextResponse.json({ models: OPENAI_FALLBACK, source: "preset" as const });
+  }
+
+  try {
+    const res = await fetch(`${base}/models`, {
+      headers: { Authorization: `Bearer ${key}` },
+      signal: AbortSignal.timeout(12_000),
+    });
+
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      return NextResponse.json({
+        models: OPENAI_FALLBACK,
+        source: "preset" as const,
+        error: `OpenAI ${res.status}${text ? `: ${text.slice(0, 220)}` : ""}`,
+      });
+    }
+
+    const data = (await res.json()) as { data?: Array<{ id: string }> };
+    const raw = (data.data ?? []).map((m) => m.id);
+    const models = filterOpenAiChatModelIds(raw);
+
+    if (models.length === 0) {
+      return NextResponse.json({
+        models: OPENAI_FALLBACK,
+        source: "preset" as const,
+        error: "No chat-capable models returned from API",
+      });
+    }
+
+    return NextResponse.json({ models, source: "api" as const });
+  } catch (err) {
+    return NextResponse.json({
+      models: OPENAI_FALLBACK,
+      source: "preset" as const,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+}