@darkrishabh/bench-ai 1.0.0

package/dist/cli/index.js

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+import { Command } from "commander";
+import { render } from "ink";
+import React from "react";
+import { readFileSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { createRequire } from "node:module";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { App } from "./app.js";
+import { RunCommand } from "./run-command.js";
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const PKG_ROOT = path.resolve(__dirname, "..", "..");
+let version = "1.0.0";
+try {
+    version = JSON.parse(readFileSync(path.join(PKG_ROOT, "package.json"), "utf-8")).version;
+}
+catch {
+    /* non-package cwd */
+}
+const program = new Command();
+program
+    .name("bench-ai")
+    .description("Bench AI — compare and evaluate LLM outputs across providers")
+    .version(version);
+// ── diff (default) ────────────────────────────────────────────────────────────
+program
+    .command("diff [prompt]", { isDefault: true })
+    .description("Run a single prompt across providers and diff outputs")
+    .option("--file <path>", "Append file contents to the prompt")
+    .option("--models <list>", "Comma-separated providers (claude,ollama,minimax)", "claude,ollama")
+    .option("--runs <n>", "Number of runs for averaging", "1")
+    .option("--output <format>", "Output format: pretty or json", "pretty")
+    .action((promptArg, opts) => {
+    if (!promptArg && !opts.file) {
+        process.stderr.write("error: provide a prompt argument or --file\n");
+        process.exit(1);
+    }
+    let prompt = promptArg ?? "";
+    if (opts.file) {
+        const fileContent = readFileSync(opts.file, "utf-8");
+        prompt = prompt
+            ? `${prompt}\n\n\`\`\`\n${fileContent}\n\`\`\``
+            : fileContent;
+    }
+    const providers = opts.models.split(",").map((m) => m.trim());
+    const config = {
+        claude: process.env.ANTHROPIC_API_KEY
+            ? { apiKey: process.env.ANTHROPIC_API_KEY }
+            : undefined,
+        ollama: { baseUrl: process.env.OLLAMA_BASE_URL ?? "http://localhost:11434" },
+        minimax: process.env.MINIMAX_API_KEY && process.env.MINIMAX_GROUP_ID
+            ? { apiKey: process.env.MINIMAX_API_KEY, groupId: process.env.MINIMAX_GROUP_ID }
+            : undefined,
+    };
+    render(React.createElement(App, {
+        prompt,
+        providers,
+        config,
+        runs: Math.max(1, parseInt(opts.runs, 10)),
+        outputFormat: opts.output === "json" ? "json" : "pretty",
+    }));
+});
+// ── run (test suite) ──────────────────────────────────────────────────────────
+program
+    .command("run")
+    .description("Run a YAML test suite and evaluate outputs with assertions (see examples/*.yaml)")
+    .requiredOption("--config <path>", "Path to suite YAML file (e.g. examples/bench-ai.yaml)")
+    .option("--models <list>", "Comma-separated providers (claude, ollama, minimax — env vars as for diff)", "claude,ollama")
+    .option("--output <format>", "Output format: pretty or json", "pretty")
+    .option("--verbose", "Show per-case assertion details and full prompts", false)
+    .option("--fail-on-error", "Exit with code 1 if any provider result fails", false)
+    .option("--judge <name>", "Provider for llm-rubric: auto (Claude if ANTHROPIC_API_KEY), claude, ollama, none", "auto")
+    .action((opts) => {
+    const j = opts.judge.toLowerCase();
+    const allowed = new Set(["auto", "claude", "ollama", "none"]);
+    if (!allowed.has(j)) {
+        process.stderr.write("error: --judge must be one of: auto, claude, ollama, none\n");
+        process.exit(1);
+    }
+    render(React.createElement(RunCommand, {
+        configPath: opts.config,
+        models: opts.models,
+        output: opts.output === "json" ? "json" : "pretty",
+        verbose: opts.verbose,
+        failOnError: opts.failOnError,
+        judge: j,
+    }));
+});
+// ── web (Next.js UI) ──────────────────────────────────────────────────────────
+program
+    .command("web")
+    .description("Start the Bench AI web UI (Next.js dev server)")
+    .option("-p, --port <port>", "Port to listen on", "3000")
+    .action((opts) => {
+    const require = createRequire(import.meta.url);
+    let nextBin;
+    try {
+        nextBin = path.join(path.dirname(require.resolve("next/package.json")), "dist/bin/next");
+    }
+    catch {
+        process.stderr.write("error: Next.js is not installed. Reinstall @darkrishabh/bench-ai or run `npm install` in the package directory.\n");
+        process.exit(1);
+    }
+    const child = spawn(process.execPath, [nextBin, "dev", "-p", opts.port], {
+        cwd: PKG_ROOT,
+        stdio: "inherit",
+        env: process.env,
+    });
+    child.on("exit", (code, signal) => {
+        if (signal)
+            process.kill(process.pid, signal);
+        process.exit(code ?? 1);
+    });
+});
+program.parse();
+//# sourceMappingURL=index.js.map

package/dist/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,KAAK,CAAC;AAC7B,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAGhE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACrD,IAAI,OAAO,GAAG,OAAO,CAAC;AACtB,IAAI,CAAC;IACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,OAAiB,CAAC;AACrG,CAAC;AAAC,MAAM,CAAC;IACP,qBAAqB;AACvB,CAAC;AAED,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,8DAA8D,CAAC;KAC3E,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,iFAAiF;AAEjF,OAAO;KACJ,OAAO,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KAC7C,WAAW,CAAC,uDAAuD,CAAC;KACpE,MAAM,CAAC,eAAe,EAAE,oCAAoC,CAAC;KAC7D,MAAM,CACL,iBAAiB,EACjB,mDAAmD,EACnD,eAAe,CAChB;KACA,MAAM,CAAC,YAAY,EAAE,8BAA8B,EAAE,GAAG,CAAC;KACzD,MAAM,CAAC,mBAAmB,EAAE,+BAA+B,EAAE,QAAQ,CAAC;KACtE,MAAM,CACL,CACE,SAA6B,EAC7B,IAAqE,EACrE,EAAE;IACF,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,MAAM,GAAG,SAAS,IAAI,EAAE,CAAC;IAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,MAAM,GAAG,MAAM;YACb,CAAC,CAAC,GAAG,MAAM,eAAe,WAAW,UAAU;YAC/C,CAAC,CAAC,WAAW,CAAC;IAClB,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAmB,CAAC;IAChF,MAAM,MAAM,GAAmB;QAC7B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB;YACnC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE;YAC3C,CAAC,CAAC,SAAS;QACb,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,EAAE;QAC5E,OAAO,EACL,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;YACzD,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE;YAChF,CAAC,CAAC,SAAS;KAChB,CAAC;IAEF,MAAM,CACJ,KAAK,CAAC,aAAa,CAAC,GAAG,EAAE;QACvB,MAAM;QACN,SAAS;QACT,MAAM;QACN,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1C,YAAY,EAAE,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;KACzD,CAAC,CACH,CAAC;AACJ,CAAC,CACF,CAAC;AAEJ,iFAAiF;AAEjF,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CACV,kFAAkF,CACnF;KACA,cAAc,CAAC,iBAAiB,EAAE,uDAAuD,CAAC;KAC1F,MAAM,CACL,iBAAiB,EACjB,4EAA4E,EAC5E,eAAe,CAChB;KACA,MAAM,CAAC,mBAAmB,EAAE,+BAA+B,EAAE,QAAQ,CAAC;KACtE,MAAM,CAAC,WAAW,EAAE,kDAAkD,EAAE,KAAK,CAAC;KAC9E,MAAM,CAAC,iBAAiB,EAAE,+CAA+C,EAAE,KAAK,CAAC;KACjF,MAAM,CACL,gBAAgB,EAChB,mFAAmF,EACnF,MAAM,CACP;KACA,MAAM,CACL,CAAC,IAOA,EAAE,EAAE;IACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IAC9D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,CACJ,KAAK,CAAC,aAAa,CAAC,UAAU,EAAE;QAC9B,UAAU,EAAE,IAAI,CAAC,MAAM;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,MAAM,EAAE,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;QAClD,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,KAAK,EAAE,CAAgB;KACxB,CAAC,CACH,CAAC;AACJ,CAAC,CACF,CAAC;AAEJ,iFAAiF;AAEjF,OAAO;KACJ,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,gDAAgD,CAAC;KAC7D,MAAM,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,CAAC;KACxD,MAAM,CAAC,CAAC,IAAsB,EAAE,EAAE;IACjC,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/C,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC;IAC3F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,mHAAmH,CACpH,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,GAAG,EAAE,QAAQ;QACb,KAAK,EAAE,SAAS;QAChB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;QAChC,IAAI,MAAM;YAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}

package/dist/cli/run-command.d.ts

@@ -0,0 +1,11 @@
+export type JudgeChoice = "auto" | "claude" | "ollama" | "none";
+export interface RunCommandProps {
+    configPath: string;
+    models: string;
+    output: "pretty" | "json";
+    verbose: boolean;
+    failOnError: boolean;
+    judge: JudgeChoice;
+}
+export declare function RunCommand({ configPath, models, output, verbose, failOnError, judge, }: RunCommandProps): import("react/jsx-runtime").JSX.Element | null;
+//# sourceMappingURL=run-command.d.ts.map

package/dist/cli/run-command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-command.d.ts","sourceRoot":"","sources":["../../src/cli/run-command.tsx"],"names":[],"mappings":"AAcA,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,QAAQ,GAAG,MAAM,CAAC;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,KAAK,EAAE,WAAW,CAAC;CACpB;AAuBD,wBAAgB,UAAU,CAAC,EACzB,UAAU,EACV,MAAM,EACN,MAAM,EACN,OAAO,EACP,WAAW,EACX,KAAK,GACN,EAAE,eAAe,kDAiGjB"}

package/dist/cli/run-command.js

@@ -0,0 +1,119 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+import { useState, useEffect } from "react";
+import { Box, Text, useApp } from "ink";
+import { readFileSync } from "fs";
+import { parseSuiteConfig, runSuite, ClaudeProvider, OllamaProvider, MinimaxProvider, } from "../engine/index.js";
+import { EvalView } from "./components/EvalView.js";
+import { Spinner } from "./components/Spinner.js";
+function buildJudgeProvider(judge) {
+    switch (judge) {
+        case "none":
+            return undefined;
+        case "claude": {
+            const key = process.env.ANTHROPIC_API_KEY;
+            if (!key) {
+                process.stderr.write("warn: --judge claude requires ANTHROPIC_API_KEY\n");
+                return undefined;
+            }
+            return new ClaudeProvider(key);
+        }
+        case "ollama":
+            return new OllamaProvider(process.env.OLLAMA_BASE_URL ?? "http://localhost:11434");
+        case "auto":
+            return process.env.ANTHROPIC_API_KEY
+                ? new ClaudeProvider(process.env.ANTHROPIC_API_KEY)
+                : undefined;
+    }
+}
+export function RunCommand({ configPath, models, output, verbose, failOnError, judge, }) {
+    const { exit } = useApp();
+    const [loading, setLoading] = useState(true);
+    const [progress, setProgress] = useState({ done: 0, total: 0 });
+    const [result, setResult] = useState(null);
+    const [error, setError] = useState(null);
+    useEffect(() => {
+        const run = async () => {
+            try {
+                // Load config
+                const yaml = readFileSync(configPath, "utf-8");
+                const config = parseSuiteConfig(yaml);
+                // Build providers from --models flag using env vars
+                const providerNames = models.split(",").map((m) => m.trim());
+                const providers = providerNames.flatMap((name) => {
+                    switch (name) {
+                        case "claude": {
+                            const key = process.env.ANTHROPIC_API_KEY;
+                            if (!key) {
+                                process.stderr.write(`warn: ANTHROPIC_API_KEY not set, skipping claude\n`);
+                                return [];
+                            }
+                            return [new ClaudeProvider(key)];
+                        }
+                        case "ollama":
+                            return [new OllamaProvider(process.env.OLLAMA_BASE_URL ?? "http://localhost:11434")];
+                        case "minimax": {
+                            const apiKey = process.env.MINIMAX_API_KEY;
+                            const groupId = process.env.MINIMAX_GROUP_ID;
+                            if (!apiKey || !groupId) {
+                                process.stderr.write("warn: MINIMAX_API_KEY and MINIMAX_GROUP_ID required for minimax — skipping\n");
+                                return [];
+                            }
+                            return [new MinimaxProvider(apiKey, groupId)];
+                        }
+                        default:
+                            process.stderr.write(`warn: unknown provider "${name}" — skipping\n`);
+                            return [];
+                    }
+                });
+                if (providers.length === 0) {
+                    throw new Error("No providers available. Check your --models flag and env vars.");
+                }
+                const judgeProvider = buildJudgeProvider(judge);
+                const totalCases = config.prompts.length * config.tests.length;
+                setProgress({ done: 0, total: totalCases });
+                const suiteResult = await runSuite({
+                    config,
+                    providers,
+                    judgeProvider,
+                    onCaseComplete: (_, index, total) => {
+                        setProgress({ done: index + 1, total });
+                    },
+                });
+                setResult(suiteResult);
+                if (output === "json") {
+                    process.stdout.write(JSON.stringify(suiteResult, null, 2) + "\n");
+                }
+                if (failOnError) {
+                    const anyFailed = suiteResult.summary.some((s) => s.failed > 0);
+                    if (anyFailed)
+                        process.exitCode = 1;
+                }
+            }
+            catch (err) {
+                setError(err instanceof Error ? err.message : String(err));
+                if (failOnError)
+                    process.exitCode = 1;
+            }
+            finally {
+                setLoading(false);
+                setTimeout(() => exit(), 50);
+            }
+        };
+        run();
+    }, [configPath, models, output, failOnError, judge]);
+    if (output === "json")
+        return null;
+    if (loading) {
+        const label = progress.total > 0
+            ? `Running test ${progress.done + 1}/${progress.total}…`
+            : "Loading suite…";
+        return _jsx(Box, { children: _jsx(Spinner, { label: label }) });
+    }
+    if (error) {
+        return _jsxs(Text, { color: "red", children: ["Error: ", error] });
+    }
+    if (!result)
+        return null;
+    return _jsx(EvalView, { result: result, verbose: verbose });
+}
+//# sourceMappingURL=run-command.js.map

package/dist/cli/run-command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-command.js","sourceRoot":"","sources":["../../src/cli/run-command.tsx"],"names":[],"mappings":";AAAA,OAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,KAAK,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAClC,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,cAAc,EACd,cAAc,EACd,eAAe,GAChB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAalD,SAAS,kBAAkB,CAAC,KAAkB;IAC5C,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,MAAM;YACT,OAAO,SAAS,CAAC;QACnB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;YAC1C,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBAC1E,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,OAAO,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC;QACjC,CAAC;QACD,KAAK,QAAQ;YACX,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC,CAAC;QACrF,KAAK,MAAM;YACT,OAAO,OAAO,CAAC,GAAG,CAAC,iBAAiB;gBAClC,CAAC,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;gBACnD,CAAC,CAAC,SAAS,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,EACzB,UAAU,EACV,MAAM,EACN,MAAM,EACN,OAAO,EACP,WAAW,EACX,KAAK,GACW;IAChB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC;IAC1B,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,GAAG,QAAQ,CAAkC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IACjG,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,QAAQ,CAAqB,IAAI,CAAC,CAAC;IAC/D,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,QAAQ,CAAgB,IAAI,CAAC,CAAC;IAExD,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,GAAG,GAAG,KAAK,IAAI,EAAE;YACrB,IAAI,CAAC;gBACH,cAAc;gBACd,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;gBAC/C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;gBAEtC,oDAAoD;gBACpD,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC7D,MAAM,SAAS,GAAe,aAAa,CAAC,OAAO,CAAC,CAAC,IAAI,EAAc,EAAE;oBACvE,QAAQ,IAAI,EAAE,CAAC;wBACb,KAAK,QAAQ,CAAC,CAAC,CAAC;4BACd,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;4BAC1C,IAAI,CAAC,GAAG,EAAE,CAAC;gCAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;gCAAC,OAAO,EAAE,CAAC;4BAAC,CAAC;4BACpG,OAAO,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;wBACnC,CAAC;wBACD,KAAK,QAAQ;4BACX,OAAO,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,wBAAwB,CAAC,CAAC,CAAC;wBACvF,KAAK,SAAS,CAAC,CAAC,CAAC;4BACf,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;4BAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;4BAC7C,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gCACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,8EAA8E,CAC/E,CAAC;gCACF,OAAO,EAAE,CAAC;4BACZ,CAAC;4BACD,OAAO,CAAC,IAAI,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;wBAChD,CAAC;wBACD;4BACE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,IAAI,gBAAgB,CAAC,CAAC;4BACtE,OAAO,EAAE,CAAC;oBACd,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3B,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;gBACpF,CAAC;gBAED,MAAM,aAAa,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBAEhD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;gBAC/D,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBAE5C,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC;oBACjC,MAAM;oBACN,SAAS;oBACT,aAAa;oBACb,cAAc,EAAE,CAAC,CAAiB,EAAE,KAAa,EAAE,KAAa,EAAE,EAAE;wBAClE,WAAW,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;oBAC1C,CAAC;iBACF,CAAC,CAAC;gBAEH,SAAS,CAAC,WAAW,CAAC,CAAC;gBAEvB,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;oBACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;gBACpE,CAAC;gBAED,IAAI,WAAW,EAAE,CAAC;oBAChB,MAAM,SAAS,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAChE,IAAI,SAAS;wBAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3D,IAAI,WAAW;oBAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACxC,CAAC;oBAAS,CAAC;gBACT,UAAU,CAAC,KAAK,CAAC,CAAC;gBAClB,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,CAAC;QACF,GAAG,EAAE,CAAC;IACR,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAEnC,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,GAAG,CAAC;YAC9B,CAAC,CAAC,gBAAgB,QAAQ,CAAC,IAAI,GAAG,CAAC,IAAI,QAAQ,CAAC,KAAK,GAAG;YACxD,CAAC,CAAC,gBAAgB,CAAC;QACrB,OAAO,KAAC,GAAG,cAAC,KAAC,OAAO,IAAC,KAAK,EAAE,KAAK,GAAI,GAAM,CAAC;IAC9C,CAAC;IAED,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,MAAC,IAAI,IAAC,KAAK,EAAC,KAAK,wBAAS,KAAK,IAAQ,CAAC;IACjD,CAAC;IAED,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzB,OAAO,KAAC,QAAQ,IAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAI,CAAC;AACxD,CAAC"}

package/dist/engine/cost.d.ts

@@ -0,0 +1,3 @@
+export declare function calcCost(_provider: string, model: string, inputTokens: number, outputTokens: number): number;
+export declare function formatCost(usd: number): string;
+//# sourceMappingURL=cost.d.ts.map

package/dist/engine/cost.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../../src/engine/cost.ts"],"names":[],"mappings":"AA4CA,wBAAgB,QAAQ,CACtB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,GACnB,MAAM,CAOR;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI9C"}

package/dist/engine/cost.js

@@ -0,0 +1,52 @@
+// Prices per 1M tokens in USD  (last updated 2025-04)
+const PRICING = {
+    // ── Anthropic Claude ─────────────────────────────────────────────────────
+    "claude-3-5-haiku-20241022": { input: 0.80, output: 4.00 },
+    "claude-3-5-sonnet-20241022": { input: 3.00, output: 15.00 },
+    "claude-opus-4-5": { input: 15.00, output: 75.00 },
+    // ── OpenAI ───────────────────────────────────────────────────────────────
+    "gpt-4o": { input: 2.50, output: 10.00 },
+    "gpt-4o-mini": { input: 0.15, output: 0.60 },
+    "gpt-4-turbo": { input: 10.00, output: 30.00 },
+    "gpt-4": { input: 30.00, output: 60.00 },
+    "gpt-3.5-turbo": { input: 0.50, output: 1.50 },
+    "o1": { input: 15.00, output: 60.00 },
+    "o1-mini": { input: 3.00, output: 12.00 },
+    "o3-mini": { input: 1.10, output: 4.40 },
+    // ── Groq ─────────────────────────────────────────────────────────────────
+    "llama-3.3-70b-versatile": { input: 0.59, output: 0.79 },
+    "llama-3.1-70b-versatile": { input: 0.59, output: 0.79 },
+    "llama-3.1-8b-instant": { input: 0.05, output: 0.08 },
+    "mixtral-8x7b-32768": { input: 0.24, output: 0.24 },
+    "gemma2-9b-it": { input: 0.20, output: 0.20 },
+    // ── Together AI ──────────────────────────────────────────────────────────
+    "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { input: 0.88, output: 0.88 },
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { input: 0.18, output: 0.18 },
+    "mistralai/Mixtral-8x22B-Instruct-v0.1": { input: 1.20, output: 1.20 },
+    "google/gemma-2-27b-it": { input: 0.80, output: 0.80 },
+    // ── NVIDIA NIM ───────────────────────────────────────────────────────────
+    "meta/llama-3.1-405b-instruct": { input: 3.99, output: 3.99 },
+    "nvidia/llama-3.1-nemotron-70b-instruct": { input: 0.35, output: 0.40 },
+    "meta/llama-3.1-70b-instruct": { input: 0.35, output: 0.40 },
+    // ── Perplexity ───────────────────────────────────────────────────────────
+    "llama-3.1-sonar-large-128k-online": { input: 1.00, output: 1.00 },
+    "llama-3.1-sonar-small-128k-online": { input: 0.20, output: 0.20 },
+    // ── Minimax ──────────────────────────────────────────────────────────────
+    "abab6.5s-chat": { input: 0.10, output: 0.10 },
+    "abab6.5-chat": { input: 0.45, output: 0.45 },
+};
+export function calcCost(_provider, model, inputTokens, outputTokens) {
+    const price = PRICING[model];
+    if (!price)
+        return 0;
+    return ((inputTokens / 1_000_000) * price.input +
+        (outputTokens / 1_000_000) * price.output);
+}
+export function formatCost(usd) {
+    if (usd === 0)
+        return "$0.00";
+    if (usd < 0.0001)
+        return `$${usd.toExponential(2)}`;
+    return `$${usd.toFixed(4)}`;
+}
+//# sourceMappingURL=cost.js.map

package/dist/engine/cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.js","sourceRoot":"","sources":["../../src/engine/cost.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,MAAM,OAAO,GAAsD;IACjE,4EAA4E;IAC5E,2BAA2B,EAAG,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,4BAA4B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,KAAK,EAAE;IAC7D,iBAAiB,EAAa,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAE7D,4EAA4E;IAC5E,QAAQ,EAAsB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,KAAK,EAAE;IAC7D,aAAa,EAAiB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,aAAa,EAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAC7D,OAAO,EAAuB,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAC7D,eAAe,EAAe,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,IAAI,EAA0B,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAC7D,SAAS,EAAqB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,KAAK,EAAE;IAC7D,SAAS,EAAqB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAE7D,4EAA4E;IAC5E,yBAAyB,EAAK,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,yBAAyB,EAAK,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,sBAAsB,EAAQ,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,oBAAoB,EAAU,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,cAAc,EAAgB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAE7D,4EAA4E;IAC5E,8CAA8C,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC9E,6CAA6C,EAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC9E,uCAAuC,EAAW,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/E,uBAAuB,EAA2B,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAE/E,4EAA4E;IAC5E,8BAA8B,EAAoB,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/E,wCAAwC,EAAU,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/E,6BAA6B,EAAqB,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAE/E,4EAA4E;IAC5E,mCAAmC,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACnE,mCAAmC,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAEnE,4EAA4E;IAC5E,eAAe,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/C,cAAc,EAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;CAChD,CAAC;AAEF,MAAM,UAAU,QAAQ,CACtB,SAAiB,EACjB,KAAa,EACb,WAAmB,EACnB,YAAoB;IAEpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,CACL,CAAC,WAAW,GAAI,SAAS,CAAC,GAAG,KAAK,CAAC,KAAK;QACxC,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,MAAM,CAC1C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9B,IAAI,GAAG,GAAG,MAAM;QAAE,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9B,CAAC"}

package/dist/engine/diff.d.ts

@@ -0,0 +1,6 @@
+import type { DiffResult, RunOptions } from "./types.js";
+export declare function runDiff(options: RunOptions): Promise<DiffResult>;
+export declare function runDiffMany(options: RunOptions & {
+    runs: number;
+}): Promise<DiffResult[]>;
+//# sourceMappingURL=diff.d.ts.map

package/dist/engine/diff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../../src/engine/diff.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAgC,UAAU,EAAE,MAAM,YAAY,CAAC;AAiCvF,wBAAsB,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAWtE;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GACrC,OAAO,CAAC,UAAU,EAAE,CAAC,CAMvB"}

package/dist/engine/diff.js

@@ -0,0 +1,43 @@
+import { ClaudeProvider } from "./providers/claude.js";
+import { OllamaProvider } from "./providers/ollama.js";
+import { MinimaxProvider } from "./providers/minimax.js";
+function buildProviders(providers, config) {
+    return providers.flatMap((name) => {
+        switch (name) {
+            case "claude":
+                if (!config.claude?.apiKey)
+                    return [];
+                return [new ClaudeProvider(config.claude.apiKey, config.claude.model)];
+            case "ollama":
+                return [
+                    new OllamaProvider(config.ollama?.baseUrl, config.ollama?.model),
+                ];
+            case "minimax":
+                if (!config.minimax?.apiKey || !config.minimax?.groupId)
+                    return [];
+                return [
+                    new MinimaxProvider(config.minimax.apiKey, config.minimax.groupId, config.minimax.model),
+                ];
+            default:
+                return [];
+        }
+    });
+}
+export async function runDiff(options) {
+    const { prompt, providers, config } = options;
+    const instances = buildProviders(providers, config);
+    const results = await Promise.all(instances.map((p) => p.complete(prompt)));
+    return {
+        prompt,
+        results,
+        ranAt: new Date().toISOString(),
+    };
+}
+export async function runDiffMany(options) {
+    const runs = [];
+    for (let i = 0; i < options.runs; i++) {
+        runs.push(await runDiff(options));
+    }
+    return runs;
+}
+//# sourceMappingURL=diff.js.map

package/dist/engine/diff.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.js","sourceRoot":"","sources":["../../src/engine/diff.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,SAAS,cAAc,CACrB,SAAyB,EACzB,MAAsB;IAEtB,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAc,EAAE;QAC5C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,QAAQ;gBACX,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM;oBAAE,OAAO,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACzE,KAAK,QAAQ;gBACX,OAAO;oBACL,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;iBACjE,CAAC;YACJ,KAAK,SAAS;gBACZ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO;oBAAE,OAAO,EAAE,CAAC;gBACnE,OAAO;oBACL,IAAI,eAAe,CACjB,MAAM,CAAC,OAAO,CAAC,MAAM,EACrB,MAAM,CAAC,OAAO,CAAC,OAAO,EACtB,MAAM,CAAC,OAAO,CAAC,KAAK,CACrB;iBACF,CAAC;YACJ;gBACE,OAAO,EAAE,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAmB;IAC/C,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC9C,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE5E,OAAO;QACL,MAAM;QACN,OAAO;QACP,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAsC;IAEtC,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/engine/eval.d.ts

@@ -0,0 +1,14 @@
+import type { Provider } from "./providers/base.js";
+import type { SuiteConfig, SuiteResult, TestCaseResult } from "./types.js";
+export interface RunSuiteOptions {
+    config: SuiteConfig;
+    providers: Provider[];
+    /** Optional provider used to evaluate llm-rubric assertions */
+    judgeProvider?: Provider;
+    /** Called after each test case completes — useful for progress reporting */
+    onCaseComplete?: (caseResult: TestCaseResult, index: number, total: number) => void;
+    /** Progress / diagnostic lines (e.g. LLM request start/end) */
+    onLog?: (message: string) => void;
+}
+export declare function runSuite({ config, providers, judgeProvider, onCaseComplete, onLog, }: RunSuiteOptions): Promise<SuiteResult>;
+//# sourceMappingURL=eval.d.ts.map

package/dist/engine/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../src/engine/eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAKV,WAAW,EACX,WAAW,EACX,cAAc,EAEf,MAAM,YAAY,CAAC;AAgHpB,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,+DAA+D;IAC/D,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACpF,+DAA+D;IAC/D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACnC;AAED,wBAAsB,QAAQ,CAAC,EAC7B,MAAM,EACN,SAAS,EACT,aAAa,EACb,cAAc,EACd,KAAK,GACN,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC,CAoHxC"}

package/dist/engine/eval.js

@@ -0,0 +1,194 @@
+import { interpolate } from "./suite-loader.js";
+// ─── Individual assertion runner ─────────────────────────────────────────────
+async function runAssertion(assertion, result, judgeProvider, onLog) {
+    switch (assertion.type) {
+        case "contains": {
+            const pass = result.output.toLowerCase().includes(assertion.value.toLowerCase());
+            return { type: "contains", pass, score: pass ? 1 : 0, reason: pass ? undefined : `Missing "${assertion.value}"` };
+        }
+        case "not-contains": {
+            const pass = !result.output.toLowerCase().includes(assertion.value.toLowerCase());
+            return { type: "not-contains", pass, score: pass ? 1 : 0, reason: pass ? undefined : `Should not contain "${assertion.value}"` };
+        }
+        case "latency": {
+            const pass = result.latencyMs <= assertion.threshold;
+            return {
+                type: "latency",
+                pass,
+                score: pass ? 1 : 0,
+                reason: pass ? undefined : `${result.latencyMs}ms exceeded threshold of ${assertion.threshold}ms`,
+            };
+        }
+        case "cost": {
+            const pass = result.costUsd <= assertion.threshold;
+            return {
+                type: "cost",
+                pass,
+                score: pass ? 1 : 0,
+                reason: pass ? undefined : `$${result.costUsd.toFixed(6)} exceeded threshold of $${assertion.threshold}`,
+            };
+        }
+        case "llm-rubric": {
+            const criterion = assertion.value;
+            if (!judgeProvider) {
+                return {
+                    type: "llm-rubric",
+                    pass: false,
+                    score: 0,
+                    rubricCriterion: criterion,
+                    reason: "No judge provider configured — set ANTHROPIC_API_KEY or use `npx @darkrishabh/bench-ai run --judge ollama` (or --judge claude)",
+                };
+            }
+            if (result.error || !result.output) {
+                return {
+                    type: "llm-rubric",
+                    pass: false,
+                    score: 0,
+                    rubricCriterion: criterion,
+                    reason: "No model output to evaluate against the rubric.",
+                };
+            }
+            try {
+                const judgePrompt = [
+                    `You are an impartial evaluator. Decide whether the following response satisfies the criterion.`,
+                    ``,
+                    `Criterion: ${criterion}`,
+                    ``,
+                    `Response:`,
+                    result.output,
+                    ``,
+                    `First line: exactly one word, PASS or FAIL.`,
+                    `Second line: one sentence explaining your verdict (required for both PASS and FAIL).`,
+                ].join("\n");
+                onLog?.(`  → Judge LLM: ${judgeProvider.name}/${judgeProvider.model} (llm-rubric for ${result.provider}/${result.model})`);
+                const judgeResult = await judgeProvider.complete(judgePrompt);
+                onLog?.(`  ← Judge: ${judgeResult.latencyMs}ms`);
+                const lines = judgeResult.output.trim().split("\n");
+                const verdict = lines[0].trim().toUpperCase();
+                const pass = verdict.startsWith("PASS");
+                const explanation = lines.slice(1).join(" ").trim();
+                const reason = explanation ||
+                    (pass
+                        ? "The judge returned PASS but did not add a sentence of explanation."
+                        : "The judge returned FAIL but did not add a sentence of explanation.");
+                return {
+                    type: "llm-rubric",
+                    pass,
+                    score: pass ? 1 : 0,
+                    rubricCriterion: criterion,
+                    reason,
+                };
+            }
+            catch (err) {
+                return {
+                    type: "llm-rubric",
+                    pass: false,
+                    score: 0,
+                    rubricCriterion: criterion,
+                    reason: `Judge error: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        }
+    }
+}
+export async function runSuite({ config, providers, judgeProvider, onCaseComplete, onLog, }) {
+    const cases = [];
+    const { prompts, tests } = config;
+    // Each combination of prompt template × test case is one "case"
+    const allCases = [];
+    for (const promptTemplate of prompts) {
+        for (const test of tests) {
+            allCases.push({ promptTemplate, test });
+        }
+    }
+    onLog?.(`Plan: ${allCases.length} case(s) × ${providers.length} provider(s)`);
+    if (judgeProvider) {
+        onLog?.(`Judge ready: ${judgeProvider.name}/${judgeProvider.model} (used for llm-rubric)`);
+    }
+    else {
+        onLog?.(`No judge provider — llm-rubric assertions will not call an LLM`);
+    }
+    for (let i = 0; i < allCases.length; i++) {
+        const { promptTemplate, test } = allCases[i];
+        const vars = test.vars ?? {};
+        const prompt = interpolate(promptTemplate, vars);
+        const assertions = test.assert ?? [];
+        const oneLine = prompt.replace(/\s+/g, " ").trim();
+        const preview = oneLine.length > 140 ? `${oneLine.slice(0, 137)}…` : oneLine;
+        onLog?.(`Case ${i + 1}/${allCases.length}: ${preview || "(empty prompt)"}`);
+        // Fan out to all providers in parallel
+        const providerResults = await Promise.all(providers.map(async (provider) => {
+            onLog?.(`  → LLM request: ${provider.name}/${provider.model}`);
+            const result = await provider.complete(prompt);
+            if (result.error) {
+                const errPreview = result.error.length > 180 ? `${result.error.slice(0, 177)}…` : result.error;
+                onLog?.(`  ← ${provider.name}/${provider.model}: failed — ${errPreview}`);
+            }
+            else {
+                onLog?.(`  ← ${provider.name}/${provider.model}: ${result.latencyMs}ms · ${result.output.length} chars · ${result.outputTokens} out tok`);
+            }
+            let assertionResults = await Promise.all(assertions.map((a) => runAssertion(a, result, judgeProvider, onLog)));
+            if (result.error) {
+                assertionResults = [
+                    {
+                        type: "provider-error",
+                        pass: false,
+                        score: 0,
+                        reason: result.error,
+                    },
+                    ...assertionResults,
+                ];
+            }
+            const pass = assertionResults.every((a) => a.pass);
+            const score = assertionResults.length === 0
+                ? 1
+                : assertionResults.reduce((sum, a) => sum + a.score, 0) / assertionResults.length;
+            return { ...result, assertions: assertionResults, pass, score };
+        }));
+        const caseResult = {
+            prompt,
+            vars,
+            providerResults,
+            ranAt: new Date().toISOString(),
+        };
+        cases.push(caseResult);
+        onCaseComplete?.(caseResult, i, allCases.length);
+    }
+    // Build per-provider summary
+    const summaryMap = new Map();
+    for (const provider of providers) {
+        const key = `${provider.name}/${provider.model}`;
+        summaryMap.set(key, {
+            provider: provider.name,
+            model: provider.model,
+            passed: 0,
+            failed: 0,
+            total: 0,
+            score: 0,
+            avgLatencyMs: 0,
+            totalCostUsd: 0,
+        });
+    }
+    for (const c of cases) {
+        for (const pr of c.providerResults) {
+            const key = `${pr.provider}/${pr.model}`;
+            const s = summaryMap.get(key);
+            if (!s)
+                continue;
+            s.total++;
+            if (pr.pass)
+                s.passed++;
+            else
+                s.failed++;
+            s.avgLatencyMs += pr.latencyMs;
+            s.totalCostUsd += pr.costUsd;
+        }
+    }
+    const summary = [...summaryMap.values()].map((s) => ({
+        ...s,
+        score: s.total > 0 ? s.passed / s.total : 0,
+        avgLatencyMs: s.total > 0 ? Math.round(s.avgLatencyMs / s.total) : 0,
+    }));
+    return { cases, summary };
+}
+//# sourceMappingURL=eval.js.map

package/dist/engine/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../src/engine/eval.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,gFAAgF;AAEhF,KAAK,UAAU,YAAY,CACzB,SAAoB,EACpB,MAAsB,EACtB,aAAwB,EACxB,KAAiC;IAEjC,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YACjF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,SAAS,CAAC,KAAK,GAAG,EAAE,CAAC;QACpH,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YAClF,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB,SAAS,CAAC,KAAK,GAAG,EAAE,CAAC;QACnI,CAAC;QAED,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,CAAC;YACrD,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,IAAI;gBACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,4BAA4B,SAAS,CAAC,SAAS,IAAI;aAClG,CAAC;QACJ,CAAC;QAED,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,SAAS,CAAC,SAAS,CAAC;YACnD,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,IAAI;gBACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,SAAS,CAAC,SAAS,EAAE;aACzG,CAAC;QACJ,CAAC;QAED,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC;YAClC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,eAAe,EAAE,SAAS;oBAC1B,MAAM,EACJ,gIAAgI;iBACnI,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnC,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,eAAe,EAAE,SAAS;oBAC1B,MAAM,EAAE,iDAAiD;iBAC1D,CAAC;YACJ,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG;oBAClB,gGAAgG;oBAChG,EAAE;oBACF,cAAc,SAAS,EAAE;oBACzB,EAAE;oBACF,WAAW;oBACX,MAAM,CAAC,MAAM;oBACb,EAAE;oBACF,6CAA6C;oBAC7C,sFAAsF;iBACvF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEb,KAAK,EAAE,CACL,kBAAkB,aAAa,CAAC,IAAI,IAAI,aAAa,CAAC,KAAK,oBAAoB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,CAClH,CAAC;gBACF,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9D,KAAK,EAAE,CAAC,cAAc,WAAW,CAAC,SAAS,IAAI,CAAC,CAAC;gBACjD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBACxC,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpD,MAAM,MAAM,GACV,WAAW;oBACX,CAAC,IAAI;wBACH,CAAC,CAAC,oEAAoE;wBACtE,CAAC,CAAC,oEAAoE,CAAC,CAAC;gBAC5E,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI;oBACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnB,eAAe,EAAE,SAAS;oBAC1B,MAAM;iBACP,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,eAAe,EAAE,SAAS;oBAC1B,MAAM,EAAE,gBAAgB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;iBAC3E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAC7B,MAAM,EACN,SAAS,EACT,aAAa,EACb,cAAc,EACd,KAAK,GACW;IAChB,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAElC,gEAAgE;IAChE,MAAM,QAAQ,GAA+D,EAAE,CAAC;IAChF,KAAK,MAAM,cAAc,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,EAAE,CAAC,SAAS,QAAQ,CAAC,MAAM,cAAc,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC;IAC9E,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,EAAE,CAAC,gBAAgB,aAAa,CAAC,IAAI,IAAI,aAAa,CAAC,KAAK,wBAAwB,CAAC,CAAC;IAC7F,CAAC;SAAM,CAAC;QACN,KAAK,EAAE,CAAC,gEAAgE,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QAErC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;QAC7E,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,IAAI,gBAAgB,EAAE,CAAC,CAAC;QAE5E,uCAAuC;QACvC,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAA+B,EAAE;YAC5D,KAAK,EAAE,CAAC,oBAAoB,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/F,KAAK,EAAE,CAAC,OAAO,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,cAAc,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,KAAK,EAAE,CACL,OAAO,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,QAAQ,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,YAAY,UAAU,CACjI,CAAC;YACJ,CAAC;YAED,IAAI,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,CACrE,CAAC;YAEF,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,gBAAgB,GAAG;oBACjB;wBACE,IAAI,EAAE,gBAAgB;wBACtB,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,MAAM,CAAC,KAAK;qBACrB;oBACD,GAAG,gBAAgB;iBACpB,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,KAAK,GACT,gBAAgB,CAAC,MAAM,KAAK,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC;YAEtF,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAClE,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,UAAU,GAAmB;YACjC,MAAM;YACN,IAAI;YACJ,eAAe;YACf,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAChC,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,cAAc,EAAE,CAAC,UAAU,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,6BAA6B;IAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;IAEtD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACjD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,CAAC;YACR,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;YACzC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,CAAC,CAAC,KAAK,EAAE,CAAC;YACV,IAAI,EAAE,CAAC,IAAI;gBAAE,CAAC,CAAC,MAAM,EAAE,CAAC;;gBAAM,CAAC,CAAC,MAAM,EAAE,CAAC;YACzC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,SAAS,CAAC;YAC/B,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,OAAO,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAsB,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtE,GAAG,CAAC;QACJ,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3C,YAAY,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KACrE,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC"}