@prompt-diff/core 0.1.0

package/dist/cost.d.ts

@@ -0,0 +1,3 @@
+export declare function calcCost(_provider: string, model: string, inputTokens: number, outputTokens: number): number;
+export declare function formatCost(usd: number): string;
+//# sourceMappingURL=cost.d.ts.map

package/dist/cost.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../src/cost.ts"],"names":[],"mappings":"AA4CA,wBAAgB,QAAQ,CACtB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,GACnB,MAAM,CAOR;AAED,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAI9C"}

package/dist/cost.js

@@ -0,0 +1,52 @@
+// Prices per 1M tokens in USD  (last updated 2025-04)
+const PRICING = {
+    // ── Anthropic Claude ─────────────────────────────────────────────────────
+    "claude-3-5-haiku-20241022": { input: 0.80, output: 4.00 },
+    "claude-3-5-sonnet-20241022": { input: 3.00, output: 15.00 },
+    "claude-opus-4-5": { input: 15.00, output: 75.00 },
+    // ── OpenAI ───────────────────────────────────────────────────────────────
+    "gpt-4o": { input: 2.50, output: 10.00 },
+    "gpt-4o-mini": { input: 0.15, output: 0.60 },
+    "gpt-4-turbo": { input: 10.00, output: 30.00 },
+    "gpt-4": { input: 30.00, output: 60.00 },
+    "gpt-3.5-turbo": { input: 0.50, output: 1.50 },
+    "o1": { input: 15.00, output: 60.00 },
+    "o1-mini": { input: 3.00, output: 12.00 },
+    "o3-mini": { input: 1.10, output: 4.40 },
+    // ── Groq ─────────────────────────────────────────────────────────────────
+    "llama-3.3-70b-versatile": { input: 0.59, output: 0.79 },
+    "llama-3.1-70b-versatile": { input: 0.59, output: 0.79 },
+    "llama-3.1-8b-instant": { input: 0.05, output: 0.08 },
+    "mixtral-8x7b-32768": { input: 0.24, output: 0.24 },
+    "gemma2-9b-it": { input: 0.20, output: 0.20 },
+    // ── Together AI ──────────────────────────────────────────────────────────
+    "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { input: 0.88, output: 0.88 },
+    "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": { input: 0.18, output: 0.18 },
+    "mistralai/Mixtral-8x22B-Instruct-v0.1": { input: 1.20, output: 1.20 },
+    "google/gemma-2-27b-it": { input: 0.80, output: 0.80 },
+    // ── NVIDIA NIM ───────────────────────────────────────────────────────────
+    "meta/llama-3.1-405b-instruct": { input: 3.99, output: 3.99 },
+    "nvidia/llama-3.1-nemotron-70b-instruct": { input: 0.35, output: 0.40 },
+    "meta/llama-3.1-70b-instruct": { input: 0.35, output: 0.40 },
+    // ── Perplexity ───────────────────────────────────────────────────────────
+    "llama-3.1-sonar-large-128k-online": { input: 1.00, output: 1.00 },
+    "llama-3.1-sonar-small-128k-online": { input: 0.20, output: 0.20 },
+    // ── Minimax ──────────────────────────────────────────────────────────────
+    "abab6.5s-chat": { input: 0.10, output: 0.10 },
+    "abab6.5-chat": { input: 0.45, output: 0.45 },
+};
+export function calcCost(_provider, model, inputTokens, outputTokens) {
+    const price = PRICING[model];
+    if (!price)
+        return 0;
+    return ((inputTokens / 1_000_000) * price.input +
+        (outputTokens / 1_000_000) * price.output);
+}
+export function formatCost(usd) {
+    if (usd === 0)
+        return "$0.00";
+    if (usd < 0.0001)
+        return `$${usd.toExponential(2)}`;
+    return `$${usd.toFixed(4)}`;
+}
+//# sourceMappingURL=cost.js.map

package/dist/cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.js","sourceRoot":"","sources":["../src/cost.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,MAAM,OAAO,GAAsD;IACjE,4EAA4E;IAC5E,2BAA2B,EAAG,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,4BAA4B,EAAE,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,KAAK,EAAE;IAC7D,iBAAiB,EAAa,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAE7D,4EAA4E;IAC5E,QAAQ,EAAsB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,KAAK,EAAE;IAC7D,aAAa,EAAiB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,aAAa,EAAiB,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAC7D,OAAO,EAAuB,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAC7D,eAAe,EAAe,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,IAAI,EAA0B,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE;IAC7D,SAAS,EAAqB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,KAAK,EAAE;IAC7D,SAAS,EAAqB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAE7D,4EAA4E;IAC5E,yBAAyB,EAAK,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,yBAAyB,EAAK,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,sBAAsB,EAAQ,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,oBAAoB,EAAU,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAC7D,cAAc,EAAgB,EAAE,KAAK,EAAE,IAAI,EAAG,MAAM,EAAE,IAAI,EAAG;IAE7D,4EAA4E;IAC5E,8CAA8C,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC9E,6CAA6C,EAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC9E,uCAAuC,EAAW,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/E,uBAAuB,EAA2B,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAE/E,4EAA4E;IAC5E,8BAA8B,EAAoB,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/E,wCAAwC,EAAU,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/E,6BAA6B,EAAqB,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAE/E,4EAA4E;IAC5E,mCAAmC,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACnE,mCAAmC,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAEnE,4EAA4E;IAC5E,eAAe,EAAG,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IAC/C,cAAc,EAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;CAChD,CAAC;AAEF,MAAM,UAAU,QAAQ,CACtB,SAAiB,EACjB,KAAa,EACb,WAAmB,EACnB,YAAoB;IAEpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,KAAK;QAAE,OAAO,CAAC,CAAC;IACrB,OAAO,CACL,CAAC,WAAW,GAAI,SAAS,CAAC,GAAG,KAAK,CAAC,KAAK;QACxC,CAAC,YAAY,GAAG,SAAS,CAAC,GAAG,KAAK,CAAC,MAAM,CAC1C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAC9B,IAAI,GAAG,GAAG,MAAM;QAAE,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;IACpD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9B,CAAC"}

package/dist/diff.d.ts

@@ -0,0 +1,6 @@
+import type { DiffResult, RunOptions } from "./types.js";
+export declare function runDiff(options: RunOptions): Promise<DiffResult>;
+export declare function runDiffMany(options: RunOptions & {
+    runs: number;
+}): Promise<DiffResult[]>;
+//# sourceMappingURL=diff.d.ts.map

package/dist/diff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.d.ts","sourceRoot":"","sources":["../src/diff.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAgC,UAAU,EAAE,MAAM,YAAY,CAAC;AAiCvF,wBAAsB,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAWtE;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GACrC,OAAO,CAAC,UAAU,EAAE,CAAC,CAMvB"}

package/dist/diff.js

@@ -0,0 +1,43 @@
+import { ClaudeProvider } from "./providers/claude.js";
+import { OllamaProvider } from "./providers/ollama.js";
+import { MinimaxProvider } from "./providers/minimax.js";
+function buildProviders(providers, config) {
+    return providers.flatMap((name) => {
+        switch (name) {
+            case "claude":
+                if (!config.claude?.apiKey)
+                    return [];
+                return [new ClaudeProvider(config.claude.apiKey, config.claude.model)];
+            case "ollama":
+                return [
+                    new OllamaProvider(config.ollama?.baseUrl, config.ollama?.model),
+                ];
+            case "minimax":
+                if (!config.minimax?.apiKey || !config.minimax?.groupId)
+                    return [];
+                return [
+                    new MinimaxProvider(config.minimax.apiKey, config.minimax.groupId, config.minimax.model),
+                ];
+            default:
+                return [];
+        }
+    });
+}
+export async function runDiff(options) {
+    const { prompt, providers, config } = options;
+    const instances = buildProviders(providers, config);
+    const results = await Promise.all(instances.map((p) => p.complete(prompt)));
+    return {
+        prompt,
+        results,
+        ranAt: new Date().toISOString(),
+    };
+}
+export async function runDiffMany(options) {
+    const runs = [];
+    for (let i = 0; i < options.runs; i++) {
+        runs.push(await runDiff(options));
+    }
+    return runs;
+}
+//# sourceMappingURL=diff.js.map

package/dist/diff.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff.js","sourceRoot":"","sources":["../src/diff.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,SAAS,cAAc,CACrB,SAAyB,EACzB,MAAsB;IAEtB,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAc,EAAE;QAC5C,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,QAAQ;gBACX,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM;oBAAE,OAAO,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACzE,KAAK,QAAQ;gBACX,OAAO;oBACL,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC;iBACjE,CAAC;YACJ,KAAK,SAAS;gBACZ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO;oBAAE,OAAO,EAAE,CAAC;gBACnE,OAAO;oBACL,IAAI,eAAe,CACjB,MAAM,CAAC,OAAO,CAAC,MAAM,EACrB,MAAM,CAAC,OAAO,CAAC,OAAO,EACtB,MAAM,CAAC,OAAO,CAAC,KAAK,CACrB;iBACF,CAAC;YACJ;gBACE,OAAO,EAAE,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAmB;IAC/C,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC9C,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAE5E,OAAO;QACL,MAAM;QACN,OAAO;QACP,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KAChC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAsC;IAEtC,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/eval.d.ts

@@ -0,0 +1,14 @@
+import type { Provider } from "./providers/base.js";
+import type { SuiteConfig, SuiteResult, TestCaseResult } from "./types.js";
+export interface RunSuiteOptions {
+    config: SuiteConfig;
+    providers: Provider[];
+    /** Optional provider used to evaluate llm-rubric assertions */
+    judgeProvider?: Provider;
+    /** Called after each test case completes — useful for progress reporting */
+    onCaseComplete?: (caseResult: TestCaseResult, index: number, total: number) => void;
+    /** Progress / diagnostic lines (e.g. LLM request start/end) */
+    onLog?: (message: string) => void;
+}
+export declare function runSuite({ config, providers, judgeProvider, onCaseComplete, onLog, }: RunSuiteOptions): Promise<SuiteResult>;
+//# sourceMappingURL=eval.d.ts.map

package/dist/eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAKV,WAAW,EACX,WAAW,EACX,cAAc,EAEf,MAAM,YAAY,CAAC;AAgHpB,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,+DAA+D;IAC/D,aAAa,CAAC,EAAE,QAAQ,CAAC;IACzB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,CAAC,UAAU,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACpF,+DAA+D;IAC/D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACnC;AAED,wBAAsB,QAAQ,CAAC,EAC7B,MAAM,EACN,SAAS,EACT,aAAa,EACb,cAAc,EACd,KAAK,GACN,EAAE,eAAe,GAAG,OAAO,CAAC,WAAW,CAAC,CAoHxC"}

package/dist/eval.js

@@ -0,0 +1,194 @@
+import { interpolate } from "./suite-loader.js";
+// ─── Individual assertion runner ─────────────────────────────────────────────
+async function runAssertion(assertion, result, judgeProvider, onLog) {
+    switch (assertion.type) {
+        case "contains": {
+            const pass = result.output.toLowerCase().includes(assertion.value.toLowerCase());
+            return { type: "contains", pass, score: pass ? 1 : 0, reason: pass ? undefined : `Missing "${assertion.value}"` };
+        }
+        case "not-contains": {
+            const pass = !result.output.toLowerCase().includes(assertion.value.toLowerCase());
+            return { type: "not-contains", pass, score: pass ? 1 : 0, reason: pass ? undefined : `Should not contain "${assertion.value}"` };
+        }
+        case "latency": {
+            const pass = result.latencyMs <= assertion.threshold;
+            return {
+                type: "latency",
+                pass,
+                score: pass ? 1 : 0,
+                reason: pass ? undefined : `${result.latencyMs}ms exceeded threshold of ${assertion.threshold}ms`,
+            };
+        }
+        case "cost": {
+            const pass = result.costUsd <= assertion.threshold;
+            return {
+                type: "cost",
+                pass,
+                score: pass ? 1 : 0,
+                reason: pass ? undefined : `$${result.costUsd.toFixed(6)} exceeded threshold of $${assertion.threshold}`,
+            };
+        }
+        case "llm-rubric": {
+            const criterion = assertion.value;
+            if (!judgeProvider) {
+                return {
+                    type: "llm-rubric",
+                    pass: false,
+                    score: 0,
+                    rubricCriterion: criterion,
+                    reason: "No judge provider configured — set ANTHROPIC_API_KEY or use `prompt-diff run --judge ollama` (or --judge claude)",
+                };
+            }
+            if (result.error || !result.output) {
+                return {
+                    type: "llm-rubric",
+                    pass: false,
+                    score: 0,
+                    rubricCriterion: criterion,
+                    reason: "No model output to evaluate against the rubric.",
+                };
+            }
+            try {
+                const judgePrompt = [
+                    `You are an impartial evaluator. Decide whether the following response satisfies the criterion.`,
+                    ``,
+                    `Criterion: ${criterion}`,
+                    ``,
+                    `Response:`,
+                    result.output,
+                    ``,
+                    `First line: exactly one word, PASS or FAIL.`,
+                    `Second line: one sentence explaining your verdict (required for both PASS and FAIL).`,
+                ].join("\n");
+                onLog?.(`  → Judge LLM: ${judgeProvider.name}/${judgeProvider.model} (llm-rubric for ${result.provider}/${result.model})`);
+                const judgeResult = await judgeProvider.complete(judgePrompt);
+                onLog?.(`  ← Judge: ${judgeResult.latencyMs}ms`);
+                const lines = judgeResult.output.trim().split("\n");
+                const verdict = lines[0].trim().toUpperCase();
+                const pass = verdict.startsWith("PASS");
+                const explanation = lines.slice(1).join(" ").trim();
+                const reason = explanation ||
+                    (pass
+                        ? "The judge returned PASS but did not add a sentence of explanation."
+                        : "The judge returned FAIL but did not add a sentence of explanation.");
+                return {
+                    type: "llm-rubric",
+                    pass,
+                    score: pass ? 1 : 0,
+                    rubricCriterion: criterion,
+                    reason,
+                };
+            }
+            catch (err) {
+                return {
+                    type: "llm-rubric",
+                    pass: false,
+                    score: 0,
+                    rubricCriterion: criterion,
+                    reason: `Judge error: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        }
+    }
+}
+export async function runSuite({ config, providers, judgeProvider, onCaseComplete, onLog, }) {
+    const cases = [];
+    const { prompts, tests } = config;
+    // Each combination of prompt template × test case is one "case"
+    const allCases = [];
+    for (const promptTemplate of prompts) {
+        for (const test of tests) {
+            allCases.push({ promptTemplate, test });
+        }
+    }
+    onLog?.(`Plan: ${allCases.length} case(s) × ${providers.length} provider(s)`);
+    if (judgeProvider) {
+        onLog?.(`Judge ready: ${judgeProvider.name}/${judgeProvider.model} (used for llm-rubric)`);
+    }
+    else {
+        onLog?.(`No judge provider — llm-rubric assertions will not call an LLM`);
+    }
+    for (let i = 0; i < allCases.length; i++) {
+        const { promptTemplate, test } = allCases[i];
+        const vars = test.vars ?? {};
+        const prompt = interpolate(promptTemplate, vars);
+        const assertions = test.assert ?? [];
+        const oneLine = prompt.replace(/\s+/g, " ").trim();
+        const preview = oneLine.length > 140 ? `${oneLine.slice(0, 137)}…` : oneLine;
+        onLog?.(`Case ${i + 1}/${allCases.length}: ${preview || "(empty prompt)"}`);
+        // Fan out to all providers in parallel
+        const providerResults = await Promise.all(providers.map(async (provider) => {
+            onLog?.(`  → LLM request: ${provider.name}/${provider.model}`);
+            const result = await provider.complete(prompt);
+            if (result.error) {
+                const errPreview = result.error.length > 180 ? `${result.error.slice(0, 177)}…` : result.error;
+                onLog?.(`  ← ${provider.name}/${provider.model}: failed — ${errPreview}`);
+            }
+            else {
+                onLog?.(`  ← ${provider.name}/${provider.model}: ${result.latencyMs}ms · ${result.output.length} chars · ${result.outputTokens} out tok`);
+            }
+            let assertionResults = await Promise.all(assertions.map((a) => runAssertion(a, result, judgeProvider, onLog)));
+            if (result.error) {
+                assertionResults = [
+                    {
+                        type: "provider-error",
+                        pass: false,
+                        score: 0,
+                        reason: result.error,
+                    },
+                    ...assertionResults,
+                ];
+            }
+            const pass = assertionResults.every((a) => a.pass);
+            const score = assertionResults.length === 0
+                ? 1
+                : assertionResults.reduce((sum, a) => sum + a.score, 0) / assertionResults.length;
+            return { ...result, assertions: assertionResults, pass, score };
+        }));
+        const caseResult = {
+            prompt,
+            vars,
+            providerResults,
+            ranAt: new Date().toISOString(),
+        };
+        cases.push(caseResult);
+        onCaseComplete?.(caseResult, i, allCases.length);
+    }
+    // Build per-provider summary
+    const summaryMap = new Map();
+    for (const provider of providers) {
+        const key = `${provider.name}/${provider.model}`;
+        summaryMap.set(key, {
+            provider: provider.name,
+            model: provider.model,
+            passed: 0,
+            failed: 0,
+            total: 0,
+            score: 0,
+            avgLatencyMs: 0,
+            totalCostUsd: 0,
+        });
+    }
+    for (const c of cases) {
+        for (const pr of c.providerResults) {
+            const key = `${pr.provider}/${pr.model}`;
+            const s = summaryMap.get(key);
+            if (!s)
+                continue;
+            s.total++;
+            if (pr.pass)
+                s.passed++;
+            else
+                s.failed++;
+            s.avgLatencyMs += pr.latencyMs;
+            s.totalCostUsd += pr.costUsd;
+        }
+    }
+    const summary = [...summaryMap.values()].map((s) => ({
+        ...s,
+        score: s.total > 0 ? s.passed / s.total : 0,
+        avgLatencyMs: s.total > 0 ? Math.round(s.avgLatencyMs / s.total) : 0,
+    }));
+    return { cases, summary };
+}
+//# sourceMappingURL=eval.js.map

package/dist/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,gFAAgF;AAEhF,KAAK,UAAU,YAAY,CACzB,SAAoB,EACpB,MAAsB,EACtB,aAAwB,EACxB,KAAiC;IAEjC,QAAQ,SAAS,CAAC,IAAI,EAAE,CAAC;QACvB,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YACjF,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,SAAS,CAAC,KAAK,GAAG,EAAE,CAAC;QACpH,CAAC;QAED,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YAClF,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,uBAAuB,SAAS,CAAC,KAAK,GAAG,EAAE,CAAC;QACnI,CAAC;QAED,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,IAAI,SAAS,CAAC,SAAS,CAAC;YACrD,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,IAAI;gBACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,4BAA4B,SAAS,CAAC,SAAS,IAAI;aAClG,CAAC;QACJ,CAAC;QAED,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,SAAS,CAAC,SAAS,CAAC;YACnD,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,IAAI;gBACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,SAAS,CAAC,SAAS,EAAE;aACzG,CAAC;QACJ,CAAC;QAED,KAAK,YAAY,CAAC,CAAC,CAAC;YAClB,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC;YAClC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,eAAe,EAAE,SAAS;oBAC1B,MAAM,EACJ,kHAAkH;iBACrH,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnC,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,eAAe,EAAE,SAAS;oBAC1B,MAAM,EAAE,iDAAiD;iBAC1D,CAAC;YACJ,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG;oBAClB,gGAAgG;oBAChG,EAAE;oBACF,cAAc,SAAS,EAAE;oBACzB,EAAE;oBACF,WAAW;oBACX,MAAM,CAAC,MAAM;oBACb,EAAE;oBACF,6CAA6C;oBAC7C,sFAAsF;iBACvF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEb,KAAK,EAAE,CACL,kBAAkB,aAAa,CAAC,IAAI,IAAI,aAAa,CAAC,KAAK,oBAAoB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,CAClH,CAAC;gBACF,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;gBAC9D,KAAK,EAAE,CAAC,cAAc,WAAW,CAAC,SAAS,IAAI,CAAC,CAAC;gBACjD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACpD,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBACxC,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACpD,MAAM,MAAM,GACV,WAAW;oBACX,CAAC,IAAI;wBACH,CAAC,CAAC,oEAAoE;wBACtE,CAAC,CAAC,oEAAoE,CAAC,CAAC;gBAC5E,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI;oBACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACnB,eAAe,EAAE,SAAS;oBAC1B,MAAM;iBACP,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO;oBACL,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,KAAK;oBACX,KAAK,EAAE,CAAC;oBACR,eAAe,EAAE,SAAS;oBAC1B,MAAM,EAAE,gBAAgB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;iBAC3E,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAC7B,MAAM,EACN,SAAS,EACT,aAAa,EACb,cAAc,EACd,KAAK,GACW;IAChB,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAElC,gEAAgE;IAChE,MAAM,QAAQ,GAA+D,EAAE,CAAC;IAChF,KAAK,MAAM,cAAc,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,EAAE,CAAC,SAAS,QAAQ,CAAC,MAAM,cAAc,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC;IAC9E,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,EAAE,CAAC,gBAAgB,aAAa,CAAC,IAAI,IAAI,aAAa,CAAC,KAAK,wBAAwB,CAAC,CAAC;IAC7F,CAAC;SAAM,CAAC;QACN,KAAK,EAAE,CAAC,gEAAgE,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QAErC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;QAC7E,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,OAAO,IAAI,gBAAgB,EAAE,CAAC,CAAC;QAE5E,uCAAuC;QACvC,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CACvC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAA+B,EAAE;YAC5D,KAAK,EAAE,CAAC,oBAAoB,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/F,KAAK,EAAE,CAAC,OAAO,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,cAAc,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,KAAK,EAAE,CACL,OAAO,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,KAAK,MAAM,CAAC,SAAS,QAAQ,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,YAAY,UAAU,CACjI,CAAC;YACJ,CAAC;YAED,IAAI,gBAAgB,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,CACrE,CAAC;YAEF,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBACjB,gBAAgB,GAAG;oBACjB;wBACE,IAAI,EAAE,gBAAgB;wBACtB,IAAI,EAAE,KAAK;wBACX,KAAK,EAAE,CAAC;wBACR,MAAM,EAAE,MAAM,CAAC,KAAK;qBACrB;oBACD,GAAG,gBAAgB;iBACpB,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,KAAK,GACT,gBAAgB,CAAC,MAAM,KAAK,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBACH,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC;YAEtF,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAClE,CAAC,CAAC,CACH,CAAC;QAEF,MAAM,UAAU,GAAmB;YACjC,MAAM;YACN,IAAI;YACJ,eAAe;YACf,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SAChC,CAAC;QAEF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvB,cAAc,EAAE,CAAC,UAAU,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,6BAA6B;IAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;IAEtD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACjD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,KAAK,EAAE,CAAC;YACR,YAAY,EAAE,CAAC;YACf,YAAY,EAAE,CAAC;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,KAAK,MAAM,EAAE,IAAI,CAAC,CAAC,eAAe,EAAE,CAAC;YACnC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;YACzC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,CAAC,CAAC,KAAK,EAAE,CAAC;YACV,IAAI,EAAE,CAAC,IAAI;gBAAE,CAAC,CAAC,MAAM,EAAE,CAAC;;gBAAM,CAAC,CAAC,MAAM,EAAE,CAAC;YACzC,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,SAAS,CAAC;YAC/B,CAAC,CAAC,YAAY,IAAI,EAAE,CAAC,OAAO,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAsB,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtE,GAAG,CAAC;QACJ,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3C,YAAY,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;KACrE,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+export type { ProviderName, ProviderConfig, ProviderResult, DiffResult, RunOptions, Assertion, AssertionResult, TestCase, SuiteConfig, ProviderTestResult, TestCaseResult, ProviderSummary, SuiteResult, } from "./types.js";
+export { calcCost, formatCost } from "./cost.js";
+export { runDiff, runDiffMany } from "./diff.js";
+export { ClaudeProvider } from "./providers/claude.js";
+export { OllamaProvider } from "./providers/ollama.js";
+export { MinimaxProvider } from "./providers/minimax.js";
+export { OpenAICompatibleProvider } from "./providers/openai-compatible.js";
+export type { OpenAICompatibleOptions } from "./providers/openai-compatible.js";
+export { SubprocessProvider, createClaudeCLIProvider, createCodexProvider } from "./providers/subprocess.js";
+export type { SubprocessOptions } from "./providers/subprocess.js";
+export type { Provider } from "./providers/base.js";
+export { runSuite } from "./eval.js";
+export type { RunSuiteOptions } from "./eval.js";
+export { parseSuiteConfig, interpolate } from "./suite-loader.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,UAAU,EACV,UAAU,EACV,SAAS,EACT,eAAe,EACf,QAAQ,EACR,WAAW,EACX,kBAAkB,EAClB,cAAc,EACd,eAAe,EACf,WAAW,GACZ,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,YAAY,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAChF,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAC7G,YAAY,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACnE,YAAY,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/index.js

@@ -0,0 +1,10 @@
+export { calcCost, formatCost } from "./cost.js";
+export { runDiff, runDiffMany } from "./diff.js";
+export { ClaudeProvider } from "./providers/claude.js";
+export { OllamaProvider } from "./providers/ollama.js";
+export { MinimaxProvider } from "./providers/minimax.js";
+export { OpenAICompatibleProvider } from "./providers/openai-compatible.js";
+export { SubprocessProvider, createClaudeCLIProvider, createCodexProvider } from "./providers/subprocess.js";
+export { runSuite } from "./eval.js";
+export { parseSuiteConfig, interpolate } from "./suite-loader.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAE5E,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAG7G,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC"}

package/dist/providers/base.d.ts

@@ -0,0 +1,7 @@
+import type { ProviderResult } from "../types.js";
+export interface Provider {
+    readonly name: string;
+    readonly model: string;
+    complete(prompt: string): Promise<ProviderResult>;
+}
+//# sourceMappingURL=base.d.ts.map

package/dist/providers/base.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.d.ts","sourceRoot":"","sources":["../../src/providers/base.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACnD"}

package/dist/providers/base.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=base.js.map

package/dist/providers/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../src/providers/base.ts"],"names":[],"mappings":""}

package/dist/providers/claude.d.ts

@@ -0,0 +1,15 @@
+import type { ProviderResult } from "../types.js";
+import type { Provider } from "./base.js";
+export interface ClaudeOptions {
+    maxTokens?: number;
+    temperature?: number;
+}
+export declare class ClaudeProvider implements Provider {
+    readonly name = "claude";
+    readonly model: string;
+    private client;
+    private options;
+    constructor(apiKey: string, model?: string, options?: ClaudeOptions);
+    complete(prompt: string): Promise<ProviderResult>;
+}
+//# sourceMappingURL=claude.d.ts.map

package/dist/providers/claude.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude.d.ts","sourceRoot":"","sources":["../../src/providers/claude.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,cAAe,YAAW,QAAQ;IAC7C,QAAQ,CAAC,IAAI,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAgB;gBAEnB,MAAM,EAAE,MAAM,EAAE,KAAK,SAAgB,EAAE,OAAO,GAAE,aAAkB;IAMxE,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;CAwCxD"}

package/dist/providers/claude.js

@@ -0,0 +1,53 @@
+import Anthropic from "@anthropic-ai/sdk";
+import { calcCost } from "../cost.js";
+const DEFAULT_MODEL = "claude-3-5-haiku-20241022";
+export class ClaudeProvider {
+    name = "claude";
+    model;
+    client;
+    options;
+    constructor(apiKey, model = DEFAULT_MODEL, options = {}) {
+        this.client = new Anthropic({ apiKey });
+        this.model = model;
+        this.options = options;
+    }
+    async complete(prompt) {
+        const start = Date.now();
+        try {
+            const response = await this.client.messages.create({
+                model: this.model,
+                max_tokens: this.options.maxTokens ?? 2048,
+                ...(this.options.temperature !== undefined
+                    ? { temperature: this.options.temperature }
+                    : {}),
+                messages: [{ role: "user", content: prompt }],
+            });
+            const latencyMs = Date.now() - start;
+            const inputTokens = response.usage.input_tokens;
+            const outputTokens = response.usage.output_tokens;
+            const output = response.content[0].type === "text" ? response.content[0].text : "";
+            return {
+                provider: "claude",
+                model: this.model,
+                output,
+                latencyMs,
+                inputTokens,
+                outputTokens,
+                costUsd: calcCost("claude", this.model, inputTokens, outputTokens),
+            };
+        }
+        catch (err) {
+            return {
+                provider: "claude",
+                model: this.model,
+                output: "",
+                latencyMs: Date.now() - start,
+                inputTokens: 0,
+                outputTokens: 0,
+                costUsd: 0,
+                error: err instanceof Error ? err.message : String(err),
+            };
+        }
+    }
+}
+//# sourceMappingURL=claude.js.map

package/dist/providers/claude.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"claude.js","sourceRoot":"","sources":["../../src/providers/claude.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,MAAM,aAAa,GAAG,2BAA2B,CAAC;AAOlD,MAAM,OAAO,cAAc;IAChB,IAAI,GAAG,QAAQ,CAAC;IAChB,KAAK,CAAS;IACf,MAAM,CAAY;IAClB,OAAO,CAAgB;IAE/B,YAAY,MAAc,EAAE,KAAK,GAAG,aAAa,EAAE,UAAyB,EAAE;QAC5E,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;gBACjD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,IAAI;gBAC1C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS;oBACxC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;oBAC3C,CAAC,CAAC,EAAE,CAAC;gBACP,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;aAC9C,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YACrC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,YAAY,CAAC;YAChD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC;YAClD,MAAM,MAAM,GACV,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAEtE,OAAO;gBACL,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM;gBACN,SAAS;gBACT,WAAW;gBACX,YAAY;gBACZ,OAAO,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;aACnE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,QAAQ,EAAE,QAAQ;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC7B,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/providers/minimax.d.ts

@@ -0,0 +1,16 @@
+import type { ProviderResult } from "../types.js";
+import type { Provider } from "./base.js";
+export interface MinimaxOptions {
+    maxTokens?: number;
+    temperature?: number;
+}
+export declare class MinimaxProvider implements Provider {
+    readonly name = "minimax";
+    readonly model: string;
+    private apiKey;
+    private groupId;
+    private options;
+    constructor(apiKey: string, groupId: string, model?: string, options?: MinimaxOptions);
+    complete(prompt: string): Promise<ProviderResult>;
+}
+//# sourceMappingURL=minimax.d.ts.map

package/dist/providers/minimax.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"minimax.d.ts","sourceRoot":"","sources":["../../src/providers/minimax.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAK1C,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAOD,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,QAAQ,CAAC,IAAI,aAAa;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAiB;gBAEpB,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,SAAgB,EAAE,OAAO,GAAE,cAAmB;IAO1F,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;CAsDxD"}

package/dist/providers/minimax.js

@@ -0,0 +1,67 @@
+import { calcCost } from "../cost.js";
+const DEFAULT_MODEL = "abab6.5s-chat";
+const BASE_URL = "https://api.minimax.chat/v1";
+export class MinimaxProvider {
+    name = "minimax";
+    model;
+    apiKey;
+    groupId;
+    options;
+    constructor(apiKey, groupId, model = DEFAULT_MODEL, options = {}) {
+        this.apiKey = apiKey;
+        this.groupId = groupId;
+        this.model = model;
+        this.options = options;
+    }
+    async complete(prompt) {
+        const start = Date.now();
+        try {
+            const body = {
+                model: this.model,
+                messages: [{ role: "user", content: prompt }],
+            };
+            if (this.options.maxTokens !== undefined)
+                body.max_tokens = this.options.maxTokens;
+            if (this.options.temperature !== undefined)
+                body.temperature = this.options.temperature;
+            const res = await fetch(`${BASE_URL}/text/chatcompletion_v2?GroupId=${this.groupId}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify(body),
+            });
+            if (!res.ok) {
+                throw new Error(`Minimax ${res.status}: ${await res.text()}`);
+            }
+            const data = (await res.json());
+            const latencyMs = Date.now() - start;
+            const inputTokens = data.usage.prompt_tokens;
+            const outputTokens = data.usage.completion_tokens;
+            const output = data.choices[0]?.message.content ?? "";
+            return {
+                provider: "minimax",
+                model: this.model,
+                output,
+                latencyMs,
+                inputTokens,
+                outputTokens,
+                costUsd: calcCost("minimax", this.model, inputTokens, outputTokens),
+            };
+        }
+        catch (err) {
+            return {
+                provider: "minimax",
+                model: this.model,
+                output: "",
+                latencyMs: Date.now() - start,
+                inputTokens: 0,
+                outputTokens: 0,
+                costUsd: 0,
+                error: err instanceof Error ? err.message : String(err),
+            };
+        }
+    }
+}
+//# sourceMappingURL=minimax.js.map

package/dist/providers/minimax.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"minimax.js","sourceRoot":"","sources":["../../src/providers/minimax.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,MAAM,aAAa,GAAG,eAAe,CAAC;AACtC,MAAM,QAAQ,GAAG,6BAA6B,CAAC;AAY/C,MAAM,OAAO,eAAe;IACjB,IAAI,GAAG,SAAS,CAAC;IACjB,KAAK,CAAS;IACf,MAAM,CAAS;IACf,OAAO,CAAS;IAChB,OAAO,CAAiB;IAEhC,YAAY,MAAc,EAAE,OAAe,EAAE,KAAK,GAAG,aAAa,EAAE,UAA0B,EAAE;QAC9F,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,GAA4B;gBACpC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;aAC9C,CAAC;YACF,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS;gBAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;YACnF,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS;gBAAE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YAExF,MAAM,GAAG,GAAG,MAAM,KAAK,CACrB,GAAG,QAAQ,mCAAmC,IAAI,CAAC,OAAO,EAAE,EAC5D;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,CACF,CAAC;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAChE,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAoB,CAAC;YACnD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YACrC,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC;YAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC;YAClD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;YAEtD,OAAO;gBACL,QAAQ,EAAE,SAAS;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM;gBACN,SAAS;gBACT,WAAW;gBACX,YAAY;gBACZ,OAAO,EAAE,QAAQ,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;aACpE,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO;gBACL,QAAQ,EAAE,SAAS;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC7B,WAAW,EAAE,CAAC;gBACd,YAAY,EAAE,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/providers/ollama.d.ts

@@ -0,0 +1,14 @@
+import type { ProviderResult } from "../types.js";
+import type { Provider } from "./base.js";
+export interface OllamaOptions {
+    temperature?: number;
+}
+export declare class OllamaProvider implements Provider {
+    readonly name = "ollama";
+    readonly model: string;
+    private baseUrl;
+    private options;
+    constructor(baseUrl?: string, model?: string, options?: OllamaOptions);
+    complete(prompt: string): Promise<ProviderResult>;
+}
+//# sourceMappingURL=ollama.d.ts.map

package/dist/providers/ollama.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../src/providers/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAK1C,MAAM,WAAW,aAAa;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AASD,qBAAa,cAAe,YAAW,QAAQ;IAC7C,QAAQ,CAAC,IAAI,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAgB;gBAEnB,OAAO,SAAmB,EAAE,KAAK,SAAgB,EAAE,OAAO,GAAE,aAAkB;IAMpF,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;CAiDxD"}