@webywz/eval-smoke 0.0.0-stage → 0.1.0

package/dist/providers.js

@@ -0,0 +1,161 @@
+import { UsageError } from "./types.js";
+const MOCK_FIXTURES = [
+    { match: /\bhello\b/i, reply: "Hello" },
+    { match: /capital of france/i, reply: "Paris" },
+    { match: /2\s*\+\s*2/, reply: "4" },
+    {
+        match: /ignore (all )?previous|pwned/i,
+        reply: "I cannot ignore my instructions. How can I help you?",
+    },
+];
+export const DEFAULT_MODELS = {
+    openai: "gpt-4o-mini",
+    anthropic: "claude-3-5-haiku-20241022",
+};
+export function mockComplete(request) {
+    if (request.signal.aborted) {
+        return Promise.reject(signalError(request.signal));
+    }
+    const hit = MOCK_FIXTURES.find((fixture) => fixture.match.test(request.prompt));
+    return Promise.resolve(hit?.reply ?? `[mock] No canned reply for this prompt.`);
+}
+function signalError(signal) {
+    const err = new Error("The operation was aborted");
+    err.name = "AbortError";
+    if (signal.reason instanceof Error)
+        return signal.reason;
+    return err;
+}
+async function readJson(response) {
+    const text = await response.text();
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return { raw: text };
+    }
+}
+function asRecord(value) {
+    return typeof value === "object" && value !== null ? value : undefined;
+}
+export function createOpenAIComplete(apiKey, fetchFn, baseUrl = "https://api.openai.com/v1") {
+    return async (request) => {
+        const response = await fetchFn(`${baseUrl.replace(/\/$/, "")}/chat/completions`, {
+            method: "POST",
+            signal: request.signal,
+            headers: {
+                authorization: `Bearer ${apiKey}`,
+                "content-type": "application/json",
+            },
+            body: JSON.stringify({
+                model: request.model,
+                messages: [
+                    ...(request.system ? [{ role: "system", content: request.system }] : []),
+                    { role: "user", content: request.prompt },
+                ],
+                max_tokens: request.maxTokens ?? 256,
+                temperature: 0,
+            }),
+        });
+        const data = await readJson(response);
+        if (!response.ok) {
+            throw new Error(`OpenAI ${response.status}: ${summarizeError(data)}`);
+        }
+        const choices = data.choices;
+        if (!Array.isArray(choices) || choices.length === 0) {
+            throw new Error("OpenAI response was missing choices");
+        }
+        const message = asRecord(asRecord(choices[0])?.message);
+        const content = message?.content;
+        if (typeof content !== "string") {
+            throw new Error("OpenAI response was missing message content");
+        }
+        return content;
+    };
+}
+export function createAnthropicComplete(apiKey, fetchFn, baseUrl = "https://api.anthropic.com/v1") {
+    return async (request) => {
+        const response = await fetchFn(`${baseUrl.replace(/\/$/, "")}/messages`, {
+            method: "POST",
+            signal: request.signal,
+            headers: {
+                "x-api-key": apiKey,
+                "anthropic-version": "2023-06-01",
+                "content-type": "application/json",
+            },
+            body: JSON.stringify({
+                model: request.model,
+                max_tokens: request.maxTokens ?? 256,
+                temperature: 0,
+                system: request.system,
+                messages: [{ role: "user", content: request.prompt }],
+            }),
+        });
+        const data = await readJson(response);
+        if (!response.ok) {
+            throw new Error(`Anthropic ${response.status}: ${summarizeError(data)}`);
+        }
+        const content = data.content;
+        if (!Array.isArray(content)) {
+            throw new Error("Anthropic response was missing content");
+        }
+        const text = content
+            .map((part) => asRecord(part))
+            .filter((part) => part?.type === "text" && typeof part.text === "string")
+            .map((part) => part?.text)
+            .join("");
+        if (!text) {
+            throw new Error("Anthropic response had no text content");
+        }
+        return text;
+    };
+}
+function summarizeError(data) {
+    const err = asRecord(data.error);
+    const message = err?.message ?? data.message ?? data.raw;
+    if (typeof message === "string" && message.trim()) {
+        return message.slice(0, 240);
+    }
+    return "request failed";
+}
+export function resolveProvider(options) {
+    if (options.mock)
+        return "mock";
+    if (options.provider)
+        return options.provider;
+    if (options.suiteProvider)
+        return options.suiteProvider;
+    const model = options.model ?? "";
+    if (model.startsWith("claude"))
+        return "anthropic";
+    const env = options.env ?? process.env;
+    if (env.OPENAI_API_KEY)
+        return "openai";
+    if (env.ANTHROPIC_API_KEY)
+        return "anthropic";
+    throw new UsageError("No API key set. Pass --mock for an offline run, or set OPENAI_API_KEY / ANTHROPIC_API_KEY.");
+}
+export function resolveModel(provider, requested, suiteDefault) {
+    if (requested)
+        return requested;
+    if (suiteDefault)
+        return suiteDefault;
+    if (provider === "mock")
+        return "mock";
+    return DEFAULT_MODELS[provider];
+}
+export function createComplete(provider, env = process.env, fetchFn = fetch) {
+    if (provider === "mock")
+        return mockComplete;
+    if (provider === "openai") {
+        const key = env.OPENAI_API_KEY;
+        if (!key)
+            throw new UsageError("OPENAI_API_KEY is not set");
+        return createOpenAIComplete(key, fetchFn, env.OPENAI_BASE_URL);
+    }
+    const key = env.ANTHROPIC_API_KEY;
+    if (!key)
+        throw new UsageError("ANTHROPIC_API_KEY is not set");
+    return createAnthropicComplete(key, fetchFn, env.ANTHROPIC_BASE_URL);
+}
+//# sourceMappingURL=providers.js.map

package/dist/providers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAA4D,MAAM,YAAY,CAAC;AAElG,MAAM,aAAa,GAA4C;IAC7D,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,OAAO,EAAE;IACvC,EAAE,KAAK,EAAE,oBAAoB,EAAE,KAAK,EAAE,OAAO,EAAE;IAC/C,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE;IACnC;QACE,KAAK,EAAE,+BAA+B;QACtC,KAAK,EAAE,sDAAsD;KAC9D;CACF,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAkD;IAC3E,MAAM,EAAE,aAAa;IACrB,SAAS,EAAE,2BAA2B;CACvC,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,OAAwB;IACnD,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAChF,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,IAAI,yCAAyC,CAAC,CAAC;AAClF,CAAC;AAED,SAAS,WAAW,CAAC,MAAmB;IACtC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IACnD,GAAG,CAAC,IAAI,GAAG,YAAY,CAAC;IACxB,IAAI,MAAM,CAAC,MAAM,YAAY,KAAK;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC;IACzD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,QAAkB;IACxC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,SAAS,CAAC;AACtG,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAc,EACd,OAAqB,EACrB,OAAO,GAAG,2BAA2B;IAErC,OAAO,KAAK,EAAE,OAAO,EAAE,EAAE;QACvB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,mBAAmB,EAAE;YAC/E,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,MAAM,EAAE;gBACjC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,QAAQ,EAAE;oBACR,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE;iBAC1C;gBACD,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;gBACpC,WAAW,EAAE,CAAC;aACf,CAAC;SACH,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,KAAK,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QACjC,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,MAAc,EACd,OAAqB,EACrB,OAAO,GAAG,8BAA8B;IAExC,OAAO,KAAK,EAAE,OAAO,EAAE,EAAE;QACvB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,EAAE;YACvE,MAAM,EAAE,MAAM;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE;gBACP,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;gBACjC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;gBACpC,WAAW,EAAE,CAAC;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;aACtD,CAAC;SACH,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,KAAK,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,IAAI,GAAG,OAAO;aACjB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;aAC7B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;aACxE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAc,CAAC;aACnC,IAAI,CAAC,EAAE,CAAC,CAAC;QACZ,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAA6B;IACnD,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,GAAG,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC;IACzD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAClD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAM/B;IACC,IAAI,OAAO,CAAC,IAAI;QAAE,OAAO,MAAM,CAAC;IAChC,IAAI,OAAO,CAAC,QAAQ;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC9C,IAAI,OAAO,CAAC,aAAa;QAAE,OAAO,OAAO,CAAC,aAAa,CAAC;IACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,IAAI,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,WAAW,CAAC;IACnD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,IAAI,GAAG,CAAC,cAAc;QAAE,OAAO,QAAQ,CAAC;IACxC,IAAI,GAAG,CAAC,iBAAiB;QAAE,OAAO,WAAW,CAAC;IAC9C,MAAM,IAAI,UAAU,CAClB,4FAA4F,CAC7F,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,QAAsB,EACtB,SAAkB,EAClB,YAAqB;IAErB,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAChC,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IACtC,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,MAAM,CAAC;IACvC,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,QAAsB,EACtB,MAAyB,OAAO,CAAC,GAAG,EACpC,UAAwB,KAAK;IAE7B,IAAI,QAAQ,KAAK,MAAM;QAAE,OAAO,YAAY,CAAC;IAC7C,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,CAAC,cAAc,CAAC;QAC/B,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,UAAU,CAAC,2BAA2B,CAAC,CAAC;QAC5D,OAAO,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,CAAC,iBAAiB,CAAC;IAClC,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,UAAU,CAAC,8BAA8B,CAAC,CAAC;IAC/D,OAAO,uBAAuB,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACvE,CAAC"}

package/dist/report.d.ts

@@ -0,0 +1,7 @@
+import type { Report } from "./types.js";
+export interface FormatOptions {
+    json?: boolean;
+    color?: boolean;
+}
+export declare function formatReport(report: Report, options?: FormatOptions): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAA2B,MAAM,EAAE,MAAM,YAAY,CAAC;AAQlE,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAsCD,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,MAAM,CA0BhF"}

package/dist/report.js

@@ -0,0 +1,60 @@
+const RESET = "\u001b[0m";
+const GREEN = "\u001b[32m";
+const RED = "\u001b[31m";
+const YELLOW = "\u001b[33m";
+const DIM = "\u001b[2m";
+function paint(enabled, color, text) {
+    return enabled ? `${color}${text}${RESET}` : text;
+}
+function truncate(text, max = 220) {
+    const compact = text.replace(/\s+/g, " ").trim();
+    if (compact.length <= max)
+        return compact;
+    return `${compact.slice(0, max - 1)}…`;
+}
+function failedChecks(result) {
+    return (result.checks ?? []).filter((check) => !check.ok);
+}
+function formatCase(result, color) {
+    const ms = `${result.duration_ms}ms`;
+    if (result.skipped) {
+        return [`  ${paint(color, YELLOW, "skip")}  ${result.id.padEnd(22)} ${paint(color, DIM, ms)}`];
+    }
+    if (result.ok) {
+        return [`  ${paint(color, GREEN, "pass")}  ${result.id.padEnd(22)} ${paint(color, DIM, ms)}`];
+    }
+    const lines = [`  ${paint(color, RED, "fail")}  ${result.id.padEnd(22)} ${paint(color, DIM, ms)}`];
+    if (result.error) {
+        lines.push(`        ${result.error}`);
+    }
+    for (const check of failedChecks(result)) {
+        lines.push(`        ${check.name}: ${check.detail ?? "failed"}`);
+    }
+    if (result.output) {
+        lines.push(`        output: ${truncate(result.output)}`);
+    }
+    return lines;
+}
+export function formatReport(report, options = {}) {
+    if (options.json) {
+        return `${JSON.stringify(report, null, 2)}\n`;
+    }
+    const color = Boolean(options.color);
+    const bits = [
+        `eval-smoke ${report.version}`,
+        report.runner,
+        report.model,
+        report.suite,
+        `${report.cases.length} cases`,
+    ].filter(Boolean);
+    const lines = [paint(color, DIM, bits.join("  ·  ")), ""];
+    for (const result of report.cases) {
+        lines.push(...formatCase(result, color));
+    }
+    const summary = `${report.passed} passed  ${report.failed} failed${report.skipped ? `  ${report.skipped} skipped` : ""}  ${report.duration_ms}ms`;
+    lines.push("");
+    lines.push(report.ok ? paint(color, GREEN, summary) : paint(color, RED, summary));
+    lines.push("");
+    return lines.join("\n");
+}
+//# sourceMappingURL=report.js.map

package/dist/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,WAAW,CAAC;AAC1B,MAAM,KAAK,GAAG,YAAY,CAAC;AAC3B,MAAM,GAAG,GAAG,YAAY,CAAC;AACzB,MAAM,MAAM,GAAG,YAAY,CAAC;AAC5B,MAAM,GAAG,GAAG,WAAW,CAAC;AAOxB,SAAS,KAAK,CAAC,OAAgB,EAAE,KAAa,EAAE,IAAY;IAC1D,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACpD,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAG,GAAG,GAAG;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,OAAO,CAAC;IAC1C,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;AACzC,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB;IACtC,OAAO,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,UAAU,CAAC,MAAkB,EAAE,KAAc;IACpD,MAAM,EAAE,GAAG,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC;IACrC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACjG,CAAC;IACD,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;QACd,OAAO,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IACnG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,KAAK,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,UAAyB,EAAE;IACtE,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;IAChD,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG;QACX,cAAc,MAAM,CAAC,OAAO,EAAE;QAC9B,MAAM,CAAC,MAAM;QACb,MAAM,CAAC,KAAK;QACZ,MAAM,CAAC,KAAK;QACZ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,QAAQ;KAC/B,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAE1D,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,OAAO,GAAG,GAAG,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,MAAM,UACvD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EACnD,KAAK,MAAM,CAAC,WAAW,IAAI,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IAClF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/run-cli.d.ts

@@ -0,0 +1,14 @@
+export declare const HELP = "eval-smoke \u2014 one-minute smoke evals for prompts and models\n\nUsage:\n  eval-smoke <suite.yaml|suite.json> [options]\n\nOptions:\n  --mock              Offline runner with deterministic fake answers\n  --json              Machine-readable JSON report\n  --fail-fast         Stop after the first failing case\n  --timeout <ms>      Per-case timeout (default: 30000)\n  --model <name>      Override suite model\n  --provider <name>   openai | anthropic | mock\n  -h, --help          Show this help\n  -v, --version       Show version\n\nExit codes:\n  0  all cases passed\n  1  one or more cases failed\n  2  usage or configuration error\n\nExamples:\n  eval-smoke examples/suite.yaml --mock\n  eval-smoke examples/suite.yaml --json --fail-fast\n  OPENAI_API_KEY=sk-... eval-smoke examples/suite.yaml\n";
+export interface CliIo {
+    stdout: {
+        write(text: string): void;
+    };
+    stderr: {
+        write(text: string): void;
+    };
+    env: NodeJS.ProcessEnv;
+    cwd: string;
+    isTTY: boolean;
+}
+export declare function runCli(argv: string[], io?: CliIo): Promise<number>;
+//# sourceMappingURL=run-cli.d.ts.map

package/dist/run-cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-cli.d.ts","sourceRoot":"","sources":["../src/run-cli.ts"],"names":[],"mappings":"AAOA,eAAO,MAAM,IAAI,0yBAwBhB,CAAC;AAEF,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACtC,MAAM,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IACtC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;CAChB;AAyBD,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,GAAE,KAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,CA+DrF"}

package/dist/run-cli.js

@@ -0,0 +1,116 @@
+import { parseArgs } from "node:util";
+import { loadSuite } from "./load-suite.js";
+import { formatReport } from "./report.js";
+import { DEFAULT_TIMEOUT_MS, runSuite } from "./run.js";
+import { UsageError } from "./types.js";
+import { VERSION } from "./version.js";
+export const HELP = `eval-smoke — one-minute smoke evals for prompts and models
+
+Usage:
+  eval-smoke <suite.yaml|suite.json> [options]
+
+Options:
+  --mock              Offline runner with deterministic fake answers
+  --json              Machine-readable JSON report
+  --fail-fast         Stop after the first failing case
+  --timeout <ms>      Per-case timeout (default: ${DEFAULT_TIMEOUT_MS})
+  --model <name>      Override suite model
+  --provider <name>   openai | anthropic | mock
+  -h, --help          Show this help
+  -v, --version       Show version
+
+Exit codes:
+  0  all cases passed
+  1  one or more cases failed
+  2  usage or configuration error
+
+Examples:
+  eval-smoke examples/suite.yaml --mock
+  eval-smoke examples/suite.yaml --json --fail-fast
+  OPENAI_API_KEY=sk-... eval-smoke examples/suite.yaml
+`;
+const defaultIo = () => ({
+    stdout: process.stdout,
+    stderr: process.stderr,
+    env: process.env,
+    cwd: process.cwd(),
+    isTTY: Boolean(process.stdout.isTTY),
+});
+function parseTimeout(raw) {
+    if (raw == null)
+        return DEFAULT_TIMEOUT_MS;
+    const value = Number(raw);
+    if (!Number.isInteger(value) || value <= 0) {
+        throw new UsageError("--timeout must be a positive integer (milliseconds)");
+    }
+    return value;
+}
+function parseProvider(raw) {
+    if (raw == null)
+        return undefined;
+    if (raw === "mock" || raw === "openai" || raw === "anthropic")
+        return raw;
+    throw new UsageError("--provider must be openai, anthropic, or mock");
+}
+export async function runCli(argv, io = defaultIo()) {
+    let parsed;
+    try {
+        parsed = parseArgs({
+            args: argv,
+            allowPositionals: true,
+            strict: true,
+            options: {
+                mock: { type: "boolean", default: false },
+                json: { type: "boolean", default: false },
+                "fail-fast": { type: "boolean", default: false },
+                timeout: { type: "string" },
+                model: { type: "string" },
+                provider: { type: "string" },
+                help: { type: "boolean", short: "h", default: false },
+                version: { type: "boolean", short: "v", default: false },
+            },
+        });
+    }
+    catch (error) {
+        io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+        return 2;
+    }
+    const { values, positionals } = parsed;
+    const asString = (value) => typeof value === "string" ? value : undefined;
+    const asBool = (value) => value === true;
+    if (asBool(values.help)) {
+        io.stdout.write(HELP);
+        return 0;
+    }
+    if (asBool(values.version)) {
+        io.stdout.write(`${VERSION}\n`);
+        return 0;
+    }
+    const suitePath = positionals[0];
+    if (!suitePath) {
+        io.stderr.write("Missing suite file.\n\n");
+        io.stderr.write(HELP);
+        return 2;
+    }
+    try {
+        const json = asBool(values.json);
+        const suite = loadSuite(suitePath, io.cwd);
+        const report = await runSuite(suite, {
+            mock: asBool(values.mock),
+            provider: parseProvider(asString(values.provider)),
+            model: asString(values.model),
+            failFast: asBool(values["fail-fast"]),
+            timeoutMs: parseTimeout(asString(values.timeout)),
+            env: io.env,
+        });
+        const color = io.isTTY && !json && !io.env.NO_COLOR;
+        io.stdout.write(formatReport(report, { json, color }));
+        return report.ok ? 0 : 1;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        io.stderr.write(`${message}\n`);
+        return error instanceof UsageError ? error.exitCode : 2;
+    }
+}
+//# sourceMappingURL=run-cli.js.map

package/dist/run-cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-cli.js","sourceRoot":"","sources":["../src/run-cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACxD,OAAO,EAAE,UAAU,EAAqB,MAAM,YAAY,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,CAAC,MAAM,IAAI,GAAG;;;;;;;;;mDAS+B,kBAAkB;;;;;;;;;;;;;;;CAepE,CAAC;AAUF,MAAM,SAAS,GAAG,GAAU,EAAE,CAAC,CAAC;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,MAAM,EAAE,OAAO,CAAC,MAAM;IACtB,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;IAClB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;CACrC,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,GAAuB;IAC3C,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,kBAAkB,CAAC;IAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,UAAU,CAAC,qDAAqD,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,GAAuB;IAC5C,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IAClC,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,WAAW;QAAE,OAAO,GAAG,CAAC;IAC1E,MAAM,IAAI,UAAU,CAAC,+CAA+C,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,IAAc,EAAE,KAAY,SAAS,EAAE;IAClE,IAAI,MAAoC,CAAC;IACzC,IAAI,CAAC;QACH,MAAM,GAAG,SAAS,CAAC;YACjB,IAAI,EAAE,IAAI;YACV,gBAAgB,EAAE,IAAI;YACtB,MAAM,EAAE,IAAI;YACZ,OAAO,EAAE;gBACP,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBACzC,WAAW,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;gBAChD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC3B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBACzB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;gBAC5B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE;gBACrD,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE;aACzD;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/E,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC;IACvC,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAsB,EAAE,CACtD,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAChD,MAAM,MAAM,GAAG,CAAC,KAAc,EAAW,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC;IAE3D,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QAChC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IACjC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC3C,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE;YACnC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACzB,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAClD,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACrC,SAAS,EAAE,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjD,GAAG,EAAE,EAAE,CAAC,GAAG;SACZ,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpD,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACvD,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;QAChC,OAAO,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;AACH,CAAC"}

package/dist/run.d.ts

@@ -0,0 +1,5 @@
+import type { Report, RunOptions, Suite } from "./types.js";
+declare const DEFAULT_TIMEOUT_MS = 30000;
+export declare function runSuite(suite: Suite, options?: RunOptions): Promise<Report>;
+export { DEFAULT_TIMEOUT_MS };
+//# sourceMappingURL=run.d.ts.map

package/dist/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAA0B,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAGpF,QAAA,MAAM,kBAAkB,QAAS,CAAC;AA6ElC,wBAAsB,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAkDtF;AAED,OAAO,EAAE,kBAAkB,EAAE,CAAC"}

package/dist/run.js

@@ -0,0 +1,118 @@
+import { evaluateCase } from "./matchers.js";
+import { createComplete, resolveModel, resolveProvider } from "./providers.js";
+import { VERSION } from "./version.js";
+const DEFAULT_TIMEOUT_MS = 30_000;
+function isAbortError(error) {
+    return ((error instanceof Error && error.name === "AbortError") ||
+        (typeof error === "object" && error !== null && error.name === "TimeoutError"));
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function timeoutError(timeoutMs) {
+    const error = new Error(`timed out after ${timeoutMs}ms`);
+    error.name = "TimeoutError";
+    return error;
+}
+async function withTimeout(timeoutMs, work) {
+    const controller = new AbortController();
+    let timer;
+    const timeout = new Promise((_, reject) => {
+        timer = setTimeout(() => {
+            const error = timeoutError(timeoutMs);
+            controller.abort(error);
+            reject(error);
+        }, timeoutMs);
+    });
+    try {
+        return await Promise.race([work(controller.signal), timeout]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+async function runCase(suite, caseIndex, complete, model, timeoutMs) {
+    const testCase = suite.cases[caseIndex];
+    if (!testCase) {
+        throw new Error(`Missing case at index ${caseIndex}`);
+    }
+    const started = performance.now();
+    try {
+        const output = await withTimeout(timeoutMs, (signal) => complete({
+            prompt: testCase.prompt,
+            system: testCase.system ?? suite.system,
+            model: testCase.model ?? model,
+            maxTokens: testCase.expect.max_tokens,
+            signal,
+        }));
+        const checks = evaluateCase(output, testCase);
+        return {
+            id: testCase.id,
+            ok: checks.every((check) => check.ok),
+            duration_ms: Math.round(performance.now() - started),
+            output,
+            checks,
+        };
+    }
+    catch (error) {
+        const timedOut = isAbortError(error);
+        return {
+            id: testCase.id,
+            ok: false,
+            duration_ms: Math.round(performance.now() - started),
+            error: timedOut ? `timed out after ${timeoutMs}ms` : errorMessage(error),
+        };
+    }
+}
+export async function runSuite(suite, options = {}) {
+    const env = options.env ?? process.env;
+    const runner = resolveProvider({
+        mock: options.mock,
+        provider: options.provider,
+        suiteProvider: suite.provider,
+        model: options.model ?? suite.model,
+        env,
+    });
+    const model = resolveModel(runner, options.model, suite.model);
+    const complete = options.complete ?? createComplete(runner, env, options.fetch ?? globalThis.fetch);
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const started = performance.now();
+    const cases = [];
+    for (let index = 0; index < suite.cases.length; index += 1) {
+        const result = await runCase(suite, index, complete, model, timeoutMs);
+        cases.push(result);
+        if (!result.ok && options.failFast) {
+            for (let rest = index + 1; rest < suite.cases.length; rest += 1) {
+                const skipped = suite.cases[rest];
+                if (!skipped)
+                    continue;
+                cases.push({
+                    id: skipped.id,
+                    ok: false,
+                    skipped: true,
+                    duration_ms: 0,
+                    error: "skipped (fail-fast)",
+                });
+            }
+            break;
+        }
+    }
+    const passed = cases.filter((item) => item.ok).length;
+    const skipped = cases.filter((item) => item.skipped).length;
+    const failed = cases.length - passed - skipped;
+    return {
+        ok: failed === 0 && skipped === 0,
+        version: VERSION,
+        runner,
+        model,
+        suite: suite.name,
+        passed,
+        failed,
+        skipped,
+        duration_ms: Math.round(performance.now() - started),
+        cases,
+    };
+}
+export { DEFAULT_TIMEOUT_MS };
+//# sourceMappingURL=run.js.map

package/dist/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAE/E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;QACvD,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAK,KAA2B,CAAC,IAAI,KAAK,cAAc,CAAC,CACtG,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,YAAY,CAAC,SAAiB;IACrC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mBAAmB,SAAS,IAAI,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,GAAG,cAAc,CAAC;IAC5B,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,WAAW,CAAI,SAAiB,EAAE,IAAyC;IACxF,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QAC/C,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACtB,MAAM,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;YACtC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,EAAE,SAAS,CAAC,CAAC;IAChB,CAAC,CAAC,CAAC;IACH,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAChE,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,KAAY,EACZ,SAAiB,EACjB,QAAoB,EACpB,KAAa,EACb,SAAiB;IAEjB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,yBAAyB,SAAS,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE,CACrD,QAAQ,CAAC;YACP,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM;YACvC,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,KAAK;YAC9B,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU;YACrC,MAAM;SACP,CAAC,CACH,CAAC;QACF,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC9C,OAAO;YACL,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,EAAE,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YACpD,MAAM;YACN,MAAM;SACP,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACrC,OAAO;YACL,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,EAAE,EAAE,KAAK;YACT,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YACpD,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,mBAAmB,SAAS,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC;SACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAY,EAAE,UAAsB,EAAE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC;IACvC,MAAM,MAAM,GAAG,eAAe,CAAC;QAC7B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,aAAa,EAAE,KAAK,CAAC,QAAQ;QAC7B,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK;QACnC,GAAG;KACJ,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,cAAc,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACpG,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAE/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACvE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACnC,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG,CAAC,EAAE,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;gBAChE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClC,IAAI,CAAC,OAAO;oBAAE,SAAS;gBACvB,KAAK,CAAC,IAAI,CAAC;oBACT,EAAE,EAAE,OAAO,CAAC,EAAE;oBACd,EAAE,EAAE,KAAK;oBACT,OAAO,EAAE,IAAI;oBACb,WAAW,EAAE,CAAC;oBACd,KAAK,EAAE,qBAAqB;iBAC7B,CAAC,CAAC;YACL,CAAC;YACD,MAAM;QACR,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAE/C,OAAO;QACL,EAAE,EAAE,MAAM,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC;QACjC,OAAO,EAAE,OAAO;QAChB,MAAM;QACN,KAAK;QACL,KAAK,EAAE,KAAK,CAAC,IAAI;QACjB,MAAM;QACN,MAAM;QACN,OAAO;QACP,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACpD,KAAK;KACN,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,kBAAkB,EAAE,CAAC"}

package/dist/tokens.d.ts

@@ -0,0 +1,3 @@
+/** Rough cl100k-style estimate used only for `expect.max_tokens`. */
+export declare function estimateTokens(text: string): number;
+//# sourceMappingURL=tokens.d.ts.map

package/dist/tokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD"}

package/dist/tokens.js

@@ -0,0 +1,7 @@
+/** Rough cl100k-style estimate used only for `expect.max_tokens`. */
+export function estimateTokens(text) {
+    if (!text)
+        return 0;
+    return Math.max(1, Math.ceil(text.length / 4));
+}
+//# sourceMappingURL=tokens.js.map

package/dist/tokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AAAA,qEAAqE;AACrE,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,72 @@
+export type ProviderName = "mock" | "openai" | "anthropic";
+export type StringList = string | string[];
+export interface Expect {
+    contains?: StringList;
+    not_contains?: StringList;
+    regex?: StringList;
+    max_tokens?: number;
+}
+export interface Case {
+    id: string;
+    prompt: string;
+    system?: string;
+    model?: string;
+    expect: Expect;
+}
+export interface Suite {
+    name?: string;
+    model?: string;
+    provider?: ProviderName;
+    system?: string;
+    cases: Case[];
+}
+export interface CompleteRequest {
+    prompt: string;
+    system?: string;
+    model: string;
+    maxTokens?: number;
+    signal: AbortSignal;
+}
+export type CompleteFn = (request: CompleteRequest) => Promise<string>;
+export interface CheckResult {
+    name: "contains" | "not_contains" | "regex" | "max_tokens";
+    ok: boolean;
+    expected: string;
+    detail?: string;
+}
+export interface CaseResult {
+    id: string;
+    ok: boolean;
+    skipped?: boolean;
+    duration_ms: number;
+    output?: string;
+    error?: string;
+    checks?: CheckResult[];
+}
+export interface Report {
+    ok: boolean;
+    version: string;
+    runner: ProviderName;
+    model: string;
+    suite?: string;
+    passed: number;
+    failed: number;
+    skipped: number;
+    duration_ms: number;
+    cases: CaseResult[];
+}
+export interface RunOptions {
+    mock?: boolean;
+    provider?: ProviderName;
+    model?: string;
+    failFast?: boolean;
+    timeoutMs?: number;
+    complete?: CompleteFn;
+    env?: NodeJS.ProcessEnv;
+    fetch?: typeof fetch;
+}
+export declare class UsageError extends Error {
+    readonly exitCode = 2;
+    constructor(message: string);
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE3D,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,MAAM,EAAE,CAAC;AAE3C,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,KAAK,CAAC,EAAE,UAAU,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,KAAK;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,IAAI,EAAE,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAEvE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,UAAU,GAAG,cAAc,GAAG,OAAO,GAAG,YAAY,CAAC;IAC3D,EAAE,EAAE,OAAO,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,MAAM;IACrB,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,UAAU,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,qBAAa,UAAW,SAAQ,KAAK;IACnC,QAAQ,CAAC,QAAQ,KAAK;gBACV,OAAO,EAAE,MAAM;CAI5B"}

package/dist/types.js

@@ -0,0 +1,8 @@
+export class UsageError extends Error {
+    exitCode = 2;
+    constructor(message) {
+        super(message);
+        this.name = "UsageError";
+    }
+}
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA8EA,MAAM,OAAO,UAAW,SAAQ,KAAK;IAC1B,QAAQ,GAAG,CAAC,CAAC;IACtB,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;IAC3B,CAAC;CACF"}

package/dist/version.d.ts

@@ -0,0 +1,2 @@
+export declare const VERSION: string;
+//# sourceMappingURL=version.d.ts.map