@webywz/eval-smoke 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 webywz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,181 @@
-# Temporary Holding Version
+# eval-smoke
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+One-minute smoke evals for prompts and models.
+
+Full eval harnesses are heavy. `eval-smoke` is the tiny local suite you run before a ship: a handful of cases, pass/fail, exit code 1 on regression. YAML or JSON in, a CI-friendly report out.
+
+```bash
+npm i -g @webywz/eval-smoke
+eval-smoke examples/suite.yaml --mock
+```
+
+```
+eval-smoke 0.1.0  ·  mock  ·  gpt-4o-mini  ·  demo  ·  4 cases
+
+  pass  greeting               0ms
+  pass  capital-france         0ms
+  pass  arithmetic             0ms
+  pass  refuse-injection       0ms
+
+4 passed  0 failed  1ms
+```
+
+`--mock` is offline and deterministic. Contributors and CI do not need API keys.
+
+## Install
+
+```bash
+npm i -g @webywz/eval-smoke
+# or, from a project:
+npx @webywz/eval-smoke suite.yaml --mock
+```
+
+Requires Node 18+.
+
+## 60-second start
+
+1. Copy the example suite, or write three cases you actually care about.
+2. Run mock mode until the runner and matchers look right.
+3. Point the same file at a live model when you are ready.
+
+```bash
+eval-smoke examples/suite.yaml --mock
+eval-smoke examples/suite.yaml --mock --json
+OPENAI_API_KEY=sk-... eval-smoke examples/suite.yaml
+ANTHROPIC_API_KEY=sk-ant-... eval-smoke examples/suite.yaml --model claude-3-5-haiku-20241022
+```
+
+## Suite format
+
+A suite is a YAML or JSON list of cases, or an object with a `cases` array.
+
+```yaml
+name: demo
+model: gpt-4o-mini
+# provider: openai          # optional: openai | anthropic | mock
+# system: You are concise.
+cases:
+  - id: greeting
+    prompt: Reply with exactly the word Hello.
+    expect:
+      contains: Hello
+      not_contains: Goodbye
+      regex: "(?i)^hello\\b"
+      max_tokens: 32
+```
+
+A bare list is also valid:
+
+```yaml
+- id: capital-france
+  prompt: What is the capital of France? Answer with the city name only.
+  expect:
+    contains: Paris
+```
+
+| Field | Where | Notes |
+| --- | --- | --- |
+| `id` | case | Required, unique. |
+| `prompt` | case | Required. |
+| `expect.contains` | case | Substring, case-insensitive. String or list; all must match. |
+| `expect.not_contains` | case | Substring must be absent. String or list. |
+| `expect.regex` | case | JavaScript regex. `/pattern/i`, `(?i)pattern`, or a raw pattern. |
+| `expect.max_tokens` | case | Approximate cap using `ceil(chars / 4)`. |
+| `system` | suite or case | Optional system prompt for live providers. |
+| `model` | suite or case | Live default: `gpt-4o-mini` or `claude-3-5-haiku-20241022`. |
+| `provider` | suite | `openai`, `anthropic`, or `mock`. |
+
+At least one matcher is required on every case.
+
+## CLI
+
+```
+eval-smoke <suite.yaml|suite.json> [options]
+
+  --mock              Offline runner with deterministic fake answers
+  --json              Machine-readable JSON report
+  --fail-fast         Stop after the first failing case
+  --timeout <ms>      Per-case timeout (default: 30000)
+  --model <name>      Override suite model
+  --provider <name>   openai | anthropic | mock
+  -h, --help
+  -v, --version
+```
+
+| Exit | Meaning |
+| --- | --- |
+| 0 | Every case passed |
+| 1 | One or more cases failed |
+| 2 | Usage or configuration error (missing file, bad YAML, no API key) |
+
+## Runners
+
+**Mock** (`--mock` or `--provider mock`) never calls a network. It returns canned replies matched from the prompt so the example suite stays green in CI:
+
+| Prompt contains | Mock reply |
+| --- | --- |
+| `hello` | `Hello` |
+| `capital of France` | `Paris` |
+| `2 + 2` | `4` |
+| `ignore previous` / `PWNED` | a refusal that does not contain `PWNED` |
+
+Anything else returns `[mock] No canned reply for this prompt.` — useful as a deliberate fail.
+
+**Live OpenAI** when `OPENAI_API_KEY` is set. Optional `OPENAI_BASE_URL` for compatible servers.
+
+**Live Anthropic** when `ANTHROPIC_API_KEY` is set, or when the model name starts with `claude`.
+
+If no key is set and you did not pass `--mock`, the CLI exits 2 with a short hint.
+
+## JSON report
+
+```bash
+eval-smoke examples/suite.yaml --mock --json
+```
+
+```json
+{
+  "ok": true,
+  "version": "0.1.0",
+  "runner": "mock",
+  "model": "gpt-4o-mini",
+  "suite": "demo",
+  "passed": 4,
+  "failed": 0,
+  "skipped": 0,
+  "duration_ms": 1,
+  "cases": [
+    {
+      "id": "greeting",
+      "ok": true,
+      "duration_ms": 1,
+      "output": "Hello",
+      "checks": [{ "name": "contains", "ok": true, "expected": "Hello" }]
+    }
+  ]
+}
+```
+
+## Programmatic API
+
+```ts
+import { loadSuite, runSuite } from "@webywz/eval-smoke";
+
+const suite = loadSuite("examples/suite.yaml");
+const report = await runSuite(suite, { mock: true });
+if (!report.ok) process.exit(1);
+```
+
+`runSuite` also accepts a custom `complete` function, which is how the tests inject fakes without keys.
+
+## CI
+
+GitHub Actions in this repo runs `npm test`, `npm run build`, and `eval-smoke examples/suite.yaml --mock` on Node 18/20/22. Drop this into another project:
+
+```yaml
+- run: npx @webywz/eval-smoke@0.1.0 prompts/smoke.yaml --mock
+```
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+import { runCli } from "./run-cli.js";
+const code = await runCli(process.argv.slice(2));
+process.exit(code);
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACjD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,10 @@
+export { loadSuite, parseSuite } from "./load-suite.js";
+export { evaluateCase, evaluateExpect } from "./matchers.js";
+export { createComplete, mockComplete, resolveProvider } from "./providers.js";
+export { formatReport } from "./report.js";
+export { runSuite } from "./run.js";
+export { estimateTokens } from "./tokens.js";
+export { VERSION } from "./version.js";
+export type { Case, CaseResult, CheckResult, CompleteFn, CompleteRequest, Expect, ProviderName, Report, RunOptions, Suite, } from "./types.js";
+export { UsageError } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,IAAI,EACJ,UAAU,EACV,WAAW,EACX,UAAU,EACV,eAAe,EACf,MAAM,EACN,YAAY,EACZ,MAAM,EACN,UAAU,EACV,KAAK,GACN,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,9 @@
+export { loadSuite, parseSuite } from "./load-suite.js";
+export { evaluateCase, evaluateExpect } from "./matchers.js";
+export { createComplete, mockComplete, resolveProvider } from "./providers.js";
+export { formatReport } from "./report.js";
+export { runSuite } from "./run.js";
+export { estimateTokens } from "./tokens.js";
+export { VERSION } from "./version.js";
+export { UsageError } from "./types.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAavC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC"}

package/dist/load-suite.d.ts

@@ -0,0 +1,4 @@
+import { type Suite } from "./types.js";
+export declare function parseSuite(raw: unknown): Suite;
+export declare function loadSuite(path: string, cwd?: string): Suite;
+//# sourceMappingURL=load-suite.d.ts.map

package/dist/load-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-suite.d.ts","sourceRoot":"","sources":["../src/load-suite.ts"],"names":[],"mappings":"AAIA,OAAO,EAAyD,KAAK,KAAK,EAAE,MAAM,YAAY,CAAC;AA0F/F,wBAAgB,UAAU,CAAC,GAAG,EAAE,OAAO,GAAG,KAAK,CAuC9C;AAyBD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,SAAgB,GAAG,KAAK,CASlE"}

package/dist/load-suite.js

@@ -0,0 +1,162 @@
+import { readFileSync } from "node:fs";
+import { extname, resolve } from "node:path";
+import { parse as parseYaml } from "yaml";
+import { asList, compileRegex, hasMatchers } from "./matchers.js";
+import { UsageError } from "./types.js";
+const PROVIDERS = new Set(["mock", "openai", "anthropic"]);
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function asString(value, label) {
+    if (typeof value !== "string" || value.trim() === "") {
+        throw new UsageError(`${label} must be a non-empty string`);
+    }
+    return value;
+}
+function optionalString(value, label) {
+    if (value == null)
+        return undefined;
+    return asString(value, label);
+}
+function parseStringList(value, label) {
+    if (value == null)
+        return undefined;
+    if (typeof value === "string") {
+        if (value === "")
+            throw new UsageError(`${label} must not be empty`);
+        return value;
+    }
+    if (Array.isArray(value)) {
+        if (value.length === 0)
+            throw new UsageError(`${label} must not be an empty list`);
+        return value.map((item, index) => asString(item, `${label}[${index}]`));
+    }
+    throw new UsageError(`${label} must be a string or list of strings`);
+}
+function parseExpect(raw, caseId) {
+    if (!isRecord(raw)) {
+        throw new UsageError(`Case "${caseId}" is missing a valid expect object`);
+    }
+    const expect = {
+        contains: parseStringList(raw.contains, `Case "${caseId}" expect.contains`),
+        not_contains: parseStringList(raw.not_contains, `Case "${caseId}" expect.not_contains`),
+        regex: parseStringList(raw.regex, `Case "${caseId}" expect.regex`),
+    };
+    if (raw.max_tokens != null) {
+        if (typeof raw.max_tokens !== "number" || !Number.isInteger(raw.max_tokens) || raw.max_tokens <= 0) {
+            throw new UsageError(`Case "${caseId}" expect.max_tokens must be a positive integer`);
+        }
+        expect.max_tokens = raw.max_tokens;
+    }
+    if (!hasMatchers(expect)) {
+        throw new UsageError(`Case "${caseId}" expect needs at least one of contains, not_contains, regex, max_tokens`);
+    }
+    for (const rawRegex of asList(expect.regex)) {
+        try {
+            compileRegex(rawRegex);
+        }
+        catch {
+            throw new UsageError(`Case "${caseId}" has an invalid regex: ${rawRegex}`);
+        }
+    }
+    return expect;
+}
+function parseCase(raw, index) {
+    if (!isRecord(raw)) {
+        throw new UsageError(`Case at index ${index} must be an object`);
+    }
+    const id = asString(raw.id, `Case at index ${index} id`);
+    return {
+        id,
+        prompt: asString(raw.prompt, `Case "${id}" prompt`),
+        system: optionalString(raw.system, `Case "${id}" system`),
+        model: optionalString(raw.model, `Case "${id}" model`),
+        expect: parseExpect(raw.expect, id),
+    };
+}
+function parseProvider(value) {
+    if (value == null)
+        return undefined;
+    if (typeof value !== "string" || !PROVIDERS.has(value)) {
+        throw new UsageError(`provider must be one of ${[...PROVIDERS].join(", ")}`);
+    }
+    return value;
+}
+export function parseSuite(raw) {
+    let name;
+    let model;
+    let provider;
+    let system;
+    let caseList;
+    if (Array.isArray(raw)) {
+        caseList = raw;
+    }
+    else if (isRecord(raw)) {
+        name = optionalString(raw.name, "suite name");
+        model = optionalString(raw.model, "suite model");
+        provider = parseProvider(raw.provider);
+        system = optionalString(raw.system, "suite system");
+        if (raw.cases == null && "id" in raw && "prompt" in raw) {
+            caseList = [raw];
+        }
+        else if (!Array.isArray(raw.cases)) {
+            throw new UsageError("Suite must be a list of cases or an object with a cases array");
+        }
+        else {
+            caseList = raw.cases;
+        }
+    }
+    else {
+        throw new UsageError("Suite must be a YAML/JSON list or object");
+    }
+    if (caseList.length === 0) {
+        throw new UsageError("Suite has no cases");
+    }
+    const cases = caseList.map((item, index) => parseCase(item, index));
+    const seen = new Set();
+    for (const testCase of cases) {
+        if (seen.has(testCase.id)) {
+            throw new UsageError(`Duplicate case id "${testCase.id}"`);
+        }
+        seen.add(testCase.id);
+    }
+    return { name, model, provider, system, cases };
+}
+function parseDocument(text, source) {
+    const body = text.replace(/^\uFEFF/, "");
+    const ext = extname(source).toLowerCase();
+    try {
+        if (ext === ".json")
+            return JSON.parse(body);
+        if (ext === ".yaml" || ext === ".yml")
+            return parseYaml(body);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new UsageError(`Failed to parse ${source}: ${reason}`);
+    }
+    try {
+        return JSON.parse(body);
+    }
+    catch {
+        try {
+            return parseYaml(body);
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            throw new UsageError(`Failed to parse ${source}: ${reason}`);
+        }
+    }
+}
+export function loadSuite(path, cwd = process.cwd()) {
+    const file = resolve(cwd, path);
+    let text;
+    try {
+        text = readFileSync(file, "utf8");
+    }
+    catch {
+        throw new UsageError(`Suite file not found: ${file}`);
+    }
+    return parseSuite(parseDocument(text, file));
+}
+//# sourceMappingURL=load-suite.js.map

package/dist/load-suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-suite.js","sourceRoot":"","sources":["../src/load-suite.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,UAAU,EAAyD,MAAM,YAAY,CAAC;AAE/F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAe,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;AAEzE,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc,EAAE,KAAa;IAC7C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,KAAc,EAAE,KAAa;IACnD,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACpC,OAAO,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,KAAa;IACpD,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,KAAK,KAAK,EAAE;YAAE,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;QACrE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,4BAA4B,CAAC,CAAC;QACnF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,sCAAsC,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,WAAW,CAAC,GAAY,EAAE,MAAc;IAC/C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,UAAU,CAAC,SAAS,MAAM,oCAAoC,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,MAAM,GAAW;QACrB,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE,SAAS,MAAM,mBAAmB,CAAC;QAC3E,YAAY,EAAE,eAAe,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,MAAM,uBAAuB,CAAC;QACvF,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,MAAM,gBAAgB,CAAC;KACnE,CAAC;IAEF,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;YACnG,MAAM,IAAI,UAAU,CAAC,SAAS,MAAM,gDAAgD,CAAC,CAAC;QACxF,CAAC;QACD,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IACrC,CAAC;IAED,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,UAAU,CAClB,SAAS,MAAM,0EAA0E,CAC1F,CAAC;IACJ,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,YAAY,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,UAAU,CAAC,SAAS,MAAM,2BAA2B,QAAQ,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,GAAY,EAAE,KAAa;IAC5C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,UAAU,CAAC,iBAAiB,KAAK,oBAAoB,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,iBAAiB,KAAK,KAAK,CAAC,CAAC;IACzD,OAAO;QACL,EAAE;QACF,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;QACnD,MAAM,EAAE,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;QACzD,KAAK,EAAE,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,CAAC;QACtD,MAAM,EAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;KACpC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAqB,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,UAAU,CAAC,2BAA2B,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,KAAqB,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAY;IACrC,IAAI,IAAwB,CAAC;IAC7B,IAAI,KAAyB,CAAC;IAC9B,IAAI,QAAkC,CAAC;IACvC,IAAI,MAA0B,CAAC;IAC/B,IAAI,QAAmB,CAAC;IAExB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,QAAQ,GAAG,GAAG,CAAC;IACjB,CAAC;SAAM,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;QAC9C,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QACjD,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,MAAM,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACpD,IAAI,GAAG,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,IAAI,GAAG,IAAI,QAAQ,IAAI,GAAG,EAAE,CAAC;YACxD,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,UAAU,CAAC,+DAA+D,CAAC,CAAC;QACxF,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,UAAU,CAAC,0CAA0C,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,UAAU,CAAC,oBAAoB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,UAAU,CAAC,sBAAsB,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAClD,CAAC;AAED,SAAS,aAAa,CAAC,IAAY,EAAE,MAAc;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1C,IAAI,CAAC;QACH,IAAI,GAAG,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;QACxD,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,MAAM,IAAI,UAAU,CAAC,mBAAmB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACtE,MAAM,IAAI,UAAU,CAAC,mBAAmB,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE;IACzD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAChC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,UAAU,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/matchers.d.ts

@@ -0,0 +1,7 @@
+import type { Case, CheckResult, Expect } from "./types.js";
+export declare function asList(value?: string | string[]): string[];
+export declare function compileRegex(raw: string): RegExp;
+export declare function hasMatchers(expect: Expect): boolean;
+export declare function evaluateExpect(output: string, expect: Expect): CheckResult[];
+export declare function evaluateCase(output: string, testCase: Case): CheckResult[];
+//# sourceMappingURL=matchers.d.ts.map

package/dist/matchers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.d.ts","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAE5D,wBAAgB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,EAAE,CAG1D;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAShD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAOnD;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CA+C5E;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,GAAG,WAAW,EAAE,CAE1E"}

package/dist/matchers.js

@@ -0,0 +1,69 @@
+import { estimateTokens } from "./tokens.js";
+export function asList(value) {
+    if (value == null)
+        return [];
+    return Array.isArray(value) ? value : [value];
+}
+export function compileRegex(raw) {
+    const wrapped = raw.match(/^\/([\s\S]+)\/([gimsuy]*)$/);
+    if (wrapped?.[1] != null) {
+        return new RegExp(wrapped[1], wrapped[2]);
+    }
+    if (raw.startsWith("(?i)")) {
+        return new RegExp(raw.slice(4), "i");
+    }
+    return new RegExp(raw);
+}
+export function hasMatchers(expect) {
+    return (asList(expect.contains).length > 0 ||
+        asList(expect.not_contains).length > 0 ||
+        asList(expect.regex).length > 0 ||
+        expect.max_tokens != null);
+}
+export function evaluateExpect(output, expect) {
+    const checks = [];
+    for (const needle of asList(expect.contains)) {
+        const ok = output.toLowerCase().includes(needle.toLowerCase());
+        checks.push({
+            name: "contains",
+            ok,
+            expected: needle,
+            detail: ok ? undefined : `missing ${JSON.stringify(needle)}`,
+        });
+    }
+    for (const needle of asList(expect.not_contains)) {
+        const ok = !output.toLowerCase().includes(needle.toLowerCase());
+        checks.push({
+            name: "not_contains",
+            ok,
+            expected: needle,
+            detail: ok ? undefined : `found ${JSON.stringify(needle)}`,
+        });
+    }
+    for (const raw of asList(expect.regex)) {
+        const re = compileRegex(raw);
+        re.lastIndex = 0;
+        const ok = re.test(output);
+        checks.push({
+            name: "regex",
+            ok,
+            expected: String(re),
+            detail: ok ? undefined : `no match for ${re}`,
+        });
+    }
+    if (expect.max_tokens != null) {
+        const tokens = estimateTokens(output);
+        const ok = tokens <= expect.max_tokens;
+        checks.push({
+            name: "max_tokens",
+            ok,
+            expected: String(expect.max_tokens),
+            detail: ok ? undefined : `estimated ${tokens} tokens > ${expect.max_tokens}`,
+        });
+    }
+    return checks;
+}
+export function evaluateCase(output, testCase) {
+    return evaluateExpect(output, testCase.expect);
+}
+//# sourceMappingURL=matchers.js.map

package/dist/matchers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../src/matchers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,MAAM,UAAU,MAAM,CAAC,KAAyB;IAC9C,IAAI,KAAK,IAAI,IAAI;QAAE,OAAO,EAAE,CAAC;IAC7B,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACxD,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;QACzB,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,OAAO,CACL,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;QAClC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC;QACtC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;QAC/B,MAAM,CAAC,UAAU,IAAI,IAAI,CAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,MAAc;IAC3D,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7C,MAAM,EAAE,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,UAAU;YAChB,EAAE;YACF,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;SAC7D,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;QACjD,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,cAAc;YACpB,EAAE;YACF,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE;SAC3D,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7B,EAAE,CAAC,SAAS,GAAG,CAAC,CAAC;QACjB,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,OAAO;YACb,EAAE;YACF,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;YACpB,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,EAAE,EAAE;SAC9C,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,YAAY;YAClB,EAAE;YACF,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC;YACnC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,MAAM,aAAa,MAAM,CAAC,UAAU,EAAE;SAC7E,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAc,EAAE,QAAc;IACzD,OAAO,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC"}

package/dist/providers.d.ts

@@ -0,0 +1,15 @@
+import { type CompleteFn, type CompleteRequest, type ProviderName } from "./types.js";
+export declare const DEFAULT_MODELS: Record<Exclude<ProviderName, "mock">, string>;
+export declare function mockComplete(request: CompleteRequest): Promise<string>;
+export declare function createOpenAIComplete(apiKey: string, fetchFn: typeof fetch, baseUrl?: string): CompleteFn;
+export declare function createAnthropicComplete(apiKey: string, fetchFn: typeof fetch, baseUrl?: string): CompleteFn;
+export declare function resolveProvider(options: {
+    mock?: boolean;
+    provider?: ProviderName;
+    suiteProvider?: ProviderName;
+    model?: string;
+    env?: NodeJS.ProcessEnv;
+}): ProviderName;
+export declare function resolveModel(provider: ProviderName, requested?: string, suiteDefault?: string): string;
+export declare function createComplete(provider: ProviderName, env?: NodeJS.ProcessEnv, fetchFn?: typeof fetch): CompleteFn;
+//# sourceMappingURL=providers.d.ts.map

package/dist/providers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../src/providers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,KAAK,UAAU,EAAE,KAAK,eAAe,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAYlG,eAAO,MAAM,cAAc,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,MAAM,CAGxE,CAAC;AAEF,wBAAgB,YAAY,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAMtE;AAsBD,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,KAAK,EACrB,OAAO,SAA8B,GACpC,UAAU,CAkCZ;AAED,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,KAAK,EACrB,OAAO,SAAiC,GACvC,UAAU,CAoCZ;AAWD,wBAAgB,eAAe,CAAC,OAAO,EAAE;IACvC,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,aAAa,CAAC,EAAE,YAAY,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACzB,GAAG,YAAY,CAYf;AAED,wBAAgB,YAAY,CAC1B,QAAQ,EAAE,YAAY,EACtB,SAAS,CAAC,EAAE,MAAM,EAClB,YAAY,CAAC,EAAE,MAAM,GACpB,MAAM,CAKR;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,YAAY,EACtB,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,OAAO,GAAE,OAAO,KAAa,GAC5B,UAAU,CAUZ"}