@cliwatch/cli-bench 0.4.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CLIWatch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,145 @@
+# @cliwatch/cli-bench
+
+LLM CLI agent testing framework — benchmark how well AI models use your CLI tool. Runs tasks directly on the host (no Docker required), has models execute commands via tool-calling, and validates results with assertions.
+
+## Quick start
+
+```bash
+# 1. Scaffold a config file
+npx @cliwatch/cli-bench init
+
+# 2. Edit cli-bench.yaml (define your CLI, providers, tasks)
+
+# 3. Run (locally or in CI)
+npx @cliwatch/cli-bench
+```
+
+## Config file (`cli-bench.yaml`)
+
+```yaml
+cli: docker
+version_command: "docker --version"
+
+providers:
+  - anthropic/claude-sonnet-4-20250514
+  - openai/gpt-4o
+
+tasks:
+  - id: pull-image
+    intent: "Pull the latest nginx image"
+    assert:
+      - ran: "docker pull.*nginx"
+      - verify:
+          run: "docker images nginx --format '{{.Repository}}'"
+          output_contains: "nginx"
+
+  - id: create-project
+    intent: "Create a new project called my-app"
+    setup:
+      - "mkdir -p /tmp/bench-workspace"
+    assert:
+      - ran: "mycli create.*my-app"
+      - exit_code: 0
+      - file_exists: "/tmp/bench-workspace/my-app/package.json"
+      - verify:
+          run: "mycli list --json"
+          output_contains: "my-app"
+```
+
+### Split tasks across files
+
+```yaml
+cli: docker
+providers: [anthropic/claude-sonnet-4-20250514]
+tasks:
+  - file://tasks/basics.yaml
+  - file://tasks/advanced/*.yaml
+  - file://tasks/**/*.yaml          # recursive glob
+```
+
+Each referenced file is a plain array of tasks:
+
+```yaml
+# tasks/basics.yaml
+- id: list-containers
+  intent: "List all running containers"
+  assert:
+    - ran: "docker ps"
+    - exit_code: 0
+```
+
+### Config fields
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `cli` | Yes | CLI name (must be in PATH) |
+| `version_command` | No | e.g. "mycli --version", for tracking |
+| `providers` | No | Model IDs (default: claude-sonnet-4) |
+| `help_modes` | No | injected, discoverable, none (default: [injected]) |
+| `concurrency` | No | Max concurrent API calls (default: 3) |
+| `workdir` | No | Working directory (default: temp dir per task) |
+| `upload` | No | auto, always, never (default: auto) |
+| `backend_url` | No | Dashboard URL (default: https://api.cliwatch.dev) |
+| `tasks` | Yes | Array of tasks or `file://` references |
+
+## Assertion types
+
+| Assertion | Example | Description |
+|-----------|---------|-------------|
+| `ran` | `ran: "docker ps"` | Agent ran a command matching regex |
+| `not_ran` | `not_ran: "rm -rf"` | No command matched regex |
+| `run_count` | `run_count: {pattern: "curl", min: 1, max: 3}` | Count of matching commands |
+| `output_contains` | `output_contains: "hello"` | Last command stdout contains |
+| `output_equals` | `output_equals: "ok"` | Last command stdout exact match |
+| `error_contains` | `error_contains: "warning"` | Last command stderr contains |
+| `exit_code` | `exit_code: 0` | Last command exit code |
+| `file_exists` | `file_exists: "./my-app/package.json"` | File exists |
+| `file_contains` | `file_contains: {path: "...", text: "..."}` | File content check |
+| `verify` | `verify: {run: "cmd", output_contains: "ok"}` | Run post-agent command, check output |
+
+`verify` is the universal escape hatch — runs any command after the agent finishes.
+
+## GitHub Actions
+
+```yaml
+steps:
+  - uses: actions/checkout@v4
+  - uses: actions/setup-node@v4
+    with: { node-version: 22 }
+  - run: npm install -g my-cli
+  - run: npx @cliwatch/cli-bench
+    env:
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      CLIWATCH_API_KEY: ${{ secrets.CLIWATCH_API_KEY }}  # optional, uploads to dashboard
+```
+
+No Docker required. Commands run directly on the CI runner.
+
+## Environment variables
+
+| Variable | Description |
+|----------|-------------|
+| `ANTHROPIC_API_KEY` | Anthropic API key (for `anthropic/*` models) |
+| `OPENAI_API_KEY` | OpenAI API key (for `openai/*` models) |
+| `GOOGLE_GENERATIVE_AI_API_KEY` | Google AI API key (for `google/*` models) |
+| `CLIWATCH_API_KEY` | API key for uploading results to cliwatch.dev |
+| `CLIWATCH_BACKEND_URL` | Override backend URL (default: `https://api.cliwatch.dev`) |
+
+## Uploading results
+
+Results upload automatically when `CLIWATCH_API_KEY` is set (default `upload: auto`). Override with `upload: always` or `upload: never` in your config, or pass `--upload` on the CLI.
+
+## Available models
+
+| Model ID | Provider |
+|----------|----------|
+| `anthropic/claude-sonnet-4-20250514` | Anthropic |
+| `anthropic/claude-haiku-4-5-20251001` | Anthropic |
+| `openai/gpt-4o` | OpenAI |
+| `google/gemini-2.5-pro` | Google |
+
+Any model supported by the Vercel AI SDK gateway can be used — just pass the full `provider/model-id`.
+
+## License
+
+MIT

package/dist/assertions.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Assertion engine for @cliwatch/cli-bench.
+ *
+ * Evaluates assertions against the agent's execution trace.
+ */
+import type { Assertion, AssertionResult } from './models.js';
+export interface ExecutionTrace {
+    commandsRun: string[];
+    lastStdout: string;
+    lastStderr: string;
+    lastExitCode: number;
+}
+export declare function runAssertions(assertions: Assertion[], trace: ExecutionTrace, workdir?: string): Promise<AssertionResult[]>;
+//# sourceMappingURL=assertions.d.ts.map

package/dist/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG9D,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wBAAsB,aAAa,CACjC,UAAU,EAAE,SAAS,EAAE,EACvB,KAAK,EAAE,cAAc,EACrB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,eAAe,EAAE,CAAC,CAQ5B"}

package/dist/assertions.js

@@ -0,0 +1,161 @@
+/**
+ * Assertion engine for @cliwatch/cli-bench.
+ *
+ * Evaluates assertions against the agent's execution trace.
+ */
+import { execCommand } from './exec.js';
+export async function runAssertions(assertions, trace, workdir) {
+    const results = [];
+    for (const assertion of assertions) {
+        results.push(await evaluateAssertion(assertion, trace, workdir));
+    }
+    return results;
+}
+async function evaluateAssertion(assertion, trace, workdir) {
+    if ('output_contains' in assertion) {
+        const expected = assertion.output_contains;
+        const passed = trace.lastStdout.includes(expected);
+        return {
+            assertion,
+            passed,
+            actual: truncate(trace.lastStdout),
+            expected,
+        };
+    }
+    if ('output_equals' in assertion) {
+        const expected = assertion.output_equals;
+        const passed = trace.lastStdout.trim() === expected.trim();
+        return {
+            assertion,
+            passed,
+            actual: truncate(trace.lastStdout.trim()),
+            expected: expected.trim(),
+        };
+    }
+    if ('error_contains' in assertion) {
+        const expected = assertion.error_contains;
+        const passed = trace.lastStderr.includes(expected);
+        return {
+            assertion,
+            passed,
+            actual: truncate(trace.lastStderr),
+            expected,
+        };
+    }
+    if ('exit_code' in assertion) {
+        const expected = assertion.exit_code;
+        const passed = trace.lastExitCode === expected;
+        return {
+            assertion,
+            passed,
+            actual: String(trace.lastExitCode),
+            expected: String(expected),
+        };
+    }
+    if ('file_exists' in assertion) {
+        const path = assertion.file_exists;
+        const result = await execCommand(`test -e '${shellEscape(path)}'`, { cwd: workdir });
+        const passed = result.exitCode === 0;
+        return {
+            assertion,
+            passed,
+            actual: passed ? 'exists' : 'not found',
+            expected: path,
+        };
+    }
+    if ('file_contains' in assertion) {
+        const { path, text } = assertion.file_contains;
+        const result = await execCommand(`cat '${shellEscape(path)}'`, { cwd: workdir });
+        const passed = result.exitCode === 0 && result.stdout.includes(text);
+        return {
+            assertion,
+            passed,
+            actual: truncate(result.stdout),
+            expected: text,
+        };
+    }
+    if ('ran' in assertion) {
+        const pattern = assertion.ran;
+        let passed;
+        try {
+            const re = new RegExp(pattern);
+            passed = trace.commandsRun.some((c) => re.test(c));
+        }
+        catch {
+            passed = trace.commandsRun.some((c) => c.includes(pattern));
+        }
+        return {
+            assertion,
+            passed,
+            actual: trace.commandsRun.join(' | '),
+            expected: pattern,
+        };
+    }
+    if ('not_ran' in assertion) {
+        const pattern = assertion.not_ran;
+        let matched;
+        try {
+            const re = new RegExp(pattern);
+            matched = trace.commandsRun.some((c) => re.test(c));
+        }
+        catch {
+            matched = trace.commandsRun.some((c) => c.includes(pattern));
+        }
+        return {
+            assertion,
+            passed: !matched,
+            actual: trace.commandsRun.join(' | '),
+            expected: `NOT ${pattern}`,
+        };
+    }
+    if ('run_count' in assertion) {
+        const { pattern, min, max } = assertion.run_count;
+        let count;
+        try {
+            const re = new RegExp(pattern);
+            count = trace.commandsRun.filter((c) => re.test(c)).length;
+        }
+        catch {
+            count = trace.commandsRun.filter((c) => c.includes(pattern)).length;
+        }
+        const passMin = min === undefined || count >= min;
+        const passMax = max === undefined || count <= max;
+        return {
+            assertion,
+            passed: passMin && passMax,
+            actual: String(count),
+            expected: `${min ?? '*'}..${max ?? '*'}`,
+        };
+    }
+    if ('verify' in assertion) {
+        const { run, output_contains, output_equals } = assertion.verify;
+        const result = await execCommand(run, { cwd: workdir });
+        let passed = result.exitCode === 0;
+        if (passed && output_contains) {
+            passed = result.stdout.includes(output_contains);
+        }
+        if (passed && output_equals) {
+            passed = result.stdout.trim() === output_equals.trim();
+        }
+        return {
+            assertion,
+            passed,
+            actual: truncate(result.stdout),
+            expected: output_contains ?? output_equals ?? 'exit 0',
+        };
+    }
+    return {
+        assertion,
+        passed: false,
+        actual: 'unknown assertion type',
+    };
+}
+function truncate(s, max = 200) {
+    if (s.length <= max)
+        return s;
+    return s.slice(0, max) + '...';
+}
+function shellEscape(s) {
+    return s.replace(/'/g, "'\\''");
+}
+//# sourceMappingURL=assertions.js.map

package/dist/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AASxC,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAuB,EACvB,KAAqB,EACrB,OAAgB;IAEhB,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAAoB,EACpB,KAAqB,EACrB,OAAgB;IAEhB,IAAI,iBAAiB,IAAI,SAAS,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,SAAS,CAAC,eAAe,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnD,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC;YAClC,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,aAAa,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3D,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACzC,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE;SAC1B,CAAC;IACJ,CAAC;IAED,IAAI,gBAAgB,IAAI,SAAS,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,cAAc,CAAC;QAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnD,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC;YAClC,QAAQ;SACT,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,SAAS,CAAC;QACrC,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,KAAK,QAAQ,CAAC;QAC/C,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC;YAClC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;SAC3B,CAAC;IACJ,CAAC;IAED,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,YAAY,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACrF,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;QACrC,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;YACvC,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,aAAa,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACjF,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrE,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YAC/B,QAAQ,EAAE,IAAI;SACf,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC;QAC9B,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/B,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YACrC,QAAQ,EAAE,OAAO;SAClB,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC;QAClC,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/B,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/D,CAAC;QACD,OAAO;YACL,SAAS;YACT,MAAM,EAAE,CAAC,OAAO;YAChB,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;YACrC,QAAQ,EAAE,OAAO,OAAO,EAAE;SAC3B,CAAC;IACJ,CAAC;IAED,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,SAAS,CAAC;QAClD,IAAI,KAAa,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;YAC/B,KAAK,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7D,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,GAAG,CAAC;QAClD,MAAM,OAAO,GAAG,GAAG,KAAK,SAAS,IAAI,KAAK,IAAI,GAAG,CAAC;QAClD,OAAO;YACL,SAAS;YACT,MAAM,EAAE,OAAO,IAAI,OAAO;YAC1B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;YACrB,QAAQ,EAAE,GAAG,GAAG,IAAI,GAAG,KAAK,GAAG,IAAI,GAAG,EAAE;SACzC,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;QACjE,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,IAAI,MAAM,GAAG,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;QACnC,IAAI,MAAM,IAAI,eAAe,EAAE,CAAC;YAC9B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;YAC5B,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,aAAa,CAAC,IAAI,EAAE,CAAC;QACzD,CAAC;QACD,OAAO;YACL,SAAS;YACT,MAAM;YACN,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YAC/B,QAAQ,EAAE,eAAe,IAAI,aAAa,IAAI,QAAQ;SACvD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,KAAK;QACb,MAAM,EAAE,wBAAwB;KACjC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,GAAG,GAAG,GAAG;IACpC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC"}

package/dist/ci.d.ts

@@ -0,0 +1,29 @@
+/**
+ * CI environment auto-detection and task suite hashing.
+ */
+import type { Task } from './models.js';
+export interface CIMetadata {
+    gitSha?: string;
+    gitRef?: string;
+    gitBaseRef?: string;
+    commitMessage?: string;
+    gitAuthor?: string;
+    ciProvider?: string;
+    ciBuildUrl?: string;
+    ciEvent?: string;
+    prNumber?: number;
+    repository?: string;
+}
+/**
+ * Detect CI environment from environment variables.
+ * Falls back to local git info when not running in CI.
+ */
+export declare function detectCI(): CIMetadata;
+/**
+ * Compute a fingerprint for a set of tasks so only runs with
+ * the same task suite can be compared in trend views.
+ *
+ * Sort task IDs, concatenate id+intent+assertions, SHA256, take first 12 chars.
+ */
+export declare function computeTaskSuiteHash(tasks: Task[]): string;
+//# sourceMappingURL=ci.d.ts.map

package/dist/ci.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.d.ts","sourceRoot":"","sources":["../src/ci.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAExC,MAAM,WAAW,UAAU;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,QAAQ,IAAI,UAAU,CAOrC;AAqDD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,CAM1D"}

package/dist/ci.js

@@ -0,0 +1,75 @@
+/**
+ * CI environment auto-detection and task suite hashing.
+ */
+import { createHash } from 'node:crypto';
+import { execSync } from 'node:child_process';
+/**
+ * Detect CI environment from environment variables.
+ * Falls back to local git info when not running in CI.
+ */
+export function detectCI() {
+    if (process.env['GITHUB_ACTIONS']) {
+        return detectGitHubActions();
+    }
+    // Fallback: local git info
+    return detectLocalGit();
+}
+function detectGitHubActions() {
+    const env = process.env;
+    const serverUrl = env['GITHUB_SERVER_URL'] ?? 'https://github.com';
+    const repository = env['GITHUB_REPOSITORY'];
+    const runId = env['GITHUB_RUN_ID'];
+    const gitRef = env['GITHUB_REF'];
+    let prNumber;
+    if (gitRef) {
+        const match = gitRef.match(/^refs\/pull\/(\d+)\/merge$/);
+        if (match) {
+            prNumber = parseInt(match[1], 10);
+        }
+    }
+    return {
+        gitSha: env['GITHUB_SHA'],
+        gitRef,
+        gitBaseRef: env['GITHUB_BASE_REF'] || undefined,
+        gitAuthor: env['GITHUB_ACTOR'],
+        ciProvider: 'github-actions',
+        ciBuildUrl: repository && runId
+            ? `${serverUrl}/${repository}/actions/runs/${runId}`
+            : undefined,
+        ciEvent: env['GITHUB_EVENT_NAME'],
+        repository,
+        prNumber,
+    };
+}
+function detectLocalGit() {
+    const meta = {};
+    try {
+        meta.gitSha = execSync('git rev-parse HEAD', { encoding: 'utf-8' }).trim();
+    }
+    catch {
+        // Not a git repo or git not available
+    }
+    try {
+        meta.gitRef = execSync('git rev-parse --abbrev-ref HEAD', {
+            encoding: 'utf-8',
+        }).trim();
+    }
+    catch {
+        // Ignore
+    }
+    return meta;
+}
+/**
+ * Compute a fingerprint for a set of tasks so only runs with
+ * the same task suite can be compared in trend views.
+ *
+ * Sort task IDs, concatenate id+intent+assertions, SHA256, take first 12 chars.
+ */
+export function computeTaskSuiteHash(tasks) {
+    const sorted = [...tasks].sort((a, b) => a.id.localeCompare(b.id));
+    const content = sorted
+        .map((t) => `${t.id}:${t.intent}:${JSON.stringify(t.assert)}`)
+        .join('|');
+    return createHash('sha256').update(content).digest('hex').slice(0, 12);
+}
+//# sourceMappingURL=ci.js.map

package/dist/ci.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ci.js","sourceRoot":"","sources":["../src/ci.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAgB9C;;;GAGG;AACH,MAAM,UAAU,QAAQ;IACtB,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAClC,OAAO,mBAAmB,EAAE,CAAC;IAC/B,CAAC;IAED,2BAA2B;IAC3B,OAAO,cAAc,EAAE,CAAC;AAC1B,CAAC;AAED,SAAS,mBAAmB;IAC1B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC;IACxB,MAAM,SAAS,GAAG,GAAG,CAAC,mBAAmB,CAAC,IAAI,oBAAoB,CAAC;IACnE,MAAM,UAAU,GAAG,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,eAAe,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAEjC,IAAI,QAA4B,CAAC;IACjC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACzD,IAAI,KAAK,EAAE,CAAC;YACV,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,YAAY,CAAC;QACzB,MAAM;QACN,UAAU,EAAE,GAAG,CAAC,iBAAiB,CAAC,IAAI,SAAS;QAC/C,SAAS,EAAE,GAAG,CAAC,cAAc,CAAC;QAC9B,UAAU,EAAE,gBAAgB;QAC5B,UAAU,EACR,UAAU,IAAI,KAAK;YACjB,CAAC,CAAC,GAAG,SAAS,IAAI,UAAU,iBAAiB,KAAK,EAAE;YACpD,CAAC,CAAC,SAAS;QACf,OAAO,EAAE,GAAG,CAAC,mBAAmB,CAAC;QACjC,UAAU;QACV,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,SAAS,cAAc;IACrB,MAAM,IAAI,GAAe,EAAE,CAAC;IAE5B,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,oBAAoB,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;IACxC,CAAC;IAED,IAAI,CAAC;QACH,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,iCAAiC,EAAE;YACxD,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,SAAS;IACX,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,MAAM;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;SAC7D,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC"}

package/dist/client/client/client.gen.d.ts

@@ -0,0 +1,3 @@
+import type { Client, Config } from './types.gen.js';
+export declare const createClient: (config?: Config) => Client;
+//# sourceMappingURL=client.gen.d.ts.map

package/dist/client/client/client.gen.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.gen.d.ts","sourceRoot":"","sources":["../../../src/client/client/client.gen.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAA0C,MAAM,gBAAgB,CAAC;AAgB7F,eAAO,MAAM,YAAY,GAAI,SAAQ,MAAW,KAAG,MA0QlD,CAAC"}