@eidra-umain/greenlight 0.1.0

package/README.md

@@ -0,0 +1,391 @@
+<p align="center">
+  <img src="assets/greenlight_banner.png" alt="GreenLight — AI-driven E2E Testing" width="500">
+</p>
+
+# GreenLight
+
+Natural language driven end-to-end testing for web applications. Write tests as plain-English user stories, and an AI agent (the Pilot) executes them against your staging environment using a real browser.
+
+No selectors. No XPaths. No test IDs, drivers or glue code. Just describe what a user would do.
+
+---
+
+**[How it works](#how-it-works)** | **[Quick start](#quick-start)** | **[Project configuration](#project-configuration)** | **[CLI](#cli)** | **[Test syntax](#test-syntax)** | **[Cached plans](#cached-plans)** | **[LLM setup](#llm-setup)** | **[Architecture](#architecture)** | **[CI/CD](#cicd)**
+
+---
+
+## How it works
+
+```yaml
+suite: "Product Search"
+base_url: "https://staging.example.com"
+
+tests:
+  - name: "Filtering reduces results"
+    steps:
+      - navigate to Products from the main menu
+      - remember the number of results shown
+      - select "Electronics" in the category filter
+      - check that the number of results has decreased
+      - search for "wireless headphones"
+      - check that the page contains "wireless"
+      - fill in the inquiry form with email "test@example.com" and some test data
+      - submit the form
+      - check that you see "Thanks for your inquiry"
+```
+
+GreenLight understands form wizards, custom dropdowns, autocomplete fields, and checkbox consent flows. It fills in forms with realistic test data, handles before/after value comparisons, and works with any UI framework.
+
+The first run uses an LLM to discover the right actions (the **discovery run**). After that, GreenLight caches a concrete action plan and replays it without LLM calls — making subsequent runs fast and deterministic.
+
+## Quick start
+
+1. Add GreenLight to your project:
+
+```bash
+npm install @eidra-umain/greenlight
+```
+
+2. Create a `greenlight.yaml` in your project root:
+
+```yaml
+suites:
+  - tests/e2e/login.yaml
+  - tests/e2e/checkout.yaml
+
+deployments:
+  staging:
+    base_url: https://staging.myapp.com
+```
+
+3. Run:
+
+```bash
+greenlight run
+```
+
+## Project configuration
+
+GreenLight looks for a `greenlight.yaml` in the working directory. This file defines which suites to run and supports multiple deployment targets.
+
+### Single deployment
+
+When there is only one deployment, it is used automatically:
+
+```yaml
+suites:
+  - tests/e2e/*.yaml
+
+deployments:
+  staging:
+    base_url: https://staging.myapp.com
+```
+
+### Multiple deployments
+
+```yaml
+suites:
+  - tests/e2e/*.yaml
+
+model: anthropic/claude-sonnet-4
+timeout: 15000
+
+deployments:
+  dev:
+    base_url: https://dev.myapp.com
+  staging:
+    base_url: https://staging.myapp.com
+  prod:
+    base_url: https://myapp.com
+    timeout: 30000
+
+default_deployment: staging
+```
+
+Shared settings go at the top level. Deployment-specific settings override them.
+
+```bash
+greenlight run                  # uses default_deployment (staging)
+greenlight run -d prod          # selects the prod deployment
+greenlight run -d dev           # selects the dev deployment
+```
+
+If there are multiple deployments and no `default_deployment` is set, the `--deployment` flag is required.
+
+### All config fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `suites` | string[] | Paths or globs to suite YAML files (required) |
+| `deployments` | map | Named deployment targets |
+| `default_deployment` | string | Which deployment to use by default |
+| `base_url` | string | Base URL for the site under test |
+| `model` | string | LLM model identifier |
+| `llm_base_url` | string | Base URL for the OpenAI-compatible API |
+| `timeout` | number | Per-step timeout in milliseconds |
+| `headed` | boolean | Run browser in visible mode |
+| `parallel` | number | Number of concurrent test cases |
+| `reporter` | string | Output format: `cli`, `json`, or `html` |
+| `viewport` | object | `{ width, height }` for the browser viewport |
+
+All fields except `suites` can appear at the top level or inside a deployment. Priority: **CLI flags > deployment > top-level config > built-in defaults**.
+
+## CLI
+
+```bash
+greenlight run [suites...]              # run suite YAML files (overrides greenlight.yaml)
+greenlight run                          # run suites from greenlight.yaml
+greenlight run -d, --deployment <name>  # select a named deployment
+greenlight run -t, --test <name>        # filter by test name
+greenlight run --base-url <url>         # override base URL
+greenlight run --headed                 # visible browser
+greenlight run -p, --parallel 4         # concurrent test cases
+greenlight run -r, --reporter json      # json output (also: cli, html)
+greenlight run -o, --output results.json  # write to file
+greenlight run --timeout 15000          # per-step timeout (ms)
+greenlight run --model openai/gpt-4o    # override LLM model
+greenlight run --llm-base-url <url>     # use a different OpenAI-compatible API
+greenlight run --debug                  # verbose output (actions, LLM modes, timings)
+greenlight run --trace                  # timestamped browser events for perf analysis
+greenlight run --discover               # force discovery run, ignore cached plans
+greenlight run --plan-status            # show cache status for all tests
+greenlight run --on-drift rerun         # re-discover on cached plan drift (default: fail)
+```
+
+## GreenLight philosophy compared to Gherkin/Cucumber
+
+Traditional BDD tools like Cucumber use **Gherkin** — a structured `Given/When/Then` syntax where every step requires a developer-written **step definition** (glue code) that maps the English phrase to actual browser automation with CSS selectors or XPaths.
+
+GreenLight takes a different approach:
+
+| | GreenLight | Gherkin (Cucumber) |
+|---|---|---|
+| **Test language** | Freeform plain English | Structured `Given/When/Then` keywords |
+| **Element targeting** | AI resolves via accessibility tree — no selectors | Developers write glue code with selectors/XPaths |
+| **Maintenance** | Tests survive UI refactors that don't change behavior | Selector changes break tests, requiring glue code updates |
+| **Authoring** | Non-technical testers, no code required | Readable specs, but developers must write step definitions |
+| **Determinism** | Cached plans are deterministic; discovery runs have LLM variability | Fully deterministic — same input, same execution path |
+| **Maturity** | New, LLM-dependent | Battle-tested (15+ years), broad ecosystem |
+
+**In short:** Gherkin requires developers to bridge English and browser automation via step definitions. GreenLight uses AI as that bridge — eliminating the glue code layer at the cost of introducing LLM-dependent variability.
+
+## Test syntax
+
+Tests are plain English. The Pilot interprets intent, so phrasing is flexible. Common patterns:
+
+| Action | Example |
+|--------|---------|
+| Navigate | `go to "/products"` or `navigate to About from the menu` |
+| Click | `click "Add to Cart"` or `click the Submit button` |
+| Type | `enter "jane@example.com" into "Email"` |
+| Select | `select "Canada" from "Country"` (works with native and custom dropdowns) |
+| Form fill | `fill in the contact form with email "a@b.com" and some test data` |
+| Autocomplete | `type "Stock" into the city field and select the first suggestion` |
+| Check | `check the "I agree to terms" checkbox` |
+| Remember | `remember the number of search results` |
+| Compare | `check that the number of results is less than before` |
+| Assert | `check that page contains "Order Confirmed"` |
+| Multi-step | `Select Red - Green - Blue in the color picker` (auto-split into 3 clicks) |
+
+### Form filling
+
+Steps like "fill in the form with some test data and submit it" are automatically expanded at runtime. GreenLight inspects the actual form fields (labels, placeholders, input types) and generates appropriate test data. Autocomplete fields are detected and handled with type-wait-select flows. Consent checkboxes are automatically checked.
+
+### Value comparisons
+
+Remember a value before an action, then compare after:
+
+```yaml
+steps:
+  - remember the total shown in the results badge
+  - apply the "In Stock" filter
+  - check that the total has decreased
+```
+
+### Reusable steps
+
+Define common sequences at the suite level and invoke by name:
+
+```yaml
+reusable_steps:
+  log in as admin:
+    - enter "{{admin_email}}" into "Email"
+    - enter "{{admin_password}}" into "Password"
+    - click "Sign In"
+
+tests:
+  - name: "Admin can access settings"
+    steps:
+      - log in as admin
+      - click "Settings"
+      - check that page contains "Account Settings"
+```
+
+## Cached plans
+
+The first run of a test uses LLM calls to discover the right browser actions (**discovery run**). After a successful run, GreenLight caches the concrete action sequence as a **heuristic plan** in `.greenlight/plans/`.
+
+Subsequent runs replay the cached plan directly via Playwright — no LLM calls, no API costs, significantly faster. If you change the test steps in YAML, the hash changes and GreenLight automatically re-discovers.
+
+```bash
+greenlight run                    # uses cached plans where available
+greenlight run --discover         # force fresh discovery, ignore cache
+greenlight run --plan-status      # show which tests have cached plans
+```
+
+## LLM setup
+
+### API key
+
+Set your API key via the `LLM_API_KEY` environment variable or a `.env` file in the project root:
+
+```bash
+LLM_API_KEY=sk-...
+```
+
+The same key is used regardless of provider. `OPENROUTER_API_KEY` is also accepted for backward compatibility.
+
+### Providers
+
+GreenLight supports four LLM providers with native API integrations:
+
+| Provider | Value | API |
+|----------|-------|-----|
+| [OpenRouter](https://openrouter.ai) | `openrouter` (default) | Access all models through a single API |
+| [OpenAI](https://platform.openai.com) | `openai` | GPT-4o, GPT-4o-mini, etc. |
+| [Google Gemini](https://ai.google.dev) | `gemini` | Gemini 2.5 Flash, Pro, etc. |
+| [Anthropic Claude](https://console.anthropic.com) | `claude` | Claude Sonnet, Haiku, etc. |
+
+Set the provider in `greenlight.yaml` or via CLI:
+
+```yaml
+provider: openai
+```
+
+```bash
+greenlight run --provider gemini
+```
+
+Only one provider is active at a time. OpenRouter is the default — it lets you access models from all vendors through a single API key, which is the easiest way to get started.
+
+### Model selection
+
+GreenLight uses the LLM in two distinct roles with different requirements:
+
+- **Planner** — interprets the test steps, splits compound actions, and expands form-filling steps. This runs once per test case and benefits from a more capable model for consistent, correct results.
+- **Pilot** — resolves individual steps against the live page (picking which element to click/type). This runs many times per test and should use a fast, inexpensive model to keep costs and execution time low.
+
+Configure both in `greenlight.yaml`:
+
+```yaml
+provider: openrouter
+model:
+  planner: anthropic/claude-sonnet-4      # smarter model, runs once per test
+  pilot: openai/gpt-4o-mini              # fast model, runs per step
+```
+
+Or use a single model for both roles:
+
+```yaml
+model: anthropic/claude-sonnet-4
+```
+
+The `--model` CLI flag sets both roles to the same model (useful for quick overrides):
+
+```bash
+greenlight run --model openai/gpt-4o
+```
+
+Model names must match the provider's naming convention:
+
+| Provider | Example model names |
+|----------|-------------------|
+| OpenRouter | `anthropic/claude-sonnet-4`, `openai/gpt-4o-mini`, `google/gemini-2.5-flash` |
+| OpenAI | `gpt-4o`, `gpt-4o-mini` |
+| Gemini | `gemini-2.5-flash`, `gemini-2.5-pro` |
+| Claude | `claude-sonnet-4-20250514`, `claude-haiku-4-20250514` |
+
+### Custom LLM endpoint
+
+Each provider has a default API endpoint. You can override it with `llm_base_url` for proxies, self-hosted models, or compatible APIs:
+
+```yaml
+# Local Ollama (OpenAI-compatible)
+provider: openai
+llm_base_url: http://localhost:11434/v1
+model: llama3
+```
+
+```bash
+greenlight run --provider openai --llm-base-url http://localhost:11434/v1 --model llama3
+```
+
+## Tech stack
+
+| Layer | Technology |
+|-------|-----------|
+| Browser automation | Playwright (Chromium) |
+| Page representation | Accessibility tree with stable element refs |
+| AI | OpenRouter (any OpenAI-compatible provider) |
+| Plan caching | SHA-256 hash-based invalidation, `.greenlight/plans/` |
+| Test definitions | YAML |
+| Language | TypeScript (Node.js, ESM) |
+
+## Architecture
+
+```mermaid
+flowchart TD
+    subgraph cli[greenlight CLI]
+        yaml[YAML Parser]
+        output[CLI Output<br/>step results, pass/fail]
+
+        subgraph runner[Test Orchestrator]
+            orchestrator[Run Loop<br/>suite loading, browser lifecycle,<br/>plan cache decisions]
+            planner[Plan Cache<br/>.greenlight/plans/]
+
+            subgraph pilot[The Pilot — discovery run]
+                state[Page State Capture<br/>a11y snapshot + stable refs]
+                llm[LLM Client<br/>plan, resolve, expand]
+                executor[Action Executor<br/>click, type, select, autocomplete,<br/>check, remember, compare, ...]
+            end
+
+            replay[Plan Runner<br/>cached replay, no LLM]
+        end
+    end
+
+    subgraph browser[Browser Layer]
+        chromium[(Chromium<br/>Browser Context<br/>staging site)]
+    end
+
+    yaml --> orchestrator
+    orchestrator -->|no cache| pilot
+    orchestrator -->|cached| replay
+    orchestrator -->|save plan| planner
+    planner -->|load plan| orchestrator
+    state --> llm
+    llm --> executor
+    state <--> chromium
+    executor --> chromium
+    replay <--> chromium
+    orchestrator --> output
+```
+
+**Discovery run:** capture page state (a11y tree with stable refs) → LLM determines action → execute via Playwright → record for cache.
+
+**Cached run:** replay stored actions directly via Playwright — no LLM calls, no API costs.
+
+## Documentation
+
+- [Specifications](docs/specifications.md) — full feature spec, technology decisions, MCP strategy
+- [Implementation Plan](docs/implementation.md) — step-by-step build plan
+
+## CI/CD
+
+```yaml
+- name: Run E2E tests
+  run: greenlight run -d staging --reporter json --output results.json
+  env:
+    OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+```
+
+Exit code 0 on all-pass, non-zero on any failure.

package/dist/browser/browser.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Playwright browser lifecycle management.
+ * Wraps launch, context creation, page creation, and cleanup.
+ */
+import { type Browser, type BrowserContext, type Page } from "playwright";
+import type { RunConfig } from "../types.js";
+export interface BrowserOptions {
+    headed: boolean;
+    viewport: {
+        width: number;
+        height: number;
+    };
+}
+/** Launch a Chromium browser instance. */
+export declare function launchBrowser(config: BrowserOptions): Promise<Browser>;
+/** Create an isolated browser context with configured viewport and test mode headers. */
+export declare function createContext(browser: Browser, config: BrowserOptions): Promise<BrowserContext>;
+/** Create a new page within a browser context and inject test mode global. */
+export declare function createPage(context: BrowserContext): Promise<Page>;
+/** Close the browser and all its contexts. */
+export declare function closeBrowser(browser: Browser): Promise<void>;
+/** Extract browser options from RunConfig. */
+export declare function toBrowserOptions(config: RunConfig): BrowserOptions;
+//# sourceMappingURL=browser.d.ts.map

package/dist/browser/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/browser/browser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAE5C,MAAM,WAAW,cAAc;IAC9B,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAA;CAC3C;AAED,0CAA0C;AAC1C,wBAAsB,aAAa,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAI5E;AAED,yFAAyF;AACzF,wBAAsB,aAAa,CAClC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,cAAc,GACpB,OAAO,CAAC,cAAc,CAAC,CAOzB;AAED,8EAA8E;AAC9E,wBAAsB,UAAU,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;AAED,8CAA8C;AAC9C,wBAAsB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAElE;AAED,8CAA8C;AAC9C,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,GAAG,cAAc,CAKlE"}

package/dist/browser/browser.js

@@ -0,0 +1,44 @@
+/**
+ * Playwright browser lifecycle management.
+ * Wraps launch, context creation, page creation, and cleanup.
+ */
+import { chromium } from "playwright";
+/** Launch a Chromium browser instance. */
+export async function launchBrowser(config) {
+    return chromium.launch({
+        headless: !config.headed,
+    });
+}
+/** Create an isolated browser context with configured viewport and test mode headers. */
+export async function createContext(browser, config) {
+    return browser.newContext({
+        viewport: config.viewport,
+        extraHTTPHeaders: {
+            "X-E2E-Test": "true",
+        },
+    });
+}
+/** Create a new page within a browser context and inject test mode global. */
+export async function createPage(context) {
+    const page = await context.newPage();
+    await page.addInitScript(() => {
+        Object.defineProperty(window, "__E2E_TEST__", {
+            value: true,
+            writable: false,
+            configurable: false,
+        });
+    });
+    return page;
+}
+/** Close the browser and all its contexts. */
+export async function closeBrowser(browser) {
+    await browser.close();
+}
+/** Extract browser options from RunConfig. */
+export function toBrowserOptions(config) {
+    return {
+        headed: config.headed,
+        viewport: config.viewport,
+    };
+}
+//# sourceMappingURL=browser.js.map

package/dist/browser/browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/browser/browser.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAgD,MAAM,YAAY,CAAA;AAQnF,0CAA0C;AAC1C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAsB;IACzD,OAAO,QAAQ,CAAC,MAAM,CAAC;QACtB,QAAQ,EAAE,CAAC,MAAM,CAAC,MAAM;KACxB,CAAC,CAAA;AACH,CAAC;AAED,yFAAyF;AACzF,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAAgB,EAChB,MAAsB;IAEtB,OAAO,OAAO,CAAC,UAAU,CAAC;QACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,gBAAgB,EAAE;YACjB,YAAY,EAAE,MAAM;SACpB;KACD,CAAC,CAAA;AACH,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAuB;IACvD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IACpC,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE;QAC7B,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,EAAE;YAC7C,KAAK,EAAE,IAAI;YACX,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;SACnB,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IACF,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,8CAA8C;AAC9C,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAgB;IAClD,MAAM,OAAO,CAAC,KAAK,EAAE,CAAA;AACtB,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,gBAAgB,CAAC,MAAiB;IACjD,OAAO;QACN,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;KACzB,CAAA;AACF,CAAC"}

package/dist/cli/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import "dotenv/config";
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAA"}

package/dist/cli/index.js

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+import "dotenv/config";
+import { Command } from "commander";
+import { DEFAULTS } from "../types.js";
+import { loadProjectConfig } from "../config.js";
+import { createTraceLogger } from "../pilot/trace.js";
+import { resolve } from "node:path";
+import { glob } from "node:fs";
+import { initGlobals } from "../globals.js";
+import { runCommand, showPlanStatus } from "./run.js";
+const program = new Command();
+program
+    .name("greenlight")
+    .description("AI-driven E2E testing tool")
+    .version("0.1.0");
+program
+    .command("run")
+    .description("Run test suites against a staging environment")
+    .argument("[suites...]", "paths to suite YAML files (overrides greenlight.yaml)")
+    .option("-t, --test <name>", "run only the test case matching this name")
+    .option("--base-url <url>", "override the suite base URL")
+    .option("-r, --reporter <format>", "output format: cli, json, or html")
+    .option("-o, --output <path>", "write report to file instead of stdout")
+    .option("--headed", "run browser in visible (headed) mode")
+    .option("-p, --parallel <n>", "number of test cases to run concurrently")
+    .option("--timeout <ms>", "per-step timeout in milliseconds")
+    .option("--model <model>", "LLM model identifier (e.g. anthropic/claude-sonnet-4)")
+    .option("--llm-base-url <url>", "base URL for the LLM API (override)")
+    .option("--provider <name>", "LLM provider: openrouter, openai, gemini, or claude")
+    .option("-d, --deployment <name>", "select a named deployment from greenlight.yaml")
+    .option("--debug", "enable verbose debug output", false)
+    .option("--trace", "log timestamped browser events for performance analysis", false)
+    .option("--discover", "force full discovery run, ignore cached plans", false)
+    .option("--on-drift <mode>", 'behavior on plan drift: "fail" or "rerun" (default: fail)')
+    .option("--plan-status", "show cached plan status for all test cases and exit", false)
+    .action(async (suitesArg, opts) => {
+    // Load project config (greenlight.yaml)
+    const projectConfig = await loadProjectConfig(opts.deployment);
+    // Resolve suite files: CLI args > greenlight.yaml > error
+    let suiteFiles;
+    if (suitesArg.length > 0) {
+        suiteFiles = suitesArg.map((s) => resolve(s));
+    }
+    else if (projectConfig?.suites) {
+        suiteFiles = projectConfig.suites.map((s) => resolve(s));
+    }
+    else {
+        console.error("No suite files specified. Provide them as arguments or in greenlight.yaml.");
+        process.exit(1);
+    }
+    // Build config: CLI flags > greenlight.yaml > built-in defaults
+    const config = {
+        suiteFiles,
+        testFilter: opts.test,
+        baseUrl: opts.baseUrl ?? projectConfig?.base_url,
+        reporter: parseReporter(opts.reporter ?? projectConfig?.reporter ?? DEFAULTS.reporter),
+        outputPath: opts.output ? resolve(opts.output) : undefined,
+        headed: opts.headed ?? projectConfig?.headed ?? DEFAULTS.headed,
+        parallel: opts.parallel
+            ? parseInt(opts.parallel, 10)
+            : (projectConfig?.parallel ?? DEFAULTS.parallel),
+        timeout: opts.timeout
+            ? parseInt(opts.timeout, 10)
+            : (projectConfig?.timeout ?? DEFAULTS.timeout),
+        viewport: projectConfig?.viewport
+            ? { ...projectConfig.viewport }
+            : { ...DEFAULTS.viewport },
+        model: opts.model ?? projectConfig?.model ?? DEFAULTS.model,
+        provider: parseProvider(opts.provider ??
+            projectConfig?.provider ??
+            DEFAULTS.provider),
+        llmBaseUrl: opts.llmBaseUrl ?? projectConfig?.llm_base_url,
+        discover: opts.discover,
+        onDrift: parseOnDrift(opts.onDrift ?? DEFAULTS.onDrift),
+    };
+    // Initialize global runtime state (debug, trace)
+    initGlobals({
+        debug: opts.debug,
+        trace: createTraceLogger(opts.trace),
+    });
+    // Resolve glob patterns in suite file paths
+    const resolvedFiles = await resolveGlobs(config.suiteFiles);
+    if (resolvedFiles.length === 0) {
+        console.error("No suite files found matching:", config.suiteFiles);
+        process.exit(1);
+    }
+    // --plan-status: show cache status and exit
+    if (opts.planStatus) {
+        await showPlanStatus(resolvedFiles, process.cwd(), config);
+        return;
+    }
+    await runCommand(config, resolvedFiles);
+});
+function parseProvider(value) {
+    if (value === "openrouter" ||
+        value === "openai" ||
+        value === "gemini" ||
+        value === "claude") {
+        return value;
+    }
+    console.error(`Invalid provider "${value}". Must be openrouter, openai, gemini, or claude.`);
+    process.exit(1);
+}
+function parseReporter(value) {
+    if (value === "cli" || value === "json" || value === "html") {
+        return value;
+    }
+    console.error(`Invalid reporter "${value}". Must be cli, json, or html.`);
+    process.exit(1);
+}
+function parseOnDrift(value) {
+    if (value === "fail" || value === "rerun") {
+        return value;
+    }
+    console.error(`Invalid --on-drift value "${value}". Must be "fail" or "rerun".`);
+    process.exit(1);
+}
+/** Expand glob patterns into concrete file paths. */
+async function resolveGlobs(patterns) {
+    const files = [];
+    for (const pattern of patterns) {
+        // If pattern has no glob chars, treat as literal path
+        if (!pattern.includes("*")) {
+            files.push(pattern);
+            continue;
+        }
+        const matches = await new Promise((res, rej) => {
+            glob(pattern, (err, result) => {
+                if (err)
+                    rej(err);
+                else
+                    res(result);
+            });
+        });
+        files.push(...matches);
+    }
+    return files;
+}
+program.parse();
+//# sourceMappingURL=index.js.map

package/dist/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAA;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,QAAQ,EAAiC,MAAM,aAAa,CAAA;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAA;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAC3C,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAErD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;AAE7B,OAAO;KACL,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,4BAA4B,CAAC;KACzC,OAAO,CAAC,OAAO,CAAC,CAAA;AAElB,OAAO;KACL,OAAO,CAAC,KAAK,CAAC;KACd,WAAW,CAAC,+CAA+C,CAAC;KAC5D,QAAQ,CACR,aAAa,EACb,uDAAuD,CACvD;KACA,MAAM,CAAC,mBAAmB,EAAE,2CAA2C,CAAC;KACxE,MAAM,CAAC,kBAAkB,EAAE,6BAA6B,CAAC;KACzD,MAAM,CAAC,yBAAyB,EAAE,mCAAmC,CAAC;KACtE,MAAM,CAAC,qBAAqB,EAAE,wCAAwC,CAAC;KACvE,MAAM,CAAC,UAAU,EAAE,sCAAsC,CAAC;KAC1D,MAAM,CAAC,oBAAoB,EAAE,0CAA0C,CAAC;KACxE,MAAM,CAAC,gBAAgB,EAAE,kCAAkC,CAAC;KAC5D,MAAM,CACN,iBAAiB,EACjB,uDAAuD,CACvD;KACA,MAAM,CAAC,sBAAsB,EAAE,qCAAqC,CAAC;KACrE,MAAM,CACN,mBAAmB,EACnB,qDAAqD,CACrD;KACA,MAAM,CACN,yBAAyB,EACzB,gDAAgD,CAChD;KACA,MAAM,CAAC,SAAS,EAAE,6BAA6B,EAAE,KAAK,CAAC;KACvD,MAAM,CACN,SAAS,EACT,yDAAyD,EACzD,KAAK,CACL;KACA,MAAM,CAAC,YAAY,EAAE,+CAA+C,EAAE,KAAK,CAAC;KAC5E,MAAM,CACN,mBAAmB,EACnB,2DAA2D,CAC3D;KACA,MAAM,CACN,eAAe,EACf,qDAAqD,EACrD,KAAK,CACL;KACA,MAAM,CACN,KAAK,EACJ,SAAmB,EACnB,IAiBC,EACA,EAAE;IACH,wCAAwC;IACxC,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE9D,0DAA0D;IAC1D,IAAI,UAAoB,CAAA;IACxB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAC9C,CAAC;SAAM,IAAI,aAAa,EAAE,MAAM,EAAE,CAAC;QAClC,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IACzD,CAAC;SAAM,CAAC;QACP,OAAO,CAAC,KAAK,CACZ,4EAA4E,CAC5E,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,gEAAgE;IAChE,MAAM,MAAM,GAAc;QACzB,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,IAAI;QACrB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,aAAa,EAAE,QAAQ;QAChD,QAAQ,EAAE,aAAa,CACtB,IAAI,CAAC,QAAQ,IAAI,aAAa,EAAE,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAC7D;QACD,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;QAC1D,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,aAAa,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM;QAC/D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACtB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;YAC7B,CAAC,CAAC,CAAC,aAAa,EAAE,QAAQ,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACjD,OAAO,EAAE,IAAI,CAAC,OAAO;YACpB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5B,CAAC,CAAC,CAAC,aAAa,EAAE,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;QAC/C,QAAQ,EAAE,aAAa,EAAE,QAAQ;YAChC,CAAC,CAAC,EAAE,GAAG,aAAa,CAAC,QAAQ,EAAE;YAC/B,CAAC,CAAC,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE;QAC3B,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,aAAa,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;QAC3D,QAAQ,EAAE,aAAa,CACtB,IAAI,CAAC,QAAQ;YACZ,aAAa,EAAE,QAAQ;YACvB,QAAQ,CAAC,QAAQ,CAClB;QACD,UAAU,EACT,IAAI,CAAC,UAAU,IAAI,aAAa,EAAE,YAAY;QAC/C,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;KACvD,CAAA;IAED,iDAAiD;IACjD,WAAW,CAAC;QACX,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,KAAK,EAAE,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC;KACpC,CAAC,CAAA;IAEF,4CAA4C;IAC5C,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;IAE3D,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,4CAA4C;IAC5C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACrB,MAAM,cAAc,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;QAC1D,OAAM;IACP,CAAC;IAED,MAAM,UAAU,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;AACxC,CAAC,CACD,CAAA;AAEF,SAAS,aAAa,CAAC,KAAa;IACnC,IACC,KAAK,KAAK,YAAY;QACtB,KAAK,KAAK,QAAQ;QAClB,KAAK,KAAK,QAAQ;QAClB,KAAK,KAAK,QAAQ,EACjB,CAAC;QACF,OAAO,KAAK,CAAA;IACb,CAAC;IACD,OAAO,CAAC,KAAK,CACZ,qBAAqB,KAAK,mDAAmD,CAC7E,CAAA;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IACnC,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QAC7D,OAAO,KAAK,CAAA;IACb,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,qBAAqB,KAAK,gCAAgC,CAAC,CAAA;IACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IAClC,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;QAC3C,OAAO,KAAK,CAAA;IACb,CAAC;IACD,OAAO,CAAC,KAAK,CACZ,6BAA6B,KAAK,+BAA+B,CACjE,CAAA;IACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,qDAAqD;AACrD,KAAK,UAAU,YAAY,CAAC,QAAkB;IAC7C,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,sDAAsD;QACtD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACnB,SAAQ;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,IAAI,OAAO,CAAW,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACxD,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBAC7B,IAAI,GAAG;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAA;;oBACZ,GAAG,CAAC,MAAM,CAAC,CAAA;YACjB,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAA;IACvB,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,OAAO,CAAC,KAAK,EAAE,CAAA"}

package/dist/cli/run.d.ts

@@ -0,0 +1,9 @@
+import type { RunConfig } from "../types.js";
+/** Show plan status for all test cases across all suites. */
+export declare function showPlanStatus(suiteFiles: string[], cwd: string, config: RunConfig): Promise<void>;
+/**
+ * Run all test suites according to the resolved configuration.
+ * This is the main test execution entry point called by the CLI.
+ */
+export declare function runCommand(config: RunConfig, resolvedFiles: string[]): Promise<void>;
+//# sourceMappingURL=run.d.ts.map

package/dist/cli/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/cli/run.ts"],"names":[],"mappings":"AA6BA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AAuC5C,6DAA6D;AAC7D,wBAAsB,cAAc,CACnC,UAAU,EAAE,MAAM,EAAE,EACpB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,SAAS,GACf,OAAO,CAAC,IAAI,CAAC,CAgDf;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAC/B,MAAM,EAAE,SAAS,EACjB,aAAa,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,IAAI,CAAC,CAuQf"}