elasticdash-test 0.1.18 → 0.1.19-alpha-1

package/README.md

@@ -10,6 +10,7 @@ An AI-native test runner for ElasticDash workflow testing. Built for async AI pi
 - [Tool Recording](#tool-recording)
 - [AI Call Recording](#ai-call-recording)
 - [HTTP Workflow Mode](#http-workflow-mode)
+- [CI/CD Runner](#cicd-runner)
 - [Configuration](#configuration)
 
 ### Open Detailed Docs
@@ -32,6 +33,7 @@ An AI-native test runner for ElasticDash workflow testing. Built for async AI pi
 - 📊 **Interactive dashboard** — browse workflows, debug traces, validate fixes visually
 - 🤖 **Agent mid-trace replay** — resume long-running agents from any task without re-execution
 - 🌐 **HTTP workflow mode** — run workflows against your live dev server for framework-heavy apps (Next.js, Remix, etc.) with full AI and tool call observability
+- 🚀 **CI/CD runner** — fetch test groups from your ElasticDash project, execute tests, submit results, and fail the build on regressions
 
 ---
 
@@ -389,6 +391,162 @@ This mode is intended for local development and testing scenarios. For productio
 
 ---
 
+## CI/CD Runner
+
+Run your ElasticDash test groups directly from CI pipelines. The `ci` command fetches active test groups from your project via API key, executes each test locally, submits results back to the backend, and exits with code 1 if any test fails.
+
+### How It Works
+
+```
+┌──────────────┐     GET /testgroups/by-project     ┌──────────────────┐
+│   CI Runner  │ ──────────────────────────────────→ │  ElasticDash API │
+│  (SDK side)  │ ←────────────────────────────────── │   (your backend) │
+│              │   test groups + tests + expectations │                  │
+│              │                                      │                  │
+│  execute     │     POST /testgroups/:id/runs        │                  │
+│  each test   │ ──────────────────────────────────→ │  stores results  │
+│  locally     │                                      │                  │
+│              │     POST /testgroups/batches          │                  │
+│              │ ──────────────────────────────────→ │  groups the runs │
+└──────────────┘                                      └──────────────────┘
+```
+
+1. **Fetch** — Calls `GET /testgroups/by-project` with the API key (scoped to project). Returns all active test groups with their tests and expectations.
+2. **Execute** — For each test, runs it locally using existing SDK infrastructure:
+   - **Single-step tests** — replays a specific tool or AI step with `mock_input` and `frozen_events`
+   - **Full-flow tests** — runs the entire workflow from `ed_workflows.ts` with `workflow_input`
+3. **Evaluate** — Checks all expectations (token-budget, latency-budget, output-contains, output-schema, tool-called, determinism, llm-judge). Respects `run_count` and `pass_threshold`.
+4. **Submit** — POSTs each result to `POST /testgroups/:id/runs` with single run data, expectation results, and git metadata.
+5. **Batch** — Creates a batch grouping all run IDs for dashboard viewing.
+
+### CLI Usage
+
+```bash
+# Basic — uses env vars
+ELASTICDASH_API_URL=https://your-api.com ELASTICDASH_API_KEY=ed_xxx npx elasticdash ci
+
+# Explicit flags
+npx elasticdash ci --server https://your-api.com --api-key ed_xxx
+
+# Filter by workflow or tags
+npx elasticdash ci --server $ELASTICDASH_API_URL --api-key $ELASTICDASH_API_KEY \
+  --workflow checkout --tags payment,critical
+
+# Pass git metadata (auto-detected in GitHub Actions / GitLab CI)
+npx elasticdash ci --server $ELASTICDASH_API_URL --api-key $ELASTICDASH_API_KEY \
+  --git-branch main --git-commit abc123
+```
+
+**All flags:**
+
+| Flag | Env Var | Description |
+|------|---------|-------------|
+| `--server <url>` | `ELASTICDASH_API_URL` | Backend API URL (required) |
+| `--api-key <key>` | `ELASTICDASH_API_KEY` | Project API key (required) |
+| `--workflow <name>` | — | Filter test groups by workflow name |
+| `--tags <t1,t2>` | — | Filter test groups by tags (comma-separated) |
+| `--triggered-by <src>` | — | Trigger source label (default: `ci`) |
+| `--git-branch <branch>` | Auto-detected | Git branch name |
+| `--git-commit <sha>` | Auto-detected | Git commit SHA |
+| `--git-commit-message <msg>` | Auto-detected | Commit message |
+| `--git-pr-number <n>` | Auto-detected | PR number |
+| `--git-pr-url <url>` | Auto-detected | PR URL |
+
+### GitHub Actions Example
+
+```yaml
+name: AI Tests
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - run: npm ci
+
+      - name: Run ElasticDash CI tests
+        run: npx elasticdash ci
+        env:
+          ELASTICDASH_API_URL: ${{ secrets.ELASTICDASH_API_URL }}
+          ELASTICDASH_API_KEY: ${{ secrets.ELASTICDASH_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}  # if tests use OpenAI
+```
+
+Git branch, commit SHA, PR number, and PR URL are auto-detected from GitHub Actions environment variables — no extra flags needed.
+
+### GitLab CI Example
+
+```yaml
+ai-tests:
+  stage: test
+  image: node:20
+  script:
+    - npm ci
+    - npx elasticdash ci
+  variables:
+    ELASTICDASH_API_URL: $ELASTICDASH_API_URL
+    ELASTICDASH_API_KEY: $ELASTICDASH_API_KEY
+```
+
+### Programmatic Usage
+
+```ts
+import { runCI } from 'elasticdash-test'
+
+const summary = await runCI({
+  serverUrl: 'https://your-api.com',
+  apiKey: 'ed_xxx',
+  workflowName: 'checkout',       // optional filter
+  tags: ['payment', 'critical'],  // optional filter
+})
+
+console.log(`${summary.passed}/${summary.total} passed`)
+process.exit(summary.failed > 0 ? 1 : 0)
+```
+
+### Output
+
+```
+[elasticdash ci] Fetching test groups...
+[elasticdash ci] Found 2 test group(s), 5 test(s) total.
+
+  Checkout Flow (3 tests)
+    validate-input ... PASS (234ms)
+    charge-card ... PASS (1823ms)
+    send-confirmation ... FAIL (945ms)
+      [output-contains] Output text check failed.
+
+  Refund Flow (2 tests)
+    check-eligibility ... PASS (412ms)
+    process-refund ... PASS (1567ms)
+
+──────────────────────────────────────────────────
+Summary
+──────────────────────────────────────────────────
+  Total:    5
+  Passed:   4
+  Failed:   1
+  Duration: 5.0s
+  Batch ID: 42
+──────────────────────────────────────────────────
+
+[elasticdash ci] 1 test(s) failed.
+```
+
+### Prerequisites
+
+- An ElasticDash project with an API key (create one in the dashboard under Settings → API Keys)
+- Active test groups with tests and expectations configured in the dashboard
+- `ed_tools.ts` and/or `ed_workflows.ts` in your project root (for the executor to discover tools and workflows)
+- AI provider API keys in the environment if tests use LLM calls (e.g., `OPENAI_API_KEY`)
+
+---
+
 ## Configuration
 
 Optional `elasticdash.config.ts` at project root:
@@ -461,6 +619,15 @@ ELASTICDASH_RUN_ID=<run-id-from-dashboard>    # optional, enables step freezing
 
 `wrapTool` and `wrapAI` will auto-connect on their first call. See [Dashboard Auto-Detection](#dashboard-auto-detection-env-var-mode) for details.
 
+**CI runner (execute test groups from your project):**
+
+```ts
+import { runCI } from 'elasticdash-test'
+
+const summary = await runCI({ serverUrl: 'https://your-api.com', apiKey: 'ed_xxx' })
+// summary.total, summary.passed, summary.failed, summary.batchId, summary.results
+```
+
 ---
 
 ## License

package/dist/ci/api-client.d.ts

@@ -0,0 +1,23 @@
+import type { APITestGroup } from './types.js';
+/**
+ * Fetch all active test groups (with nested tests & expectations) for the project
+ * scoped to the API key.
+ */
+export declare function fetchTestGroups(serverUrl: string, apiKey: string, filters?: {
+    workflowName?: string;
+    tags?: string[];
+    status?: string;
+}): Promise<APITestGroup[]>;
+/**
+ * Submit a test run result to the backend.
+ */
+export declare function submitTestRun(serverUrl: string, apiKey: string, testGroupId: number, payload: Record<string, unknown>): Promise<{
+    id: number;
+}>;
+/**
+ * Create a batch grouping multiple test run IDs.
+ */
+export declare function createBatch(serverUrl: string, apiKey: string, payload: Record<string, unknown>): Promise<{
+    id: number;
+}>;
+//# sourceMappingURL=api-client.d.ts.map

package/dist/ci/api-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-client.d.ts","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAgC9C;;;GAGG;AACH,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC,YAAY,EAAE,CAAC,CAUzB;AAED;;GAEG;AACH,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAOzB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAOzB"}

package/dist/ci/api-client.js

@@ -0,0 +1,62 @@
+import { getOriginalFetch } from '../interceptors/http.js';
+// ─── API Client ──────────────────────────────────────────────
+// Uses getOriginalFetch() to bypass SDK interceptors.
+function headers(apiKey) {
+    return {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`,
+    };
+}
+async function apiRequest(url, apiKey, options = {}) {
+    const res = await getOriginalFetch()(url, {
+        ...options,
+        headers: { ...headers(apiKey), ...(options.headers ?? {}) },
+    });
+    if (!res.ok) {
+        const text = await res.text().catch(() => '');
+        throw new Error(`API ${res.status}: ${text || res.statusText}`);
+    }
+    const json = await res.json();
+    // Backend wraps responses in generalApiResponseSender which sends { result: ... }
+    return (json.result ?? json.data ?? json);
+}
+/**
+ * Fetch all active test groups (with nested tests & expectations) for the project
+ * scoped to the API key.
+ */
+export async function fetchTestGroups(serverUrl, apiKey, filters) {
+    const base = serverUrl.replace(/\/$/, '');
+    const params = new URLSearchParams();
+    if (filters?.workflowName)
+        params.set('workflowName', filters.workflowName);
+    if (filters?.tags?.length)
+        params.set('tags', filters.tags.join(','));
+    if (filters?.status)
+        params.set('status', filters.status);
+    const qs = params.toString();
+    const url = `${base}/api/testgroups/by-project${qs ? `?${qs}` : ''}`;
+    return apiRequest(url, apiKey);
+}
+/**
+ * Submit a test run result to the backend.
+ */
+export async function submitTestRun(serverUrl, apiKey, testGroupId, payload) {
+    const base = serverUrl.replace(/\/$/, '');
+    const url = `${base}/api/testgroups/${testGroupId}/runs`;
+    return apiRequest(url, apiKey, {
+        method: 'POST',
+        body: JSON.stringify(payload),
+    });
+}
+/**
+ * Create a batch grouping multiple test run IDs.
+ */
+export async function createBatch(serverUrl, apiKey, payload) {
+    const base = serverUrl.replace(/\/$/, '');
+    const url = `${base}/api/testgroups/batches`;
+    return apiRequest(url, apiKey, {
+        method: 'POST',
+        body: JSON.stringify(payload),
+    });
+}
+//# sourceMappingURL=api-client.js.map

package/dist/ci/api-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-client.js","sourceRoot":"","sources":["../../src/ci/api-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAG1D,gEAAgE;AAChE,sDAAsD;AAEtD,SAAS,OAAO,CAAC,MAAc;IAC7B,OAAO;QACL,cAAc,EAAE,kBAAkB;QAClC,eAAe,EAAE,UAAU,MAAM,EAAE;KACpC,CAAA;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,GAAW,EACX,MAAc,EACd,UAAuB,EAAE;IAEzB,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE,CAAC,GAAG,EAAE;QACxC,GAAG,OAAO;QACV,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,OAAiC,IAAI,EAAE,CAAC,EAAE;KACtF,CAAC,CAAA;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAA;QAC7C,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;IACjE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA8B,CAAA;IACzD,kFAAkF;IAClF,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAM,CAAA;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,SAAiB,EACjB,MAAc,EACd,OAAqE;IAErE,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IACzC,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAA;IACpC,IAAI,OAAO,EAAE,YAAY;QAAE,MAAM,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,CAAC,YAAY,CAAC,CAAA;IAC3E,IAAI,OAAO,EAAE,IAAI,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrE,IAAI,OAAO,EAAE,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAEzD,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;IAC5B,MAAM,GAAG,GAAG,GAAG,IAAI,6BAA6B,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;IACpE,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,CAAC,CAAA;AAChD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,MAAc,EACd,WAAmB,EACnB,OAAgC;IAEhC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IACzC,MAAM,GAAG,GAAG,GAAG,IAAI,mBAAmB,WAAW,OAAO,CAAA;IACxD,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAiB,EACjB,MAAc,EACd,OAAgC;IAEhC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;IACzC,MAAM,GAAG,GAAG,GAAG,IAAI,yBAAyB,CAAA;IAC5C,OAAO,UAAU,CAAiB,GAAG,EAAE,MAAM,EAAE;QAC7C,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAA;AACJ,CAAC"}

package/dist/ci/executor.d.ts

@@ -0,0 +1,13 @@
+import type { APITestGroupTest, CISingleRunResult, CIExpectationResult } from './types.js';
+interface ExecutionResult {
+    passed: boolean;
+    singleRuns: CISingleRunResult[];
+    expectationResults: CIExpectationResult[];
+    durationMs: number;
+}
+/**
+ * Execute a test (single-step or full-flow) according to its configuration.
+ */
+export declare function executeTest(test: APITestGroupTest, cwd: string): Promise<ExecutionResult>;
+export {};
+//# sourceMappingURL=executor.d.ts.map

package/dist/ci/executor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../src/ci/executor.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,gBAAgB,EAEhB,iBAAiB,EACjB,mBAAmB,EACpB,MAAM,YAAY,CAAA;AAMnB,UAAU,eAAe;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,UAAU,EAAE,iBAAiB,EAAE,CAAA;IAC/B,kBAAkB,EAAE,mBAAmB,EAAE,CAAA;IACzC,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;GAEG;AACH,wBAAsB,WAAW,CAC/B,IAAI,EAAE,gBAAgB,EACtB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,eAAe,CAAC,CA+C1B"}