elasticdash-test 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 ElasticDash
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,215 @@
+# ElasticDash Test
+
+An AI-native test runner for ElasticDash workflow testing. Built for async AI pipelines — not a general-purpose test runner.
+
+- Trace-first: every test receives a `trace` context to record and assert on LLM calls and tool invocations
+- AI-specific matchers: `toHaveLLMStep`, `toCallTool`, `toMatchSemanticOutput`
+- Sequential execution, no parallelism overhead
+- No Jest dependency
+
+---
+
+## Installation
+
+```bash
+npm install elasticdash-test
+```
+
+Requires Node 20+.
+
+---
+
+## Quick Start
+
+**1. Write a test file** (`my-flow.ai.test.ts`):
+
+```ts
+import '../node_modules/elasticdash-test/dist/test-setup.js'
+import { expect } from 'expect'
+
+aiTest('checkout flow', async (ctx) => {
+  await runCheckout(ctx)
+
+  expect(ctx.trace).toHaveLLMStep({ model: 'gpt-4', contains: 'order confirmed' })
+  expect(ctx.trace).toCallTool('chargeCard')
+})
+```
+
+**2. Run it:**
+
+```bash
+elasticdash test              # discover all *.ai.test.ts files
+elasticdash test ./ai-tests   # discover in a specific directory
+elasticdash run my-flow.ai.test.ts  # run a single file
+```
+
+**3. Read the output:**
+
+```
+  ✓ checkout flow (1.2s)
+  ✗ refund flow (0.8s)
+    → Expected tool "chargeCard" to be called, but no tool calls were recorded
+
+2 passed
+1 failed
+Total: 3
+Duration: 3.4s
+```
+
+---
+
+## Writing Tests
+
+### Globals
+
+After importing `test-setup`, these are available globally — no imports needed:
+
+| Global | Description |
+|---|---|
+| `aiTest(name, fn)` | Register a test |
+| `beforeAll(fn)` | Run once before all tests in the file |
+| `afterAll(fn)` | Run once after all tests in the file |
+
+### Test context
+
+Each test function receives a `ctx: AITestContext` argument:
+
+```ts
+aiTest('my test', async (ctx) => {
+  // ctx.trace — record and inspect LLM steps and tool calls
+})
+```
+
+### Recording trace data
+
+Your AI workflow code should call these on `ctx.trace` to make assertions possible:
+
+```ts
+ctx.trace.recordLLMStep({
+  model: 'gpt-4',
+  prompt: 'What is the order status?',
+  completion: 'The order has been confirmed.',
+})
+
+ctx.trace.recordToolCall({
+  name: 'chargeCard',
+  args: { amount: 99.99 },
+})
+```
+
+### Matchers
+
+#### `toHaveLLMStep(config?)`
+
+Assert the trace contains at least one LLM step matching the given config. All fields are optional.
+
+```ts
+expect(ctx.trace).toHaveLLMStep({ model: 'gpt-4' })
+expect(ctx.trace).toHaveLLMStep({ contains: 'order confirmed' })
+expect(ctx.trace).toHaveLLMStep({ model: 'gpt-4', contains: 'order confirmed' })
+```
+
+#### `toCallTool(toolName)`
+
+Assert the trace contains a tool call with the given name.
+
+```ts
+expect(ctx.trace).toCallTool('chargeCard')
+```
+
+#### `toMatchSemanticOutput(expected)`
+
+Assert the combined LLM output contains the expected string (case-insensitive substring match — designed for a future embedding-based similarity upgrade).
+
+```ts
+expect(ctx.trace).toMatchSemanticOutput('order confirmed')
+```
+
+---
+
+## Configuration
+
+Create an optional `elasticdash.config.ts` at the project root:
+
+```ts
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  traceMode: 'local' as const,
+}
+```
+
+| Option | Default | Description |
+|---|---|---|
+| `testMatch` | `['**/*.ai.test.ts']` | Glob patterns for test discovery |
+| `traceMode` | `'local'` | `'local'` (stub) or `'remote'` (future ElasticDash backend) |
+
+---
+
+## TypeScript Setup
+
+Add an `examples/tsconfig.json` (or your test directory's `tsconfig.json`) that extends the root config and includes the `src` types:
+
+```json
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "..",
+    "noEmit": true
+  },
+  "include": [
+    "../src/**/*",
+    "./**/*"
+  ]
+}
+```
+
+This gives you typed `aiTest`, `beforeAll`, `afterAll` globals and typed custom matchers in your test files.
+
+---
+
+## Project Structure
+
+```
+src/
+  cli.ts                 CLI entry point (commander + fast-glob)
+  runner.ts              Sequential test runner engine
+  reporter.ts            Color-coded terminal output
+  test-setup.ts          Import in test files for globals + matcher types
+  index.ts               Programmatic API
+  core/
+    registry.ts          aiTest / beforeAll / afterAll registry
+  trace-adapter/
+    context.ts           TraceHandle, AITestContext, RunnerHooks scaffold
+  matchers/
+    index.ts             Custom expect matchers
+```
+
+---
+
+## Programmatic API
+
+```ts
+import { runFiles, reportResults, registerMatchers } from 'elasticdash-test'
+
+registerMatchers()
+const results = await runFiles(['./tests/flow.ai.test.ts'])
+reportResults(results)
+```
+
+---
+
+## Non-Goals
+
+This runner intentionally does not support:
+
+- Parallel execution
+- Watch mode
+- Snapshot testing
+- Coverage reporting
+- Jest compatibility
+
+---
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import fg from 'fast-glob';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { existsSync } from 'node:fs';
+import { registerMatchers } from './matchers/index.js';
+import { runFiles } from './runner.js';
+import { reportResults } from './reporter.js';
+async function loadConfig(cwd) {
+    const configPath = path.join(cwd, 'elasticdash.config.ts');
+    const configPathJs = path.join(cwd, 'elasticdash.config.js');
+    for (const p of [configPath, configPathJs]) {
+        if (existsSync(p)) {
+            const mod = await import(pathToFileURL(p).href);
+            return (mod.default ?? {});
+        }
+    }
+    return {};
+}
+// --- File discovery ---
+async function discoverTestFiles(patterns, cwd) {
+    const files = await fg(patterns, { cwd, absolute: true });
+    return files.sort();
+}
+// --- Bootstrap ---
+async function bootstrap() {
+    registerMatchers();
+    const cwd = process.cwd();
+    const config = await loadConfig(cwd);
+    const defaultPattern = config.testMatch ?? ['**/*.ai.test.ts', '**/*.ai.test.js'];
+    const program = new Command();
+    program
+        .name('elasticdash')
+        .description('AI-native test runner for ElasticDash workflow testing')
+        .version('0.1.0');
+    // elasticdash test [dir]
+    program
+        .command('test [dir]')
+        .description('Discover and run all AI test files')
+        .action(async (dir) => {
+        const searchBase = dir ? path.resolve(cwd, dir) : cwd;
+        const files = await discoverTestFiles(defaultPattern, searchBase);
+        if (files.length === 0) {
+            console.error(`No test files found matching: ${defaultPattern.join(', ')}`);
+            process.exit(1);
+        }
+        const results = await runFiles(files);
+        reportResults(results);
+        const anyFailed = results.some((fr) => fr.results.some((r) => !r.passed));
+        process.exit(anyFailed ? 1 : 0);
+    });
+    // elasticdash run <file>
+    program
+        .command('run <file>')
+        .description('Run a single AI test file')
+        .action(async (file) => {
+        const absFile = pathToFileURL(path.resolve(cwd, file)).href;
+        const results = await runFiles([absFile]);
+        reportResults(results);
+        const anyFailed = results.some((fr) => fr.results.some((r) => !r.passed));
+        process.exit(anyFailed ? 1 : 0);
+    });
+    await program.parseAsync(process.argv);
+}
+bootstrap().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,MAAM,WAAW,CAAA;AAC1B,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAEpC,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAQ7C,KAAK,UAAU,UAAU,CAAC,GAAW;IACnC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAA;IAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAA;IAE5D,KAAK,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,YAAY,CAAC,EAAE,CAAC;QAC3C,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC/C,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAsB,CAAA;QACjD,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAA;AACX,CAAC;AAED,yBAAyB;AACzB,KAAK,UAAU,iBAAiB,CAAC,QAAkB,EAAE,GAAW;IAC9D,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;IACzD,OAAO,KAAK,CAAC,IAAI,EAAE,CAAA;AACrB,CAAC;AAED,oBAAoB;AACpB,KAAK,UAAU,SAAS;IACtB,gBAAgB,EAAE,CAAA;IAElB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;IACzB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,iBAAiB,EAAE,iBAAiB,CAAC,CAAA;IAEjF,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAA;IAE7B,OAAO;SACJ,IAAI,CAAC,aAAa,CAAC;SACnB,WAAW,CAAC,wDAAwD,CAAC;SACrE,OAAO,CAAC,OAAO,CAAC,CAAA;IAEnB,yBAAyB;IACzB,OAAO;SACJ,OAAO,CAAC,YAAY,CAAC;SACrB,WAAW,CAAC,oCAAoC,CAAC;SACjD,MAAM,CAAC,KAAK,EAAE,GAAY,EAAE,EAAE;QAC7B,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QACrD,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,cAAc,EAAE,UAAU,CAAC,CAAA;QAEjE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,KAAK,CAAC,iCAAiC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;QACrC,aAAa,CAAC,OAAO,CAAC,CAAA;QAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QACzE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;IAEJ,yBAAyB;IACzB,OAAO;SACJ,OAAO,CAAC,YAAY,CAAC;SACrB,WAAW,CAAC,2BAA2B,CAAC;SACxC,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE;QAC7B,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;QAE3D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;QACzC,aAAa,CAAC,OAAO,CAAC,CAAA;QAEtB,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QACzE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;IAEJ,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;AACxC,CAAC;AAED,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACxB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}

package/dist/core/registry.d.ts

@@ -0,0 +1,22 @@
+import type { AITestContext } from '../trace-adapter/context.js';
+export type TestFunction = (ctx: AITestContext) => Promise<void> | void;
+export interface TestEntry {
+    name: string;
+    fn: TestFunction;
+}
+export interface Registry {
+    tests: TestEntry[];
+    beforeAllHooks: Array<() => Promise<void> | void>;
+    afterAllHooks: Array<() => Promise<void> | void>;
+}
+export declare function clearRegistry(): void;
+export declare function getRegistry(): Registry;
+export declare function aiTest(name: string, fn: TestFunction): void;
+export declare function beforeAll(fn: () => Promise<void> | void): void;
+export declare function afterAll(fn: () => Promise<void> | void): void;
+declare global {
+    var aiTest: (name: string, fn: TestFunction) => void;
+    var beforeAll: (fn: () => Promise<void> | void) => void;
+    var afterAll: (fn: () => Promise<void> | void) => void;
+}
+//# sourceMappingURL=registry.d.ts.map

package/dist/core/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAEhE,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;AAEvE,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,YAAY,CAAA;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,SAAS,EAAE,CAAA;IAClB,cAAc,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;IACjD,aAAa,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;CACjD;AAYD,wBAAgB,aAAa,IAAI,IAAI,CAEpC;AAED,wBAAgB,WAAW,IAAI,QAAQ,CAEtC;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,GAAG,IAAI,CAE3D;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAE9D;AAED,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI,CAE7D;AAGD,OAAO,CAAC,MAAM,CAAC;IAEb,IAAI,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,YAAY,KAAK,IAAI,CAAA;IAEpD,IAAI,SAAS,EAAE,CAAC,EAAE,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAA;IAEvD,IAAI,QAAQ,EAAE,CAAC,EAAE,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAA;CACvD"}

package/dist/core/registry.js

@@ -0,0 +1,27 @@
+let _registry = createEmptyRegistry();
+function createEmptyRegistry() {
+    return {
+        tests: [],
+        beforeAllHooks: [],
+        afterAllHooks: [],
+    };
+}
+export function clearRegistry() {
+    _registry = createEmptyRegistry();
+}
+export function getRegistry() {
+    return _registry;
+}
+export function aiTest(name, fn) {
+    _registry.tests.push({ name, fn });
+}
+export function beforeAll(fn) {
+    _registry.beforeAllHooks.push(fn);
+}
+export function afterAll(fn) {
+    _registry.afterAllHooks.push(fn);
+}
+globalThis.aiTest = aiTest;
+globalThis.beforeAll = beforeAll;
+globalThis.afterAll = afterAll;
+//# sourceMappingURL=registry.js.map

package/dist/core/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AAeA,IAAI,SAAS,GAAa,mBAAmB,EAAE,CAAA;AAE/C,SAAS,mBAAmB;IAC1B,OAAO;QACL,KAAK,EAAE,EAAE;QACT,cAAc,EAAE,EAAE;QAClB,aAAa,EAAE,EAAE;KAClB,CAAA;AACH,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,SAAS,GAAG,mBAAmB,EAAE,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,IAAY,EAAE,EAAgB;IACnD,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;AACpC,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,EAA8B;IACtD,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AACnC,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,EAA8B;IACrD,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;AAClC,CAAC;AAYD,UAAU,CAAC,MAAM,GAAG,MAAM,CAAA;AAC1B,UAAU,CAAC,SAAS,GAAG,SAAS,CAAA;AAChC,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA"}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+export { aiTest, beforeAll, afterAll, getRegistry, clearRegistry } from './core/registry.js';
+export { createTraceHandle, startTraceSession } from './trace-adapter/context.js';
+export { registerMatchers } from './matchers/index.js';
+export { runFiles } from './runner.js';
+export { reportResults } from './reporter.js';
+export type { TestResult, FileResult, RunnerOptions } from './runner.js';
+export type { AITestContext, TraceHandle, LLMStep, ToolCall, TraceStep, RunnerHooks } from './trace-adapter/context.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAC5F,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AACxE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA"}

package/dist/index.js

@@ -0,0 +1,7 @@
+// Public API surface for programmatic use
+export { aiTest, beforeAll, afterAll, getRegistry, clearRegistry } from './core/registry.js';
+export { createTraceHandle, startTraceSession } from './trace-adapter/context.js';
+export { registerMatchers } from './matchers/index.js';
+export { runFiles } from './runner.js';
+export { reportResults } from './reporter.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,0CAA0C;AAC1C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAC5F,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AACjF,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA"}

package/dist/matchers/index.d.ts

@@ -0,0 +1,18 @@
+interface LLMStepConfig {
+    model?: string;
+    contains?: string;
+}
+declare module 'expect' {
+    interface Matchers<R> {
+        toHaveLLMStep(config?: LLMStepConfig): R;
+        toCallTool(toolName: string): R;
+        toMatchSemanticOutput(expected: string): R;
+    }
+}
+/**
+ * Register all AI-specific custom matchers onto the `expect` instance.
+ * Call this once on runner startup.
+ */
+export declare function registerMatchers(): void;
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/matchers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAGA,UAAU,aAAa;IACrB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAGD,OAAO,QAAQ,QAAQ,CAAC;IACtB,UAAU,QAAQ,CAAC,CAAC;QAClB,aAAa,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,CAAC,CAAA;QACxC,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAA;QAC/B,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAA;KAC3C;CACF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CAqFvC"}

package/dist/matchers/index.js

@@ -0,0 +1,85 @@
+import { expect } from 'expect';
+/**
+ * Register all AI-specific custom matchers onto the `expect` instance.
+ * Call this once on runner startup.
+ */
+export function registerMatchers() {
+    expect.extend({
+        /**
+         * Assert the trace contains at least one LLM step matching the config.
+         * Usage: expect(trace).toHaveLLMStep({ model: "gpt-4", contains: "order confirmed" })
+         */
+        toHaveLLMStep(trace, config = {}) {
+            const steps = trace.getLLMSteps();
+            const matching = steps.filter((step) => {
+                if (config.model && step.model !== config.model)
+                    return false;
+                if (config.contains) {
+                    const haystack = [step.completion, step.prompt, step.contains]
+                        .filter(Boolean)
+                        .join(' ')
+                        .toLowerCase();
+                    if (!haystack.includes(config.contains.toLowerCase()))
+                        return false;
+                }
+                return true;
+            });
+            const pass = matching.length > 0;
+            return {
+                pass,
+                message: () => {
+                    if (pass) {
+                        return `Expected trace NOT to have LLM step matching ${JSON.stringify(config)}`;
+                    }
+                    const stepSummary = steps.length === 0
+                        ? 'no LLM steps were recorded'
+                        : `recorded steps: ${JSON.stringify(steps)}`;
+                    return `Expected trace to have LLM step matching ${JSON.stringify(config)}, but ${stepSummary}`;
+                },
+            };
+        },
+        /**
+         * Assert the trace contains a tool call with the given name.
+         * Usage: expect(trace).toCallTool("chargeCard")
+         */
+        toCallTool(trace, toolName) {
+            const calls = trace.getToolCalls();
+            const pass = calls.some((c) => c.name === toolName);
+            return {
+                pass,
+                message: () => {
+                    if (pass) {
+                        return `Expected trace NOT to call tool "${toolName}"`;
+                    }
+                    const names = calls.map((c) => c.name);
+                    const recorded = names.length === 0 ? 'no tool calls were recorded' : `recorded: [${names.join(', ')}]`;
+                    return `Expected tool "${toolName}" to be called, but ${recorded}`;
+                },
+            };
+        },
+        /**
+         * Assert the trace output semantically matches an expected string.
+         * For MP: simple substring/case-insensitive match.
+         * Later: can swap in embedding-based similarity.
+         * Usage: expect(trace).toMatchSemanticOutput("order confirmed")
+         */
+        toMatchSemanticOutput(trace, expected) {
+            const steps = trace.getLLMSteps();
+            const fullOutput = steps
+                .map((s) => [s.completion, s.contains].filter(Boolean).join(' '))
+                .join(' ')
+                .toLowerCase();
+            const pass = fullOutput.includes(expected.toLowerCase());
+            return {
+                pass,
+                message: () => {
+                    if (pass) {
+                        return `Expected trace output NOT to semantically match "${expected}"`;
+                    }
+                    return `Expected trace output to semantically match "${expected}", but got: "${fullOutput || '(empty)'}"`;
+                },
+            };
+        },
+    });
+}
+//# sourceMappingURL=index.js.map

package/dist/matchers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/matchers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAA;AAiB/B;;;GAGG;AACH,MAAM,UAAU,gBAAgB;IAC9B,MAAM,CAAC,MAAM,CAAC;QACZ;;;WAGG;QACH,aAAa,CAAC,KAAkB,EAAE,SAAwB,EAAE;YAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAA;YAEjC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAa,EAAE,EAAE;gBAC9C,IAAI,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;oBAAE,OAAO,KAAK,CAAA;gBAC7D,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;oBACpB,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC;yBAC3D,MAAM,CAAC,OAAO,CAAC;yBACf,IAAI,CAAC,GAAG,CAAC;yBACT,WAAW,EAAE,CAAA;oBAChB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;wBAAE,OAAO,KAAK,CAAA;gBACrE,CAAC;gBACD,OAAO,IAAI,CAAA;YACb,CAAC,CAAC,CAAA;YAEF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhC,OAAO;gBACL,IAAI;gBACJ,OAAO,EAAE,GAAG,EAAE;oBACZ,IAAI,IAAI,EAAE,CAAC;wBACT,OAAO,gDAAgD,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAA;oBACjF,CAAC;oBACD,MAAM,WAAW,GACf,KAAK,CAAC,MAAM,KAAK,CAAC;wBAChB,CAAC,CAAC,4BAA4B;wBAC9B,CAAC,CAAC,mBAAmB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAA;oBAChD,OAAO,4CAA4C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,SAAS,WAAW,EAAE,CAAA;gBACjG,CAAC;aACF,CAAA;QACH,CAAC;QAED;;;WAGG;QACH,UAAU,CAAC,KAAkB,EAAE,QAAgB;YAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,EAAE,CAAA;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAA;YAEnD,OAAO;gBACL,IAAI;gBACJ,OAAO,EAAE,GAAG,EAAE;oBACZ,IAAI,IAAI,EAAE,CAAC;wBACT,OAAO,oCAAoC,QAAQ,GAAG,CAAA;oBACxD,CAAC;oBACD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;oBACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,cAAc,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAA;oBACvG,OAAO,kBAAkB,QAAQ,uBAAuB,QAAQ,EAAE,CAAA;gBACpE,CAAC;aACF,CAAA;QACH,CAAC;QAED;;;;;WAKG;QACH,qBAAqB,CAAC,KAAkB,EAAE,QAAgB;YACxD,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAA;YACjC,MAAM,UAAU,GAAG,KAAK;iBACrB,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;iBACzE,IAAI,CAAC,GAAG,CAAC;iBACT,WAAW,EAAE,CAAA;YAEhB,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAA;YAExD,OAAO;gBACL,IAAI;gBACJ,OAAO,EAAE,GAAG,EAAE;oBACZ,IAAI,IAAI,EAAE,CAAC;wBACT,OAAO,oDAAoD,QAAQ,GAAG,CAAA;oBACxE,CAAC;oBACD,OAAO,gDAAgD,QAAQ,gBAAgB,UAAU,IAAI,SAAS,GAAG,CAAA;gBAC3G,CAAC;aACF,CAAA;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}

package/dist/reporter.d.ts

@@ -0,0 +1,3 @@
+import type { FileResult } from './runner.js';
+export declare function reportResults(fileResults: FileResult[]): void;
+//# sourceMappingURL=reporter.d.ts.map

package/dist/reporter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,UAAU,EAAE,MAAM,aAAa,CAAA;AAEzD,wBAAgB,aAAa,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,IAAI,CAsB7D"}

package/dist/reporter.js

@@ -0,0 +1,72 @@
+import chalk from 'chalk';
+export function reportResults(fileResults) {
+    let totalPassed = 0;
+    let totalFailed = 0;
+    let totalDurationMs = 0;
+    for (const fileResult of fileResults) {
+        if (fileResults.length > 1) {
+            console.log(chalk.dim(`\n${fileResult.file}`));
+        }
+        for (const result of fileResult.results) {
+            printTestResult(result);
+            totalDurationMs += result.durationMs;
+            if (result.passed) {
+                totalPassed++;
+            }
+            else {
+                totalFailed++;
+            }
+        }
+    }
+    printSummary(totalPassed, totalFailed, totalDurationMs);
+}
+function printTestResult(result) {
+    const duration = chalk.dim(`(${formatDuration(result.durationMs)})`);
+    if (result.passed) {
+        console.log(`  ${chalk.green('✓')} ${result.name} ${duration}`);
+    }
+    else {
+        console.log(`  ${chalk.red('✗')} ${result.name} ${duration}`);
+        if (result.error) {
+            const errorLines = formatError(result.error);
+            for (const line of errorLines) {
+                console.log(`    ${chalk.red('→')} ${line}`);
+            }
+        }
+    }
+}
+function printSummary(passed, failed, totalMs) {
+    const total = passed + failed;
+    console.log('');
+    if (passed > 0) {
+        console.log(chalk.green(`${passed} passed`));
+    }
+    if (failed > 0) {
+        console.log(chalk.red(`${failed} failed`));
+    }
+    console.log(chalk.dim(`Total: ${total}`));
+    console.log(chalk.dim(`Duration: ${formatDuration(totalMs)}`));
+}
+function formatDuration(ms) {
+    if (ms >= 1000) {
+        return `${(ms / 1000).toFixed(1)}s`;
+    }
+    return `${ms}ms`;
+}
+function formatError(error) {
+    const lines = [];
+    if (error.message) {
+        lines.push(error.message);
+    }
+    if (error.stack) {
+        const stackLines = error.stack
+            .split('\n')
+            .slice(1)
+            .map((l) => l.trim())
+            .filter((l) => l.startsWith('at '))
+            .slice(0, 3);
+        lines.push(...stackLines);
+    }
+    return lines;
+}
+//# sourceMappingURL=reporter.js.map

package/dist/reporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../src/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAA;AAGzB,MAAM,UAAU,aAAa,CAAC,WAAyB;IACrD,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,eAAe,GAAG,CAAC,CAAA;IAEvB,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;QAChD,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACxC,eAAe,CAAC,MAAM,CAAC,CAAA;YACvB,eAAe,IAAI,MAAM,CAAC,UAAU,CAAA;YACpC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,WAAW,EAAE,CAAA;YACf,CAAC;iBAAM,CAAC;gBACN,WAAW,EAAE,CAAA;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAED,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,eAAe,CAAC,CAAA;AACzD,CAAC;AAED,SAAS,eAAe,CAAC,MAAkB;IACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;IAEpE,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAA;IACjE,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAA;QAC7D,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC5C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,MAAc,EAAE,MAAc,EAAE,OAAe;IACnE,MAAM,KAAK,GAAG,MAAM,GAAG,MAAM,CAAA;IAC7B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAA;IAC9C,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,SAAS,CAAC,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,CAAA;IACzC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;AAChE,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC;QACf,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;IACrC,CAAC;IACD,OAAO,GAAG,EAAE,IAAI,CAAA;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,KAAY;IAC/B,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAC3B,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK;aAC3B,KAAK,CAAC,IAAI,CAAC;aACX,KAAK,CAAC,CAAC,CAAC;aACR,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;aAClC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACd,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;IAC3B,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,16 @@
+import type { RunnerHooks } from './trace-adapter/context.js';
+export interface TestResult {
+    name: string;
+    passed: boolean;
+    durationMs: number;
+    error?: Error;
+}
+export interface FileResult {
+    file: string;
+    results: TestResult[];
+}
+export interface RunnerOptions {
+    hooks?: RunnerHooks;
+}
+export declare function runFiles(files: string[], options?: RunnerOptions): Promise<FileResult[]>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAA;AAE7D,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,OAAO,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,KAAK,CAAA;CACd;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,UAAU,EAAE,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAA;CACpB;AAED,wBAAsB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CASlG"}

package/dist/runner.js

@@ -0,0 +1,55 @@
+import { clearRegistry, getRegistry } from './core/registry.js';
+import { startTraceSession } from './trace-adapter/context.js';
+export async function runFiles(files, options = {}) {
+    const fileResults = [];
+    for (const file of files) {
+        const result = await runFile(file, options);
+        fileResults.push(result);
+    }
+    return fileResults;
+}
+async function runFile(file, options) {
+    const { hooks = {} } = options;
+    // 1. Clear the global registry before loading the file
+    clearRegistry();
+    // 2. Dynamically import the test file (triggers aiTest() registrations)
+    await import(file);
+    const registry = getRegistry();
+    const results = [];
+    // 3. Run beforeAll hooks
+    for (const hook of registry.beforeAllHooks) {
+        await hook();
+    }
+    // 4. Execute each test sequentially
+    for (const entry of registry.tests) {
+        const { context, finalise } = startTraceSession();
+        if (hooks.onTestStart) {
+            await hooks.onTestStart(entry.name);
+        }
+        const startTime = Date.now();
+        let passed = false;
+        let error;
+        try {
+            await entry.fn(context);
+            passed = true;
+        }
+        catch (err) {
+            error = err instanceof Error ? err : new Error(String(err));
+        }
+        const durationMs = Date.now() - startTime;
+        if (hooks.onTestFinish) {
+            await hooks.onTestFinish(entry.name, passed, durationMs);
+        }
+        if (hooks.onTraceComplete) {
+            await hooks.onTraceComplete(entry.name, context.trace);
+        }
+        finalise();
+        results.push({ name: entry.name, passed, durationMs, error });
+    }
+    // 5. Run afterAll hooks
+    for (const hook of registry.afterAllHooks) {
+        await hook();
+    }
+    return { file, results };
+}
+//# sourceMappingURL=runner.js.map

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAmB9D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAe,EAAE,UAAyB,EAAE;IACzE,MAAM,WAAW,GAAiB,EAAE,CAAA;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC3C,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC1B,CAAC;IAED,OAAO,WAAW,CAAA;AACpB,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,IAAY,EAAE,OAAsB;IACzD,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,GAAG,OAAO,CAAA;IAE9B,uDAAuD;IACvD,aAAa,EAAE,CAAA;IAEf,wEAAwE;IACxE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;IAElB,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAA;IAC9B,MAAM,OAAO,GAAiB,EAAE,CAAA;IAEhC,yBAAyB;IACzB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC3C,MAAM,IAAI,EAAE,CAAA;IACd,CAAC;IAED,oCAAoC;IACpC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,iBAAiB,EAAE,CAAA;QAEjD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACrC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC5B,IAAI,MAAM,GAAG,KAAK,CAAA;QAClB,IAAI,KAAwB,CAAA;QAE5B,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,CAAA;YACvB,MAAM,GAAG,IAAI,CAAA;QACf,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QAC7D,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;QAEzC,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAA;QAC1D,CAAC;QAED,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QACxD,CAAC;QAED,QAAQ,EAAE,CAAA;QAEV,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAA;IAC/D,CAAC;IAED,wBAAwB;IACxB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,aAAa,EAAE,CAAC;QAC1C,MAAM,IAAI,EAAE,CAAA;IACd,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AAC1B,CAAC"}

package/dist/test-setup.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Import this file in your AI test files to get:
+ *   - aiTest / beforeAll / afterAll available as globals (TypeScript types + runtime)
+ *   - Custom matcher types (toHaveLLMStep, toCallTool, toMatchSemanticOutput)
+ *
+ * The CLI registers matchers at startup, so this import is for TypeScript
+ * type awareness only — no double-registration occurs at runtime.
+ */
+import './core/registry.js';
+import './matchers/index.js';
+export {};
+//# sourceMappingURL=test-setup.d.ts.map

package/dist/test-setup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-setup.d.ts","sourceRoot":"","sources":["../src/test-setup.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,oBAAoB,CAAA;AAG3B,OAAO,qBAAqB,CAAA;AAE5B,OAAO,EAAE,CAAA"}

package/dist/test-setup.js

@@ -0,0 +1,13 @@
+/**
+ * Import this file in your AI test files to get:
+ *   - aiTest / beforeAll / afterAll available as globals (TypeScript types + runtime)
+ *   - Custom matcher types (toHaveLLMStep, toCallTool, toMatchSemanticOutput)
+ *
+ * The CLI registers matchers at startup, so this import is for TypeScript
+ * type awareness only — no double-registration occurs at runtime.
+ */
+// Side-effect: populates globalThis.aiTest + brings declare global into scope
+import './core/registry.js';
+// Side-effect: brings declare module 'expect' augmentation into scope
+import './matchers/index.js';
+//# sourceMappingURL=test-setup.js.map

package/dist/test-setup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-setup.js","sourceRoot":"","sources":["../src/test-setup.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,8EAA8E;AAC9E,OAAO,oBAAoB,CAAA;AAE3B,sEAAsE;AACtE,OAAO,qBAAqB,CAAA"}

package/dist/trace-adapter/context.d.ts

@@ -0,0 +1,51 @@
+export interface LLMStep {
+    model: string;
+    prompt?: string;
+    completion?: string;
+    contains?: string;
+}
+export interface ToolCall {
+    name: string;
+    args?: Record<string, unknown>;
+    result?: unknown;
+}
+export interface TraceStep {
+    type: 'llm' | 'tool' | 'generic';
+    timestamp: number;
+    durationMs: number;
+    data: Record<string, unknown>;
+}
+export interface TraceHandle {
+    /** All recorded steps in this trace session */
+    getSteps(): TraceStep[];
+    /** Only LLM inference steps */
+    getLLMSteps(): LLMStep[];
+    /** Only tool-call steps */
+    getToolCalls(): ToolCall[];
+    /** Record an LLM step (used by stubs / real adapter) */
+    recordLLMStep(step: LLMStep): void;
+    /** Record a tool call (used by stubs / real adapter) */
+    recordToolCall(call: ToolCall): void;
+}
+export interface AITestContext {
+    trace: TraceHandle;
+}
+/** Extension points for runner hooks (scaffold for future backend integration) */
+export interface RunnerHooks {
+    onTestStart?(name: string): void | Promise<void>;
+    onTestFinish?(name: string, passed: boolean, durationMs: number): void | Promise<void>;
+    onTraceComplete?(name: string, trace: TraceHandle): void | Promise<void>;
+}
+/**
+ * Create a stubbed trace handle for a single test execution.
+ * Later this can be replaced with a real ElasticDash backend call.
+ */
+export declare function createTraceHandle(): TraceHandle;
+/**
+ * Start a trace session before a test and return the context + a finalise fn.
+ */
+export declare function startTraceSession(): {
+    context: AITestContext;
+    finalise: () => void;
+};
+//# sourceMappingURL=context.d.ts.map

package/dist/trace-adapter/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../src/trace-adapter/context.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,MAAM,CAAC,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,KAAK,GAAG,MAAM,GAAG,SAAS,CAAA;IAChC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,+CAA+C;IAC/C,QAAQ,IAAI,SAAS,EAAE,CAAA;IACvB,+BAA+B;IAC/B,WAAW,IAAI,OAAO,EAAE,CAAA;IACxB,2BAA2B;IAC3B,YAAY,IAAI,QAAQ,EAAE,CAAA;IAC1B,wDAAwD;IACxD,aAAa,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,CAAA;IAClC,wDAAwD;IACxD,cAAc,CAAC,IAAI,EAAE,QAAQ,GAAG,IAAI,CAAA;CACrC;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,WAAW,CAAA;CACnB;AAED,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,WAAW,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAChD,YAAY,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACtF,eAAe,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACzE;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,IAAI,WAAW,CAsC/C;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI;IAAE,OAAO,EAAE,aAAa,CAAC;IAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;CAAE,CASpF"}

package/dist/trace-adapter/context.js

@@ -0,0 +1,52 @@
+/**
+ * Create a stubbed trace handle for a single test execution.
+ * Later this can be replaced with a real ElasticDash backend call.
+ */
+export function createTraceHandle() {
+    const steps = [];
+    const llmSteps = [];
+    const toolCalls = [];
+    return {
+        getSteps() {
+            return steps;
+        },
+        getLLMSteps() {
+            return llmSteps;
+        },
+        getToolCalls() {
+            return toolCalls;
+        },
+        recordLLMStep(step) {
+            llmSteps.push(step);
+            steps.push({
+                type: 'llm',
+                timestamp: Date.now(),
+                durationMs: 0,
+                data: step,
+            });
+        },
+        recordToolCall(call) {
+            toolCalls.push(call);
+            steps.push({
+                type: 'tool',
+                timestamp: Date.now(),
+                durationMs: 0,
+                data: call,
+            });
+        },
+    };
+}
+/**
+ * Start a trace session before a test and return the context + a finalise fn.
+ */
+export function startTraceSession() {
+    const trace = createTraceHandle();
+    const context = { trace };
+    return {
+        context,
+        finalise() {
+            // Placeholder: flush / send to ElasticDash backend here in the future
+        },
+    };
+}
+//# sourceMappingURL=context.js.map

package/dist/trace-adapter/context.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/trace-adapter/context.ts"],"names":[],"mappings":"AA4CA;;;GAGG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,KAAK,GAAgB,EAAE,CAAA;IAC7B,MAAM,QAAQ,GAAc,EAAE,CAAA;IAC9B,MAAM,SAAS,GAAe,EAAE,CAAA;IAEhC,OAAO;QACL,QAAQ;YACN,OAAO,KAAK,CAAA;QACd,CAAC;QAED,WAAW;YACT,OAAO,QAAQ,CAAA;QACjB,CAAC;QAED,YAAY;YACV,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,aAAa,CAAC,IAAa;YACzB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACnB,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,KAAK;gBACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,UAAU,EAAE,CAAC;gBACb,IAAI,EAAE,IAA0C;aACjD,CAAC,CAAA;QACJ,CAAC;QAED,cAAc,CAAC,IAAc;YAC3B,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACpB,KAAK,CAAC,IAAI,CAAC;gBACT,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,UAAU,EAAE,CAAC;gBACb,IAAI,EAAE,IAA0C;aACjD,CAAC,CAAA;QACJ,CAAC;KACF,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB;IAC/B,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAA;IACjC,MAAM,OAAO,GAAkB,EAAE,KAAK,EAAE,CAAA;IACxC,OAAO;QACL,OAAO;QACP,QAAQ;YACN,sEAAsE;QACxE,CAAC;KACF,CAAA;AACH,CAAC"}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "elasticdash-test",
+  "version": "0.1.0",
+  "description": "AI-native test runner for ElasticDash workflow testing",
+  "type": "module",
+  "bin": {
+    "elasticdash": "./dist/cli.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "commander": "^12.1.0",
+    "expect": "^29.7.0",
+    "fast-glob": "^3.3.2"
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.0",
+    "tsx": "^4.15.6",
+    "typescript": "^5.5.2"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "author": "ElasticDash <contact@elasticdash.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ElasticDash/elasticdash-test-js.git"
+  }
+}