elasticdash-test 0.1.13 → 0.1.16

package/README.md

@@ -28,7 +28,7 @@ An AI-native test runner for ElasticDash workflow testing. Built for async AI pi
 - 🎯 **Trace-first testing** — every test gets a `trace` context to record and assert on LLM calls and tool invocations
 - 🔍 **Automatic AI interception** — captures OpenAI, Gemini, and Grok calls without code changes
 - 🧪 **AI-specific matchers** — semantic output matching, LLM-judged evaluations, prompt assertions
-- 🛠️ **Tool recording & replay** — automatically trace tool calls with checkpoint-based replay
+- 🛠️ **Tool & LLM recording & replay** — automatically trace tool and AI calls with checkpoint-based replay and mock support
 - 📊 **Interactive dashboard** — browse workflows, debug traces, validate fixes visually
 - 🤖 **Agent mid-trace replay** — resume long-running agents from any task without re-execution
 - 🌐 **HTTP workflow mode** — run workflows against your live dev server for framework-heavy apps (Next.js, Remix, etc.) with full AI and tool call observability
@@ -266,6 +266,8 @@ export const callClaude = wrapAI('claude-sonnet-4-5', async (messages: Anthropic
 
 Use `wrapAI` when you have a custom AI wrapper or a provider not covered by automatic interception. For direct OpenAI/Anthropic/Gemini SDK calls inside a subprocess workflow, automatic interception via `installAIInterceptor` already handles recording without any code changes.
 
+**AI mocking (subprocess / test runner mode):** `wrapAI` also checks `resolveAIMock` at call time, so the dashboard can mock LLM responses the same way it mocks tool calls — without modifying your server code. Configure an `AIMockConfig` in the dashboard UI or pass it programmatically via the `aiMockConfig` option when running a workflow.
+
 ### HTTP Streaming Capture and Replay
 
 ElasticDash also captures non-AI `fetch` responses that stream over HTTP (for example SSE and NDJSON endpoints) in the HTTP interceptor.
@@ -354,6 +356,25 @@ The dashboard injects `x-elasticdash-run-id` and `x-elasticdash-server` headers
 
 > **Note:** Use `setHttpRunContext` (synchronous) if you only need observability and do not need step freezing. `initHttpRunContext` is required for the dashboard's breakpoint/replay functionality to work.
 
+### Dashboard Auto-Detection (env var mode)
+
+As an alternative to calling `initHttpRunContext` in your request handler, you can set two environment variables before starting your server or script. Every `wrapTool` and `wrapAI` call will then connect to the dashboard automatically — no code changes needed:
+
+```bash
+# Required: URL of the running ElasticDash dashboard
+ELASTICDASH_SERVER=http://localhost:4573
+
+# Optional: pre-registered run ID to fetch frozen steps for
+ELASTICDASH_RUN_ID=<run-id-from-dashboard>
+```
+
+- If only `ELASTICDASH_SERVER` is set, a fresh run ID is generated and all calls push live telemetry to the dashboard (observability only, no step freezing).
+- If both variables are set, frozen steps are fetched from the dashboard at startup and replayed as configured.
+- If the dashboard is unreachable the SDK falls through to live execution silently.
+- The initialization runs **once per process** — subsequent `wrapTool`/`wrapAI` calls reuse the cached context.
+
+This mode is intended for local development and testing scenarios. For production HTTP servers with concurrent requests, continue using `initHttpRunContext` inside your request handler.
+
 **Subprocess vs HTTP mode comparison:**
 
 | | Subprocess (default) | HTTP mode |
@@ -361,9 +382,10 @@ The dashboard injects `x-elasticdash-run-id` and `x-elasticdash-server` headers
 | Works with simple apps | Yes | Yes |
 | Works with Next.js / Remix | No | Yes |
 | Requires dev server running | No | Yes |
-| App code changes needed | Extract handler to `ed_workflows.ts` | Add `initHttpRunContext` to request handler |
+| App code changes needed | Extract handler to `ed_workflows.ts` | Add `initHttpRunContext` to request handler (or use env vars for auto-detect) |
 | AI / tool call observability | Automatic via interceptors | Via `wrapAI` / `wrapTool` push |
-| Step freezing / breakpoints | Yes | Yes (requires `initHttpRunContext`) |
+| Step freezing / breakpoints | Yes | Yes (`initHttpRunContext`, or `ELASTICDASH_SERVER` + `ELASTICDASH_RUN_ID` env vars) |
+| LLM response mocking | Yes (via `aiMockConfig`) | Yes (via frozen AI events) |
 
 ---
 
@@ -420,16 +442,25 @@ reportResults(results)
 **HTTP mode context (call inside your request handler):**
 
 ```ts
-import { initHttpRunContext } from 'elasticdash-test'
+import { initHttpRunContext, setHttpRunContext } from 'elasticdash-test'
 
 // Async — fetches frozen steps from dashboard to enable step freezing/breakpoints
 await initHttpRunContext(runId, dashboardUrl)
 
 // Synchronous alternative — observability only, no step freezing
-import { setHttpRunContext } from 'elasticdash-test'
 setHttpRunContext(runId, dashboardUrl)
 ```
 
+**Dashboard auto-detection (env var mode — no code changes needed):**
+
+```bash
+# Set before starting your server or script
+ELASTICDASH_SERVER=http://localhost:4573       # required
+ELASTICDASH_RUN_ID=<run-id-from-dashboard>    # optional, enables step freezing
+```
+
+`wrapTool` and `wrapAI` will auto-connect on their first call. See [Dashboard Auto-Detection](#dashboard-auto-detection-env-var-mode) for details.
+
 ---
 
 ## License

package/dist/dashboard-server.d.ts

@@ -40,6 +40,15 @@ export interface ToolMockEntry {
 export interface ToolMockConfig {
     [toolName: string]: ToolMockEntry;
 }
+/** Per-model AI mock configuration sent from the dashboard UI */
+export interface AIMockEntry {
+    mode: 'live' | 'mock-all' | 'mock-specific';
+    callIndices?: number[];
+    mockData?: Record<number, unknown>;
+}
+export interface AIMockConfig {
+    [modelName: string]: AIMockEntry;
+}
 export interface HttpWorkflowConfig {
     mode: 'http';
     url: string;

package/dist/dashboard-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dashboard-server.d.ts","sourceRoot":"","sources":["../src/dashboard-server.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AA2CD,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,oHAAoH;IACpH,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,eAAe,CAAA;IAC3C,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAAA;CAClC;AA2mCD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACtC,cAAc,CAAC,EAAE,kBAAkB,GAAG,MAAM,CAAA;CAC7C;AA0ID;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CAoZ1B;AAiFD,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC"}
+{"version":3,"file":"dashboard-server.d.ts","sourceRoot":"","sources":["../src/dashboard-server.ts"],"names":[],"mappings":"AAeA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AA2CD,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,oHAAoH;IACpH,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,eAAe,CAAA;IAC3C,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAAA;CAClC;AAED,iEAAiE;AACjE,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,eAAe,CAAA;IAC3C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,YAAY;IAC3B,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,CAAA;CACjC;AAqgGD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACtC,cAAc,CAAC,EAAE,kBAAkB,GAAG,MAAM,CAAA;CAC7C;AA4ID;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CA4b1B;AAiFD,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC"}