elasticdash-test 0.1.12 → 0.1.13-alpha

package/README.md

@@ -8,6 +8,8 @@ An AI-native test runner for ElasticDash workflow testing. Built for async AI pi
 - [Quick Start](#quick-start)
 - [Documentation](#documentation)
 - [Tool Recording](#tool-recording)
+- [AI Call Recording](#ai-call-recording)
+- [HTTP Workflow Mode](#http-workflow-mode)
 - [Configuration](#configuration)
 
 ### Open Detailed Docs
@@ -18,6 +20,8 @@ An AI-native test runner for ElasticDash workflow testing. Built for async AI pi
 - [Workflows Dashboard](docs/dashboard.md)
 - [Agent Mid-Trace Replay](docs/agents.md)
 - [Deno Support](docs/deno.md)
+- [Instrumentation Guide](docs/instrumentation.md) — how to write `ed_tools.ts`, `ed_workflows.ts`, `ed_agents.ts`
+- [Langfuse Trace Structure](docs/langfuse-trace-structure.md) — span structure required for dashboard replay
 
 ## Features
 
@@ -27,6 +31,7 @@ An AI-native test runner for ElasticDash workflow testing. Built for async AI pi
 - 🛠️ **Tool recording & replay** — automatically trace tool calls with checkpoint-based replay
 - 📊 **Interactive dashboard** — browse workflows, debug traces, validate fixes visually
 - 🤖 **Agent mid-trace replay** — resume long-running agents from any task without re-execution
+- 🌐 **HTTP workflow mode** — run workflows against your live dev server for framework-heavy apps (Next.js, Remix, etc.) with full AI and tool call observability
 
 ---
 
@@ -98,11 +103,11 @@ Total: 3
 Duration: 3.4s
 ```
 
-**Workflow export requirements:**
+**Workflow export requirements (subprocess mode):**
 
 - Export plain callable functions from `ed_workflows.ts/js`.
 - Use JSON-serializable inputs/outputs (object or array) so dashboard replay can pass args and read results.
-- Do not export framework-bound handlers directly (for example Next.js `NextRequest`/`NextResponse` route handlers).
+- Do not export framework-bound handlers directly (for example Next.js `NextRequest`/`NextResponse` route handlers) — use [HTTP workflow mode](#http-workflow-mode) instead.
 
 ---
 
@@ -113,11 +118,15 @@ Duration: 3.4s
 - **[Test Matchers](docs/matchers.md)** — all available matchers with examples
 - **[Tool Recording & Replay](docs/tools.md)** — automatic tool tracing and checkpoint-based replay
 
-### Advanced Features  
+### Advanced Features
 - **[Workflows Dashboard](docs/dashboard.md)** — interactive workflow browser, debugger, and fetching traces from Langfuse
 - **[Agent Mid-Trace Replay](docs/agents.md)** — resume long-running agents from any task
 - **[Deno Support](docs/deno.md)** — using ElasticDash Test in Deno projects
 
+### Integration & Reference
+- **[Instrumentation Guide](docs/instrumentation.md)** — how to write `ed_tools.ts`, `ed_workflows.ts`, and `ed_agents.ts` to connect your production code to ElasticDash
+- **[Langfuse Trace Structure](docs/langfuse-trace-structure.md)** — Langfuse span structure required for dashboard replay and tool-level diffing
+
 ---
 
 ## Quick Reference
@@ -192,7 +201,18 @@ No code changes needed — just run your workflow and assertions work automatica
 
 ### Tool Recording
 
-Manual instrumentation pattern: isolate tracing in the service `.then/.catch` path so tracing failures never block business logic:
+**Recommended: `wrapTool`** wraps a tool function and automatically records its name, input, output, duration, and any streaming output. Works in both subprocess mode and HTTP mode:
+
+```ts
+import { wrapTool } from 'elasticdash-test/http'
+import { runSelectQuery } from './services/dataService'
+
+export const dataService = wrapTool('dataService', async (input: { query: string }) => {
+  return await runSelectQuery(input.query)
+})
+```
+
+**Manual pattern (legacy):** isolate tracing in the service `.then/.catch` path so tracing failures never block business logic:
 
 ```ts
 import { runSelectQuery } from './services/dataService'
@@ -225,6 +245,27 @@ In manual mode, always isolate tracing in a separate `try/catch` so trace loggin
 
 **→ See [Tool Recording & Replay](docs/tools.md) for checkpoint-based replay and freezing**
 
+### AI Call Recording
+
+**`wrapAI`** wraps any AI call function and records its name, input, output, duration, and token usage (auto-detected for Anthropic, OpenAI, and Gemini SDK responses):
+
+```ts
+import { wrapAI } from 'elasticdash-test/http'
+import Anthropic from '@anthropic-ai/sdk'
+
+const client = new Anthropic()
+
+export const callClaude = wrapAI('claude-sonnet-4-5', async (messages: Anthropic.MessageParam[]) => {
+  return await client.messages.create({
+    model: 'claude-sonnet-4-5-20250929',
+    max_tokens: 1024,
+    messages,
+  })
+})
+```
+
+Use `wrapAI` when you have a custom AI wrapper or a provider not covered by automatic interception. For direct OpenAI/Anthropic/Gemini SDK calls inside a subprocess workflow, automatic interception via `installAIInterceptor` already handles recording without any code changes.
+
 ### HTTP Streaming Capture and Replay
 
 ElasticDash also captures non-AI `fetch` responses that stream over HTTP (for example SSE and NDJSON endpoints) in the HTTP interceptor.
@@ -266,6 +307,64 @@ buffer += decoder.decode()
 
 ---
 
+## HTTP Workflow Mode
+
+For apps where subprocess import fails (Next.js, Remix, SvelteKit, etc.), configure workflows to call your running dev server directly instead of importing the handler:
+
+```ts
+// elasticdash.config.ts
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  workflows: {
+    runChat: {
+      mode: 'http',
+      url: 'http://localhost:3001/api/chat',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'x-user-id': '{{env.DEV_USER_ID}}',
+      },
+      bodyTemplate: {
+        messages: [{ role: 'user', content: '{{input.message}}' }],
+        selectedModel: 'claude-sonnet-4-5-20250929',
+      },
+      responseFormat: 'vercel-ai-stream',
+    },
+  },
+}
+```
+
+To enable full AI and tool call observability in HTTP mode, install `elasticdash-test` in your app and import from the lightweight subpath (avoids bundling server-only deps into your framework build):
+
+```ts
+// app/api/chat/route.ts
+import { setHttpRunContext, wrapTool, wrapAI } from 'elasticdash-test/http'
+
+export async function POST(req: Request) {
+  const runId = req.headers.get('x-elasticdash-run-id')
+  const serverUrl = req.headers.get('x-elasticdash-server')
+  if (runId && serverUrl) {
+    setHttpRunContext(runId, serverUrl)
+  }
+  // ... rest of handler
+}
+```
+
+The dashboard injects `x-elasticdash-run-id` and `x-elasticdash-server` headers automatically when triggering a run. Every `wrapAI` and `wrapTool` call downstream pushes telemetry events back to the dashboard in real time.
+
+**Subprocess vs HTTP mode comparison:**
+
+| | Subprocess (default) | HTTP mode |
+|---|---|---|
+| Works with simple apps | Yes | Yes |
+| Works with Next.js / Remix | No | Yes |
+| Requires dev server running | No | Yes |
+| App code changes needed | Extract handler to `ed_workflows.ts` | Add `setHttpRunContext` to request handler |
+| AI / tool call observability | Automatic via interceptors | Via `wrapAI` / `wrapTool` push |
+| Tool replay | Yes | No (hits live server) |
+
+---
+
 ## Configuration
 
 Optional `elasticdash.config.ts` at project root:
@@ -277,6 +376,18 @@ export default {
 }
 ```
 
+**Dashboard port:** defaults to `4573`. Override via CLI flag or `.env`:
+
+```bash
+# .env
+ELASTICDASH_PORT=5000
+```
+
+```bash
+# or CLI flag
+npx elasticdash dashboard --port 5000
+```
+
 Optional project file: `ed_workers.ts` can be used by your app architecture (for example, exporting worker handlers), but it is not required or discovered by the ElasticDash CLI/dashboard.
 
 ## TypeScript Setup
@@ -304,6 +415,14 @@ const results = await runFiles(['./tests/flow.ai.test.ts'])
 reportResults(results)
 ```
 
+**HTTP mode context (call inside your request handler):**
+
+```ts
+import { setHttpRunContext } from 'elasticdash-test/http'
+
+setHttpRunContext(runId, dashboardUrl)
+```
+
 ---
 
 ## License

package/dist/capture/event.d.ts

@@ -7,6 +7,12 @@ export interface WorkflowEvent {
     output: unknown;
     timestamp: number;
     durationMs: number;
+    /** Token usage for LLM (ai) events */
+    usage?: {
+        inputTokens?: number;
+        outputTokens?: number;
+        totalTokens?: number;
+    };
     /** Optional: ID of the agent task that produced this event */
     agentTaskId?: string;
     /** Optional: Zero-based index of the agent task that produced this event */

package/dist/capture/event.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"event.d.ts","sourceRoot":"","sources":["../../src/capture/event.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,aAAa,CAAA;AAE7E,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,iBAAiB,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB"}
+{"version":3,"file":"event.d.ts","sourceRoot":"","sources":["../../src/capture/event.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,aAAa,CAAA;AAE7E,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,iBAAiB,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,sCAAsC;IACtC,KAAK,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC7E,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB"}

package/dist/dashboard-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dashboard-server.d.ts","sourceRoot":"","sources":["../src/dashboard-server.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAuCD,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,oHAAoH;IACpH,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,eAAe,CAAA;IAC3C,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAAA;CAClC;AAqpFD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CA+S1B;AAiFD,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC"}
+{"version":3,"file":"dashboard-server.d.ts","sourceRoot":"","sources":["../src/dashboard-server.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AA2CD,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,oHAAoH;IACpH,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,eAAe,CAAA;IAC3C,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAAA;CAClC;AAuzFD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CA2T1B;AAiFD,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC"}