elasticdash-test 0.1.24 → 0.1.25-alpha-2

package/docs/agent-integration-guide.md

@@ -0,0 +1,557 @@
+# ElasticDash SDK — Agent Integration Guide
+
+> **Audience**: Coding agents (Claude Code, Cursor, Copilot, Codex, Windsurf, etc.) integrating `elasticdash-test` into a user's project.
+>
+> **When to use**: Follow this guide step-by-step when a user asks you to "integrate ElasticDash", "set up ElasticDash testing", "implement elasticdash-test", or similar. Each step produces a specific file — work through them in order.
+>
+> **How to use**: Read the full guide first to understand the project's structure, then execute each step. Use the decision trees section to choose the right pattern for the user's project (framework type, module system, tool architecture).
+
+---
+
+## Prerequisites
+
+- Node.js >= 20
+- npm, yarn, or pnpm
+- At least one AI workflow function (a callable async function that makes LLM/tool calls)
+- LLM provider API keys for providers used in workflows (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`)
+
+### Detect project type before starting
+
+Check these before choosing patterns:
+
+| Check | How | Result |
+|-------|-----|--------|
+| Module system | `package.json` → `"type"` field | `"module"` = ESM, `"commonjs"` or missing = CJS |
+| Framework | Look for `next.config.*`, `remix.config.*`, `svelte.config.*` | Present = framework project, use HTTP mode |
+| Path aliases | `tsconfig.json` → `"paths"` field | Present = need advanced dashboard script |
+| Tool architecture | Does the project use a central `dispatchTool(name, args)` function? | Yes = Pattern A, No = Pattern B |
+
+---
+
+## Step 1: Install
+
+```bash
+npm install elasticdash-test
+```
+
+Add to `.gitignore`:
+
+```gitignore
+.temp/
+.ed_traces/
+```
+
+---
+
+## Step 2: Create `ed_tools.ts`
+
+Create `ed_tools.ts` in the project root. This file wraps each tool function with tracing so ElasticDash can record and replay tool calls.
+
+### Choose your pattern
+
+- **Pattern B (recommended for most projects)**: inline async with mock support. Use when tools are imported individually, no central dispatcher exists, or you want dashboard mock support.
+- **Pattern A**: `withTrace` HOF with central dispatcher. Use when tools are pure functions called through a single `dispatchTool(name, args)` dispatcher.
+
+### Pattern B template (recommended)
+
+```ts
+// ed_tools.ts
+// Replace imports with your actual tool functions and source paths
+import { YOUR_TOOL_1 as YOUR_TOOL_1_impl } from './YOUR_SOURCE_PATH_1'
+import { YOUR_TOOL_2 as YOUR_TOOL_2_impl } from './YOUR_SOURCE_PATH_2'
+
+// ---------------------------------------------------------------------------
+// Helpers — copy as-is, no customization needed
+// ---------------------------------------------------------------------------
+
+function resolveMock(toolName: string): { mocked: true; result: unknown } | { mocked: false } {
+  const g = globalThis as any
+  const mocks = g.__ELASTICDASH_TOOL_MOCKS__
+  if (!mocks) return { mocked: false }
+
+  const entry = mocks[toolName]
+  if (!entry || entry.mode === 'live') return { mocked: false }
+
+  if (!g.__ELASTICDASH_TOOL_CALL_COUNTERS__) g.__ELASTICDASH_TOOL_CALL_COUNTERS__ = {}
+  const counters = g.__ELASTICDASH_TOOL_CALL_COUNTERS__
+  counters[toolName] = (counters[toolName] ?? 0) + 1
+  const callNumber = counters[toolName]
+
+  if (entry.mode === 'mock-all') {
+    const data = entry.mockData ?? {}
+    const result = data[callNumber] !== undefined ? data[callNumber] : data[0]
+    return { mocked: true, result }
+  }
+
+  if (entry.mode === 'mock-specific') {
+    const indices = entry.callIndices ?? []
+    if (indices.includes(callNumber)) {
+      return { mocked: true, result: (entry.mockData ?? {})[callNumber] }
+    }
+    return { mocked: false }
+  }
+
+  return { mocked: false }
+}
+
+async function safeRecordToolCall(tool: string, input: any, result: any) {
+  if (!(globalThis as any).__ELASTICDASH_WORKER__) return
+  try {
+    const { recordToolCall } = await import('elasticdash-test')
+    recordToolCall(tool, input, result)
+  } catch { /* tracing must never block business logic */ }
+}
+
+// ---------------------------------------------------------------------------
+// Tools — one export per tool, following this pattern
+// ---------------------------------------------------------------------------
+
+// TEMPLATE: Copy this block for each tool, replacing YOUR_TOOL_NAME and YOUR_TOOL_IMPL
+export const YOUR_TOOL_1 = async (input: any) => {
+  const mock = resolveMock('YOUR_TOOL_1')
+  if (mock.mocked) {
+    await safeRecordToolCall('YOUR_TOOL_1', input, mock.result)
+    return mock.result
+  }
+
+  return await YOUR_TOOL_1_impl(input)
+    .then(async (res: any) => {
+      await safeRecordToolCall('YOUR_TOOL_1', input, res)
+      return res
+    })
+    .catch(async (err: any) => {
+      await safeRecordToolCall('YOUR_TOOL_1', input, err)
+      throw err
+    })
+}
+
+export const YOUR_TOOL_2 = async (input: any) => {
+  const mock = resolveMock('YOUR_TOOL_2')
+  if (mock.mocked) {
+    await safeRecordToolCall('YOUR_TOOL_2', input, mock.result)
+    return mock.result
+  }
+
+  return await YOUR_TOOL_2_impl(input)
+    .then(async (res: any) => {
+      await safeRecordToolCall('YOUR_TOOL_2', input, res)
+      return res
+    })
+    .catch(async (err: any) => {
+      await safeRecordToolCall('YOUR_TOOL_2', input, err)
+      throw err
+    })
+}
+```
+
+### Pattern A template (dispatcher-based)
+
+```ts
+// ed_tools.ts
+// Replace imports with your actual tool functions and source paths
+import {
+  YOUR_TOOL_1 as _YOUR_TOOL_1,
+  YOUR_TOOL_2 as _YOUR_TOOL_2,
+  dispatchTool as _dispatchTool,
+} from './YOUR_TOOLS_SOURCE'
+
+async function withTrace<I, O>(
+  toolName: string,
+  input: I,
+  fn: (input: I) => Promise<O>,
+): Promise<O> {
+  const result = await fn(input)
+  try {
+    const { recordToolCall } = await import('elasticdash-test')
+    recordToolCall(toolName, input, result)
+  } catch { /* tracing must never block business logic */ }
+  return result
+}
+
+export function YOUR_TOOL_1(input: Parameters<typeof _YOUR_TOOL_1>[0]) {
+  return withTrace('YOUR_TOOL_1', input, _YOUR_TOOL_1)
+}
+
+export function YOUR_TOOL_2(input: Parameters<typeof _YOUR_TOOL_2>[0]) {
+  return withTrace('YOUR_TOOL_2', input, _YOUR_TOOL_2)
+}
+
+export async function dispatchTool(name: string, args: Record<string, unknown>) {
+  switch (name) {
+    case 'YOUR_TOOL_1': return YOUR_TOOL_1(args as Parameters<typeof _YOUR_TOOL_1>[0])
+    case 'YOUR_TOOL_2': return YOUR_TOOL_2(args as Parameters<typeof _YOUR_TOOL_2>[0])
+    default: return _dispatchTool(name, args)
+  }
+}
+```
+
+### Alternative: `wrapTool` shorthand
+
+For simpler setups where you don't need mock support or dashboard replays:
+
+```ts
+// ed_tools.ts
+import { wrapTool } from 'elasticdash-test'
+import { YOUR_TOOL_1 as YOUR_TOOL_1_impl } from './YOUR_SOURCE_PATH'
+
+export const YOUR_TOOL_1 = wrapTool('YOUR_TOOL_1', YOUR_TOOL_1_impl)
+```
+
+### Important rules
+
+- The string name passed to `resolveMock()`, `safeRecordToolCall()`, or `wrapTool()` **must match** the exported function name exactly.
+- Each tool function must accept a single input object and return a plain value (JSON-serializable).
+- Tool functions must not close over HTTP context, framework state, or database clients — extract pure logic first.
+
+### Next.js only
+
+Add `elasticdash-test` to `serverExternalPackages` in `next.config.ts`:
+
+```ts
+// next.config.ts
+const nextConfig = {
+  serverExternalPackages: ['elasticdash-test'],
+}
+export default nextConfig
+```
+
+---
+
+## Step 3: Create `ed_workflows.ts`
+
+Create `ed_workflows.ts` in the project root. This file exports workflow functions for the ElasticDash runner.
+
+### Simple case — direct re-export
+
+```ts
+// ed_workflows.ts
+// Replace with your actual workflow function and source path
+export { YOUR_WORKFLOW } from './YOUR_SOURCE_PATH'
+```
+
+### Framework adapter case (Next.js / Remix)
+
+When the workflow lives inside a route handler, create a plain-value wrapper:
+
+```ts
+// ed_workflows.ts
+import { YOUR_HANDLER as _YOUR_HANDLER } from './app/api/YOUR_ROUTE/route'
+
+export async function YOUR_WORKFLOW(input: { message: string; sessionId: string }) {
+  return _YOUR_HANDLER(input)
+}
+```
+
+### Streaming workflow case (Vercel AI SDK)
+
+Create a separate handler file that is only imported by `ed_workflows.ts`:
+
+```ts
+// app/api/chat-stream/chatStreamHandler.ts
+import { NextRequest } from 'next/server'
+import { readVercelAIStream, recordToolCall } from 'elasticdash-test'
+import type { VercelAIStreamResult } from 'elasticdash-test'
+import { POST } from './route'
+
+export async function chatStreamHandler(args: {
+  messages: Array<{ role: string; content: string }>
+  sessionId?: string
+}): Promise<VercelAIStreamResult> {
+  const req = new NextRequest('http://localhost/api/chat-stream', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(args),
+  })
+
+  const response = await POST(req)
+
+  if (response.headers.get('x-vercel-ai-data-stream') !== 'v1') {
+    const errorMessage = await response.text().catch(() => `HTTP ${response.status}`)
+    return { message: errorMessage, type: 'error', error: errorMessage }
+  }
+
+  const result = await readVercelAIStream(response)
+  recordToolCall('chatStream', args, result)
+  return result
+}
+```
+
+Then re-export from `ed_workflows.ts`:
+
+```ts
+// ed_workflows.ts
+export { chatStreamHandler } from './app/api/chat-stream/chatStreamHandler'
+```
+
+### Requirements for all workflow exports
+
+- Accept only JSON-serializable inputs (strings, numbers, arrays, plain objects)
+- Return only JSON-serializable outputs
+- Must not depend on framework runtime APIs, HTTP request context, or live service clients
+- If a dependency is non-serializable (e.g., database client), instantiate it inside `ed_workflows.ts`, not passed as a parameter
+
+---
+
+## Step 4: Update workflow imports
+
+Change your workflow code to import tools from `ed_tools.ts` instead of the original source files:
+
+```ts
+// BEFORE
+import { YOUR_TOOL_1 } from './services/YOUR_SOURCE'
+
+// AFTER
+import { YOUR_TOOL_1 } from './ed_tools'
+```
+
+This single import change makes all tool calls observable by ElasticDash.
+
+---
+
+## Step 5: Add config and scripts
+
+### `elasticdash.config.ts`
+
+Create in project root:
+
+```ts
+// elasticdash.config.ts
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  traceMode: 'local' as const,
+}
+```
+
+### `package.json` scripts
+
+Add these scripts:
+
+```json
+{
+  "scripts": {
+    "dashboard:ai": "elasticdash dashboard",
+    "test:ai": "elasticdash test"
+  }
+}
+```
+
+**If the project uses TypeScript path aliases** (e.g., `@/lib/...` in `tsconfig.json` `paths`):
+
+```json
+{
+  "scripts": {
+    "dashboard:ai": "NODE_OPTIONS='--import tsx/esm --require tsx/cjs --require tsconfig-paths/register' elasticdash dashboard"
+  }
+}
+```
+
+---
+
+## Step 6: Environment variables
+
+Set in `.env` or CI secrets:
+
+| Variable | Description | Required |
+|----------|-------------|----------|
+| `ELASTICDASH_API_URL` | Backend server URL (`https://server.elasticdash.com` for cloud) | For upload/CI |
+| `ELASTICDASH_API_KEY` | Project API key from dashboard | For upload/CI |
+| `ELASTICDASH_CAPTURE_TRACE` | Set to `1` to record a trace fixture | For trace recording |
+| `OPENAI_API_KEY` | OpenAI API key | If using OpenAI |
+| `ANTHROPIC_API_KEY` | Anthropic API key | If using Claude |
+| `GEMINI_API_KEY` | Google Gemini API key | If using Gemini |
+| `GROK_API_KEY` | xAI Grok API key | If using Grok |
+
+---
+
+## Step 7: Write a test
+
+### Option A: `aiTest` — live workflow testing
+
+Create a file ending in `.ai.test.ts`:
+
+```ts
+// tests/YOUR_WORKFLOW.ai.test.ts
+import 'elasticdash-test/dist/test-setup.js'
+import { expect } from 'expect'
+
+// Import your workflow from ed_workflows.ts
+import { YOUR_WORKFLOW } from '../ed_workflows'
+
+aiTest('YOUR_TEST_NAME', async (ctx) => {
+  await YOUR_WORKFLOW({ /* your input */ })
+
+  // Assert an LLM step occurred
+  expect(ctx.trace).toHaveLLMStep({ model: 'gpt-4' })
+
+  // Assert a tool was called
+  expect(ctx.trace).toCallTool('YOUR_TOOL_NAME')
+
+  // Semantic output matching (LLM-judged)
+  expect(ctx.trace).toMatchSemanticOutput('expected output description')
+})
+```
+
+### Option B: `defineTest` — CI/CD fixture-based testing
+
+First, record a trace:
+
+```bash
+ELASTICDASH_CAPTURE_TRACE=1 tsx YOUR_WORKFLOW_SCRIPT.ts
+```
+
+Then create `ed_tests.ts`:
+
+```ts
+// ed_tests.ts
+import { defineTest } from 'elasticdash-test'
+import { YOUR_WORKFLOW } from './ed_workflows'
+
+defineTest({
+  name: 'YOUR_TEST_NAME',
+  trace: './.ed_traces/YOUR_TRACE_FILE.json',
+  target: { type: 'tool_call', step_id: 'tool_call_0' },
+  benchmarks: { max_duration_ms: 2000 },
+  run: async () => {
+    await YOUR_WORKFLOW({ /* your input */ })
+  },
+})
+```
+
+Run:
+
+```bash
+npx ed ed-test --no-upload
+```
+
+---
+
+## Step 8: Run and verify
+
+```bash
+# Run aiTest tests
+npx elasticdash test
+
+# Run defineTest benchmarks
+npx ed ed-test --no-upload
+
+# Open the dashboard
+npx elasticdash dashboard
+
+# Record a trace fixture
+ELASTICDASH_CAPTURE_TRACE=1 tsx your-workflow.ts
+```
+
+---
+
+## Decision Trees
+
+### Subprocess mode vs HTTP mode
+
+```
+Does your workflow live inside a framework route handler (Next.js, Remix, SvelteKit)?
+  YES → Use HTTP mode:
+    1. Configure workflow in elasticdash.config.ts with mode: 'http'
+    2. Add initHttpRunContext() to your request handler
+    3. Use wrapTool/wrapAI for observability
+  NO → Use subprocess mode (default):
+    1. Export workflow from ed_workflows.ts
+    2. Tools auto-intercepted via ed_tools.ts
+```
+
+### HTTP mode config template
+
+```ts
+// elasticdash.config.ts
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  workflows: {
+    YOUR_WORKFLOW: {
+      mode: 'http',
+      url: 'http://localhost:3001/api/YOUR_ENDPOINT',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      bodyTemplate: {
+        messages: [{ role: 'user', content: '{{input.message}}' }],
+      },
+      responseFormat: 'vercel-ai-stream',
+    },
+  },
+}
+```
+
+### HTTP mode handler setup
+
+```ts
+// app/api/YOUR_ENDPOINT/route.ts
+import { initHttpRunContext, wrapTool, wrapAI } from 'elasticdash-test'
+
+export async function POST(req: Request) {
+  const runId = req.headers.get('x-elasticdash-run-id')
+  const serverUrl = req.headers.get('x-elasticdash-server')
+  if (runId && serverUrl) {
+    await initHttpRunContext(runId, serverUrl)
+  }
+  // ... rest of handler
+}
+```
+
+### AI call recording
+
+```
+Does your workflow call LLMs via OpenAI/Gemini/Grok SDKs?
+  YES → Automatic interception, no code changes needed
+  NO (custom provider or Anthropic SDK) → Use wrapAI:
+    import { wrapAI } from 'elasticdash-test'
+    export const callLLM = wrapAI('model-name', async (messages) => {
+      return await yourLLMClient.call(messages)
+    })
+```
+
+### Agent setup (`ed_agents.ts`)
+
+```
+Does your project use a multi-step agent with a planner/executor pattern?
+  YES → Create ed_agents.ts:
+    export { plannerAgent, executorAgent } from './your-agent-logic'
+    // OR use SDK reference implementations:
+    export { plannerAgent, executorAgent, resumeAgentFromTrace } from 'elasticdash-test'
+  NO → Skip ed_agents.ts
+```
+
+---
+
+## Troubleshooting
+
+| Error | Cause | Fix |
+|-------|-------|-----|
+| `replay miss: tool_call::YOUR_TOOL` | Trace fixture is stale or workflow changed | Re-record: `ELASTICDASH_CAPTURE_TRACE=1 tsx your-workflow.ts` |
+| `MODULE_NOT_FOUND: elasticdash-test` | SDK not installed or Next.js bundling issue | Run `npm install elasticdash-test`. For Next.js, add to `serverExternalPackages` |
+| `Cannot find module '@/...'` | Path aliases not resolved at runtime | Use advanced dashboard script with `tsconfig-paths/register` |
+| `test has no run function` | `run` field missing in `defineTest` | Add `run: async () => { ... }` to the test definition |
+| `Tool "x" not found in registry` | Tool not exported from `ed_tools.ts` | Export the tool function from `ed_tools.ts` |
+| `ERR_UNKNOWN_FILE_EXTENSION` | ESM/CJS mismatch | Check `package.json` `type` field and `tsconfig.json` `module` setting |
+| Git metadata shows `unknown` | No `.git` directory | Ensure repo is checked out (common in CI with shallow clones) |
+
+---
+
+## Final checklist
+
+After integration, verify these files exist:
+
+```
+your-project/
+  ed_tools.ts           # Instrumented tool wrappers
+  ed_workflows.ts       # Workflow exports
+  elasticdash.config.ts # Test runner config
+  package.json          # dashboard:ai and test:ai scripts added
+  .gitignore            # .temp/ and .ed_traces/ added
+```
+
+Verify with:
+
+```bash
+npx elasticdash test        # Should discover and run *.ai.test.ts files
+npx elasticdash dashboard   # Should open the dashboard UI
+```

package/docs/agents.md

@@ -0,0 +1,140 @@
+# Agent Mid-Trace Replay
+
+ElasticDash supports resuming long-running agents from any task in their plan — without re-executing already-completed steps.
+
+## Use Cases
+
+- **Resuming after failures**: If task 3 of 5 fails, fix the issue and re-run from task 3 only
+- **Pausing for approval**: Capture state after task 2, get human sign-off, then continue
+- **Debugging in isolation**: Re-run a single task with modified input to diagnose a problem
+
+## How It Works
+
+Agents are structured as an **AgentPlan** — an ordered list of **AgentTask** objects. When serialized with captured trace events, this forms an **AgentState** that can be saved and replayed later.
+
+## Quick Start
+
+```ts
+import { plannerAgent, executorAgent, resumeAgentFromTrace } from './ed_agents'
+import { serializeAgentState, deserializeAgentState } from 'elasticdash-test'
+import fs from 'node:fs'
+
+// 1. Generate a plan
+const plan = await plannerAgent('Show me sales for Q1', { userToken: 'tok-abc' })
+
+// 2. Execute the plan (runs all tasks sequentially)
+const completedPlan = await executorAgent(plan)
+
+// 3. Serialize and save state (e.g., after partial execution)
+const state = serializeAgentState(completedPlan, [] /* pass recorder.events in worker context */)
+fs.writeFileSync('agent-state.json', JSON.stringify(state, null, 2))
+
+// 4. Later: load saved state and resume from task 2 (0-based index 1)
+const savedState = JSON.parse(fs.readFileSync('agent-state.json', 'utf8'))
+const stateToResume = deserializeAgentState({ ...savedState, resumeFromTaskIndex: 1 })
+const resumedPlan = await resumeAgentFromTrace(stateToResume)
+
+console.log('Resumed plan status:', resumedPlan.status)
+console.log('Task outputs:', resumedPlan.tasks.map((t) => ({ id: t.id, status: t.status })))
+```
+
+## Data Structures
+
+### AgentState
+
+```ts
+interface AgentState {
+  plan: AgentPlan               // Full plan with all tasks (completed and pending)
+  trace: WorkflowEvent[]        // Captured trace events from previous execution
+  resumeFromTaskIndex: number   // Zero-based index — tasks before this are loaded from cache
+}
+```
+
+### AgentPlan
+
+```ts
+interface AgentPlan {
+  id: string
+  tasks: AgentTask[]
+  status: 'planning' | 'executing' | 'completed' | 'failed' | 'paused'
+  currentTaskIndex: number
+  context: Record<string, unknown>
+  metadata: Record<string, unknown>
+}
+```
+
+### AgentTask
+
+```ts
+interface AgentTask {
+  id: string
+  status: 'pending' | 'in-progress' | 'completed' | 'failed'
+  description: string
+  tool: string          // Name of the tool function to invoke
+  input: unknown        // May contain { $ref: "task-N.output.fieldName" } placeholders
+  output?: unknown      // Populated after execution
+  error?: string
+  startedAt?: number
+  completedAt?: number
+}
+```
+
+## Task Input Placeholders
+
+Task inputs can reference previous task outputs using `{ $ref: "taskId.output.fieldPath" }`:
+
+```ts
+// task-2 uses the embedding produced by task-1
+{
+  id: 'task-2',
+  tool: 'taskSelectorService',
+  input: {
+    queryEmbedding: { $ref: 'task-1.output.embedding' },
+    topK: 3,
+  }
+}
+```
+
+Placeholders are resolved at execution time by `resolveTaskInput()`.
+
+## Dashboard Integration
+
+When running an agent workflow through the dashboard:
+
+1. **Agent task observations** are visually highlighted with a purple background and left border
+2. Each observation shows a **T1 / T2 / T3** badge indicating which task it belongs to
+3. In the observation detail panel, a **"Resume from Task N"** button appears (agent steps only)
+4. Clicking it calls `/api/resume-agent-from-task` with the serialized `AgentState` and chosen `taskIndex`
+5. The resumed run is added as a new trace in the comparison table
+
+## Best Practices
+
+- **Keep tasks idempotent** where possible — if a task must be re-run, ensure it produces the same result
+- **Store minimal outputs** — only record what downstream tasks need, not full API responses
+- **Version your state schema** — if tool interfaces change, old states may need migration
+- **Use sequential tasks** — the current implementation runs tasks one-by-one; parallel task support is planned
+
+## Example: Debugging a Failed Task
+
+```ts
+// 1. Original execution fails at task 3
+const plan = await plannerAgent('Process refund for order-123')
+const result = await executorAgent(plan)
+// Error: task 3 (calculateRefundAmount) failed
+
+// 2. Save the state
+const state = serializeAgentState(result, recorder.events)
+fs.writeFileSync('failed-run.json', JSON.stringify(state))
+
+// 3. Fix the issue in your tool/code
+
+// 4. Resume from task 3 with corrected state
+const savedState = JSON.parse(fs.readFileSync('failed-run.json'))
+const fixed = await resumeAgentFromTrace({
+  ...deserializeAgentState(savedState),
+  resumeFromTaskIndex: 2  // 0-based: task 3 = index 2
+})
+
+// Tasks 1-2 use cached results; task 3+ execute with fixes
+console.log('Fixed plan:', fixed.status)
+```