elasticdash-test 0.1.24 → 0.1.25-alpha-1

package/docs/agent-integration-guide.md

@@ -0,0 +1,554 @@
+# ElasticDash SDK — Agent Integration Guide
+
+> **Audience**: Coding agents (Claude Code, Cursor, Copilot, etc.) integrating `elasticdash-test` into a user's project.
+> **When to use**: Follow this guide when a user asks you to set up ElasticDash testing, add AI workflow observability, or integrate the `elasticdash-test` SDK.
+
+---
+
+## Prerequisites
+
+- Node.js >= 20
+- npm, yarn, or pnpm
+- At least one AI workflow function (a callable async function that makes LLM/tool calls)
+- LLM provider API keys for providers used in workflows (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`)
+
+### Detect project type before starting
+
+Check these before choosing patterns:
+
+| Check | How | Result |
+|-------|-----|--------|
+| Module system | `package.json` → `"type"` field | `"module"` = ESM, `"commonjs"` or missing = CJS |
+| Framework | Look for `next.config.*`, `remix.config.*`, `svelte.config.*` | Present = framework project, use HTTP mode |
+| Path aliases | `tsconfig.json` → `"paths"` field | Present = need advanced dashboard script |
+| Tool architecture | Does the project use a central `dispatchTool(name, args)` function? | Yes = Pattern A, No = Pattern B |
+
+---
+
+## Step 1: Install
+
+```bash
+npm install elasticdash-test
+```
+
+Add to `.gitignore`:
+
+```gitignore
+.temp/
+.ed_traces/
+```
+
+---
+
+## Step 2: Create `ed_tools.ts`
+
+Create `ed_tools.ts` in the project root. This file wraps each tool function with tracing so ElasticDash can record and replay tool calls.
+
+### Choose your pattern
+
+- **Pattern B (recommended for most projects)**: inline async with mock support. Use when tools are imported individually, no central dispatcher exists, or you want dashboard mock support.
+- **Pattern A**: `withTrace` HOF with central dispatcher. Use when tools are pure functions called through a single `dispatchTool(name, args)` dispatcher.
+
+### Pattern B template (recommended)
+
+```ts
+// ed_tools.ts
+// Replace imports with your actual tool functions and source paths
+import { YOUR_TOOL_1 as YOUR_TOOL_1_impl } from './YOUR_SOURCE_PATH_1'
+import { YOUR_TOOL_2 as YOUR_TOOL_2_impl } from './YOUR_SOURCE_PATH_2'
+
+// ---------------------------------------------------------------------------
+// Helpers — copy as-is, no customization needed
+// ---------------------------------------------------------------------------
+
+function resolveMock(toolName: string): { mocked: true; result: unknown } | { mocked: false } {
+  const g = globalThis as any
+  const mocks = g.__ELASTICDASH_TOOL_MOCKS__
+  if (!mocks) return { mocked: false }
+
+  const entry = mocks[toolName]
+  if (!entry || entry.mode === 'live') return { mocked: false }
+
+  if (!g.__ELASTICDASH_TOOL_CALL_COUNTERS__) g.__ELASTICDASH_TOOL_CALL_COUNTERS__ = {}
+  const counters = g.__ELASTICDASH_TOOL_CALL_COUNTERS__
+  counters[toolName] = (counters[toolName] ?? 0) + 1
+  const callNumber = counters[toolName]
+
+  if (entry.mode === 'mock-all') {
+    const data = entry.mockData ?? {}
+    const result = data[callNumber] !== undefined ? data[callNumber] : data[0]
+    return { mocked: true, result }
+  }
+
+  if (entry.mode === 'mock-specific') {
+    const indices = entry.callIndices ?? []
+    if (indices.includes(callNumber)) {
+      return { mocked: true, result: (entry.mockData ?? {})[callNumber] }
+    }
+    return { mocked: false }
+  }
+
+  return { mocked: false }
+}
+
+async function safeRecordToolCall(tool: string, input: any, result: any) {
+  if (!(globalThis as any).__ELASTICDASH_WORKER__) return
+  try {
+    const { recordToolCall } = await import('elasticdash-test')
+    recordToolCall(tool, input, result)
+  } catch { /* tracing must never block business logic */ }
+}
+
+// ---------------------------------------------------------------------------
+// Tools — one export per tool, following this pattern
+// ---------------------------------------------------------------------------
+
+// TEMPLATE: Copy this block for each tool, replacing YOUR_TOOL_NAME and YOUR_TOOL_IMPL
+export const YOUR_TOOL_1 = async (input: any) => {
+  const mock = resolveMock('YOUR_TOOL_1')
+  if (mock.mocked) {
+    await safeRecordToolCall('YOUR_TOOL_1', input, mock.result)
+    return mock.result
+  }
+
+  return await YOUR_TOOL_1_impl(input)
+    .then(async (res: any) => {
+      await safeRecordToolCall('YOUR_TOOL_1', input, res)
+      return res
+    })
+    .catch(async (err: any) => {
+      await safeRecordToolCall('YOUR_TOOL_1', input, err)
+      throw err
+    })
+}
+
+export const YOUR_TOOL_2 = async (input: any) => {
+  const mock = resolveMock('YOUR_TOOL_2')
+  if (mock.mocked) {
+    await safeRecordToolCall('YOUR_TOOL_2', input, mock.result)
+    return mock.result
+  }
+
+  return await YOUR_TOOL_2_impl(input)
+    .then(async (res: any) => {
+      await safeRecordToolCall('YOUR_TOOL_2', input, res)
+      return res
+    })
+    .catch(async (err: any) => {
+      await safeRecordToolCall('YOUR_TOOL_2', input, err)
+      throw err
+    })
+}
+```
+
+### Pattern A template (dispatcher-based)
+
+```ts
+// ed_tools.ts
+// Replace imports with your actual tool functions and source paths
+import {
+  YOUR_TOOL_1 as _YOUR_TOOL_1,
+  YOUR_TOOL_2 as _YOUR_TOOL_2,
+  dispatchTool as _dispatchTool,
+} from './YOUR_TOOLS_SOURCE'
+
+async function withTrace<I, O>(
+  toolName: string,
+  input: I,
+  fn: (input: I) => Promise<O>,
+): Promise<O> {
+  const result = await fn(input)
+  try {
+    const { recordToolCall } = await import('elasticdash-test')
+    recordToolCall(toolName, input, result)
+  } catch { /* tracing must never block business logic */ }
+  return result
+}
+
+export function YOUR_TOOL_1(input: Parameters<typeof _YOUR_TOOL_1>[0]) {
+  return withTrace('YOUR_TOOL_1', input, _YOUR_TOOL_1)
+}
+
+export function YOUR_TOOL_2(input: Parameters<typeof _YOUR_TOOL_2>[0]) {
+  return withTrace('YOUR_TOOL_2', input, _YOUR_TOOL_2)
+}
+
+export async function dispatchTool(name: string, args: Record<string, unknown>) {
+  switch (name) {
+    case 'YOUR_TOOL_1': return YOUR_TOOL_1(args as Parameters<typeof _YOUR_TOOL_1>[0])
+    case 'YOUR_TOOL_2': return YOUR_TOOL_2(args as Parameters<typeof _YOUR_TOOL_2>[0])
+    default: return _dispatchTool(name, args)
+  }
+}
+```
+
+### Alternative: `wrapTool` shorthand
+
+For simpler setups where you don't need mock support or dashboard replays:
+
+```ts
+// ed_tools.ts
+import { wrapTool } from 'elasticdash-test'
+import { YOUR_TOOL_1 as YOUR_TOOL_1_impl } from './YOUR_SOURCE_PATH'
+
+export const YOUR_TOOL_1 = wrapTool('YOUR_TOOL_1', YOUR_TOOL_1_impl)
+```
+
+### Important rules
+
+- The string name passed to `resolveMock()`, `safeRecordToolCall()`, or `wrapTool()` **must match** the exported function name exactly.
+- Each tool function must accept a single input object and return a plain value (JSON-serializable).
+- Tool functions must not close over HTTP context, framework state, or database clients — extract pure logic first.
+
+### Next.js only
+
+Add `elasticdash-test` to `serverExternalPackages` in `next.config.ts`:
+
+```ts
+// next.config.ts
+const nextConfig = {
+  serverExternalPackages: ['elasticdash-test'],
+}
+export default nextConfig
+```
+
+---
+
+## Step 3: Create `ed_workflows.ts`
+
+Create `ed_workflows.ts` in the project root. This file exports workflow functions for the ElasticDash runner.
+
+### Simple case — direct re-export
+
+```ts
+// ed_workflows.ts
+// Replace with your actual workflow function and source path
+export { YOUR_WORKFLOW } from './YOUR_SOURCE_PATH'
+```
+
+### Framework adapter case (Next.js / Remix)
+
+When the workflow lives inside a route handler, create a plain-value wrapper:
+
+```ts
+// ed_workflows.ts
+import { YOUR_HANDLER as _YOUR_HANDLER } from './app/api/YOUR_ROUTE/route'
+
+export async function YOUR_WORKFLOW(input: { message: string; sessionId: string }) {
+  return _YOUR_HANDLER(input)
+}
+```
+
+### Streaming workflow case (Vercel AI SDK)
+
+Create a separate handler file that is only imported by `ed_workflows.ts`:
+
+```ts
+// app/api/chat-stream/chatStreamHandler.ts
+import { NextRequest } from 'next/server'
+import { readVercelAIStream, recordToolCall } from 'elasticdash-test'
+import type { VercelAIStreamResult } from 'elasticdash-test'
+import { POST } from './route'
+
+export async function chatStreamHandler(args: {
+  messages: Array<{ role: string; content: string }>
+  sessionId?: string
+}): Promise<VercelAIStreamResult> {
+  const req = new NextRequest('http://localhost/api/chat-stream', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(args),
+  })
+
+  const response = await POST(req)
+
+  if (response.headers.get('x-vercel-ai-data-stream') !== 'v1') {
+    const errorMessage = await response.text().catch(() => `HTTP ${response.status}`)
+    return { message: errorMessage, type: 'error', error: errorMessage }
+  }
+
+  const result = await readVercelAIStream(response)
+  recordToolCall('chatStream', args, result)
+  return result
+}
+```
+
+Then re-export from `ed_workflows.ts`:
+
+```ts
+// ed_workflows.ts
+export { chatStreamHandler } from './app/api/chat-stream/chatStreamHandler'
+```
+
+### Requirements for all workflow exports
+
+- Accept only JSON-serializable inputs (strings, numbers, arrays, plain objects)
+- Return only JSON-serializable outputs
+- Must not depend on framework runtime APIs, HTTP request context, or live service clients
+- If a dependency is non-serializable (e.g., database client), instantiate it inside `ed_workflows.ts`, not passed as a parameter
+
+---
+
+## Step 4: Update workflow imports
+
+Change your workflow code to import tools from `ed_tools.ts` instead of the original source files:
+
+```ts
+// BEFORE
+import { YOUR_TOOL_1 } from './services/YOUR_SOURCE'
+
+// AFTER
+import { YOUR_TOOL_1 } from './ed_tools'
+```
+
+This single import change makes all tool calls observable by ElasticDash.
+
+---
+
+## Step 5: Add config and scripts
+
+### `elasticdash.config.ts`
+
+Create in project root:
+
+```ts
+// elasticdash.config.ts
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  traceMode: 'local' as const,
+}
+```
+
+### `package.json` scripts
+
+Add these scripts:
+
+```json
+{
+  "scripts": {
+    "dashboard:ai": "elasticdash dashboard",
+    "test:ai": "elasticdash test"
+  }
+}
+```
+
+**If the project uses TypeScript path aliases** (e.g., `@/lib/...` in `tsconfig.json` `paths`):
+
+```json
+{
+  "scripts": {
+    "dashboard:ai": "NODE_OPTIONS='--import tsx/esm --require tsx/cjs --require tsconfig-paths/register' elasticdash dashboard"
+  }
+}
+```
+
+---
+
+## Step 6: Environment variables
+
+Set in `.env` or CI secrets:
+
+| Variable | Description | Required |
+|----------|-------------|----------|
+| `ELASTICDASH_API_URL` | Backend server URL (`https://server.elasticdash.com` for cloud) | For upload/CI |
+| `ELASTICDASH_API_KEY` | Project API key from dashboard | For upload/CI |
+| `ELASTICDASH_CAPTURE_TRACE` | Set to `1` to record a trace fixture | For trace recording |
+| `OPENAI_API_KEY` | OpenAI API key | If using OpenAI |
+| `ANTHROPIC_API_KEY` | Anthropic API key | If using Claude |
+| `GEMINI_API_KEY` | Google Gemini API key | If using Gemini |
+| `GROK_API_KEY` | xAI Grok API key | If using Grok |
+
+---
+
+## Step 7: Write a test
+
+### Option A: `aiTest` — live workflow testing
+
+Create a file ending in `.ai.test.ts`:
+
+```ts
+// tests/YOUR_WORKFLOW.ai.test.ts
+import 'elasticdash-test/dist/test-setup.js'
+import { expect } from 'expect'
+
+// Import your workflow from ed_workflows.ts
+import { YOUR_WORKFLOW } from '../ed_workflows'
+
+aiTest('YOUR_TEST_NAME', async (ctx) => {
+  await YOUR_WORKFLOW({ /* your input */ })
+
+  // Assert an LLM step occurred
+  expect(ctx.trace).toHaveLLMStep({ model: 'gpt-4' })
+
+  // Assert a tool was called
+  expect(ctx.trace).toCallTool('YOUR_TOOL_NAME')
+
+  // Semantic output matching (LLM-judged)
+  expect(ctx.trace).toMatchSemanticOutput('expected output description')
+})
+```
+
+### Option B: `defineTest` — CI/CD fixture-based testing
+
+First, record a trace:
+
+```bash
+ELASTICDASH_CAPTURE_TRACE=1 tsx YOUR_WORKFLOW_SCRIPT.ts
+```
+
+Then create `ed_tests.ts`:
+
+```ts
+// ed_tests.ts
+import { defineTest } from 'elasticdash-test'
+import { YOUR_WORKFLOW } from './ed_workflows'
+
+defineTest({
+  name: 'YOUR_TEST_NAME',
+  trace: './.ed_traces/YOUR_TRACE_FILE.json',
+  target: { type: 'tool_call', step_id: 'tool_call_0' },
+  benchmarks: { max_duration_ms: 2000 },
+  run: async () => {
+    await YOUR_WORKFLOW({ /* your input */ })
+  },
+})
+```
+
+Run:
+
+```bash
+npx ed ed-test --no-upload
+```
+
+---
+
+## Step 8: Run and verify
+
+```bash
+# Run aiTest tests
+npx elasticdash test
+
+# Run defineTest benchmarks
+npx ed ed-test --no-upload
+
+# Open the dashboard
+npx elasticdash dashboard
+
+# Record a trace fixture
+ELASTICDASH_CAPTURE_TRACE=1 tsx your-workflow.ts
+```
+
+---
+
+## Decision Trees
+
+### Subprocess mode vs HTTP mode
+
+```
+Does your workflow live inside a framework route handler (Next.js, Remix, SvelteKit)?
+  YES → Use HTTP mode:
+    1. Configure workflow in elasticdash.config.ts with mode: 'http'
+    2. Add initHttpRunContext() to your request handler
+    3. Use wrapTool/wrapAI for observability
+  NO → Use subprocess mode (default):
+    1. Export workflow from ed_workflows.ts
+    2. Tools auto-intercepted via ed_tools.ts
+```
+
+### HTTP mode config template
+
+```ts
+// elasticdash.config.ts
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  workflows: {
+    YOUR_WORKFLOW: {
+      mode: 'http',
+      url: 'http://localhost:3001/api/YOUR_ENDPOINT',
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      bodyTemplate: {
+        messages: [{ role: 'user', content: '{{input.message}}' }],
+      },
+      responseFormat: 'vercel-ai-stream',
+    },
+  },
+}
+```
+
+### HTTP mode handler setup
+
+```ts
+// app/api/YOUR_ENDPOINT/route.ts
+import { initHttpRunContext, wrapTool, wrapAI } from 'elasticdash-test'
+
+export async function POST(req: Request) {
+  const runId = req.headers.get('x-elasticdash-run-id')
+  const serverUrl = req.headers.get('x-elasticdash-server')
+  if (runId && serverUrl) {
+    await initHttpRunContext(runId, serverUrl)
+  }
+  // ... rest of handler
+}
+```
+
+### AI call recording
+
+```
+Does your workflow call LLMs via OpenAI/Gemini/Grok SDKs?
+  YES → Automatic interception, no code changes needed
+  NO (custom provider or Anthropic SDK) → Use wrapAI:
+    import { wrapAI } from 'elasticdash-test'
+    export const callLLM = wrapAI('model-name', async (messages) => {
+      return await yourLLMClient.call(messages)
+    })
+```
+
+### Agent setup (`ed_agents.ts`)
+
+```
+Does your project use a multi-step agent with a planner/executor pattern?
+  YES → Create ed_agents.ts:
+    export { plannerAgent, executorAgent } from './your-agent-logic'
+    // OR use SDK reference implementations:
+    export { plannerAgent, executorAgent, resumeAgentFromTrace } from 'elasticdash-test'
+  NO → Skip ed_agents.ts
+```
+
+---
+
+## Troubleshooting
+
+| Error | Cause | Fix |
+|-------|-------|-----|
+| `replay miss: tool_call::YOUR_TOOL` | Trace fixture is stale or workflow changed | Re-record: `ELASTICDASH_CAPTURE_TRACE=1 tsx your-workflow.ts` |
+| `MODULE_NOT_FOUND: elasticdash-test` | SDK not installed or Next.js bundling issue | Run `npm install elasticdash-test`. For Next.js, add to `serverExternalPackages` |
+| `Cannot find module '@/...'` | Path aliases not resolved at runtime | Use advanced dashboard script with `tsconfig-paths/register` |
+| `test has no run function` | `run` field missing in `defineTest` | Add `run: async () => { ... }` to the test definition |
+| `Tool "x" not found in registry` | Tool not exported from `ed_tools.ts` | Export the tool function from `ed_tools.ts` |
+| `ERR_UNKNOWN_FILE_EXTENSION` | ESM/CJS mismatch | Check `package.json` `type` field and `tsconfig.json` `module` setting |
+| Git metadata shows `unknown` | No `.git` directory | Ensure repo is checked out (common in CI with shallow clones) |
+
+---
+
+## Final checklist
+
+After integration, verify these files exist:
+
+```
+your-project/
+  ed_tools.ts           # Instrumented tool wrappers
+  ed_workflows.ts       # Workflow exports
+  elasticdash.config.ts # Test runner config
+  package.json          # dashboard:ai and test:ai scripts added
+  .gitignore            # .temp/ and .ed_traces/ added
+```
+
+Verify with:
+
+```bash
+npx elasticdash test        # Should discover and run *.ai.test.ts files
+npx elasticdash dashboard   # Should open the dashboard UI
+```

package/docs/agents.md

@@ -0,0 +1,140 @@
+# Agent Mid-Trace Replay
+
+ElasticDash supports resuming long-running agents from any task in their plan — without re-executing already-completed steps.
+
+## Use Cases
+
+- **Resuming after failures**: If task 3 of 5 fails, fix the issue and re-run from task 3 only
+- **Pausing for approval**: Capture state after task 2, get human sign-off, then continue
+- **Debugging in isolation**: Re-run a single task with modified input to diagnose a problem
+
+## How It Works
+
+Agents are structured as an **AgentPlan** — an ordered list of **AgentTask** objects. When serialized with captured trace events, this forms an **AgentState** that can be saved and replayed later.
+
+## Quick Start
+
+```ts
+import { plannerAgent, executorAgent, resumeAgentFromTrace } from './ed_agents'
+import { serializeAgentState, deserializeAgentState } from 'elasticdash-test'
+import fs from 'node:fs'
+
+// 1. Generate a plan
+const plan = await plannerAgent('Show me sales for Q1', { userToken: 'tok-abc' })
+
+// 2. Execute the plan (runs all tasks sequentially)
+const completedPlan = await executorAgent(plan)
+
+// 3. Serialize and save state (e.g., after partial execution)
+const state = serializeAgentState(completedPlan, [] /* pass recorder.events in worker context */)
+fs.writeFileSync('agent-state.json', JSON.stringify(state, null, 2))
+
+// 4. Later: load saved state and resume from task 2 (0-based index 1)
+const savedState = JSON.parse(fs.readFileSync('agent-state.json', 'utf8'))
+const stateToResume = deserializeAgentState({ ...savedState, resumeFromTaskIndex: 1 })
+const resumedPlan = await resumeAgentFromTrace(stateToResume)
+
+console.log('Resumed plan status:', resumedPlan.status)
+console.log('Task outputs:', resumedPlan.tasks.map((t) => ({ id: t.id, status: t.status })))
+```
+
+## Data Structures
+
+### AgentState
+
+```ts
+interface AgentState {
+  plan: AgentPlan               // Full plan with all tasks (completed and pending)
+  trace: WorkflowEvent[]        // Captured trace events from previous execution
+  resumeFromTaskIndex: number   // Zero-based index — tasks before this are loaded from cache
+}
+```
+
+### AgentPlan
+
+```ts
+interface AgentPlan {
+  id: string
+  tasks: AgentTask[]
+  status: 'planning' | 'executing' | 'completed' | 'failed' | 'paused'
+  currentTaskIndex: number
+  context: Record<string, unknown>
+  metadata: Record<string, unknown>
+}
+```
+
+### AgentTask
+
+```ts
+interface AgentTask {
+  id: string
+  status: 'pending' | 'in-progress' | 'completed' | 'failed'
+  description: string
+  tool: string          // Name of the tool function to invoke
+  input: unknown        // May contain { $ref: "task-N.output.fieldName" } placeholders
+  output?: unknown      // Populated after execution
+  error?: string
+  startedAt?: number
+  completedAt?: number
+}
+```
+
+## Task Input Placeholders
+
+Task inputs can reference previous task outputs using `{ $ref: "taskId.output.fieldPath" }`:
+
+```ts
+// task-2 uses the embedding produced by task-1
+{
+  id: 'task-2',
+  tool: 'taskSelectorService',
+  input: {
+    queryEmbedding: { $ref: 'task-1.output.embedding' },
+    topK: 3,
+  }
+}
+```
+
+Placeholders are resolved at execution time by `resolveTaskInput()`.
+
+## Dashboard Integration
+
+When running an agent workflow through the dashboard:
+
+1. **Agent task observations** are visually highlighted with a purple background and left border
+2. Each observation shows a **T1 / T2 / T3** badge indicating which task it belongs to
+3. In the observation detail panel, a **"Resume from Task N"** button appears (agent steps only)
+4. Clicking it calls `/api/resume-agent-from-task` with the serialized `AgentState` and chosen `taskIndex`
+5. The resumed run is added as a new trace in the comparison table
+
+## Best Practices
+
+- **Keep tasks idempotent** where possible — if a task must be re-run, ensure it produces the same result
+- **Store minimal outputs** — only record what downstream tasks need, not full API responses
+- **Version your state schema** — if tool interfaces change, old states may need migration
+- **Use sequential tasks** — the current implementation runs tasks one-by-one; parallel task support is planned
+
+## Example: Debugging a Failed Task
+
+```ts
+// 1. Original execution fails at task 3
+const plan = await plannerAgent('Process refund for order-123')
+const result = await executorAgent(plan)
+// Error: task 3 (calculateRefundAmount) failed
+
+// 2. Save the state
+const state = serializeAgentState(result, recorder.events)
+fs.writeFileSync('failed-run.json', JSON.stringify(state))
+
+// 3. Fix the issue in your tool/code
+
+// 4. Resume from task 3 with corrected state
+const savedState = JSON.parse(fs.readFileSync('failed-run.json'))
+const fixed = await resumeAgentFromTrace({
+  ...deserializeAgentState(savedState),
+  resumeFromTaskIndex: 2  // 0-based: task 3 = index 2
+})
+
+// Tasks 1-2 use cached results; task 3+ execute with fixes
+console.log('Fixed plan:', fixed.status)
+```