elasticdash-sdk 0.2.0

package/docs/workflow-modes.md

@@ -0,0 +1,253 @@
+# Workflow Modes: Subprocess vs HTTP
+
+ElasticDash supports two workflow execution modes. Choosing the right mode determines whether the dashboard can capture inner tool and AI events from your workflow.
+
+---
+
+## Quick Reference
+
+| | Subprocess mode | HTTP mode |
+|---|---|---|
+| Defined in | `ed_workflows.ts` | `elasticdash.config.ts` |
+| How it runs | Dashboard spawns a worker process, imports and calls your function | Dashboard sends an HTTP request directly to your running dev server |
+| Context mechanism | `CaptureContext` (TraceRecorder) in the worker process | `HttpRunContext` (ALS + global fallback) in the dev server |
+| Inner step visibility | Only if tools run in the same process as the workflow function | Yes — `wrapTool`/`wrapAI` push events back to the dashboard via HTTP |
+| Best for | Pure functions that can run standalone (no framework dependencies) | Framework route handlers (Next.js, Remix, Fastify) where tools run inside the server |
+
+---
+
+## Subprocess Mode (`ed_workflows.ts`)
+
+The dashboard spawns a child process, imports your function from `ed_workflows.ts`, and calls it. The SDK sets up a `CaptureContext` (TraceRecorder/ReplayController) in the worker process to record events.
+
+```ts
+// ed_workflows.ts
+export async function generateAIResponse(input: { message: string }) {
+  // Tools called HERE are captured — they run in the same process
+  const refined = await queryRefinement(input)
+  const result = await callLLM(refined)
+  return result
+}
+```
+
+**When it works well:** The workflow function and all its tool/AI calls run in the same Node.js process. The `CaptureContext` is available to `wrapTool`/`wrapAI` because they share the same `AsyncLocalStorage`.
+
+**When it breaks:** The workflow function calls a remote server via `fetch`. The remote server runs in a different process and has no `CaptureContext`. Inner tool/AI calls on the server are invisible to the SDK.
+
+```ts
+// ed_workflows.ts — THIS LOSES INNER STEPS
+export async function chatHandler(input: { messages: Message[] }) {
+  // This fetch goes to a DIFFERENT process (Next.js dev server)
+  const response = await fetch('http://localhost:3001/api/chat', {
+    method: 'POST',
+    body: JSON.stringify(input),
+  })
+  // Only the top-level result is recorded
+  // Inner wrapTool/wrapAI calls on the server are NOT captured
+  const result = await response.json()
+  recordToolCall('chatHandler', input, result)
+  return result
+}
+```
+
+**Rule of thumb:** If your `ed_workflows.ts` function calls `fetch()` to your own server, you should use HTTP mode instead.
+
+---
+
+## HTTP Mode (`elasticdash.config.ts`)
+
+The dashboard calls your dev server directly via HTTP. It injects special headers (`x-elasticdash-run-id`, `x-elasticdash-server`) that the SDK reads in your route handler to set up `HttpRunContext`. Inner `wrapTool`/`wrapAI` calls push telemetry events back to the dashboard.
+
+**Important:** HTTP mode workflows require entries in **both** files:
+
+1. **`ed_workflows.ts`** — export a function so the workflow appears in the dashboard dropdown
+2. **`elasticdash.config.ts`** — define the same name with `mode: 'http'` so the dashboard uses HTTP mode when running it
+
+The dashboard only discovers workflows from `ed_workflows.ts`. When it runs a workflow, it checks `elasticdash.config.ts` — if the name matches an HTTP config, it uses HTTP mode instead of subprocess. If you only define it in the config, it won't appear in the dropdown.
+
+```ts
+// ed_workflows.ts — stub so the workflow appears in the dashboard
+// The actual HTTP call is handled by elasticdash.config.ts, not this function.
+// This function is only called if the config entry is missing or misconfigured.
+export async function chatHandler(input: { messages: Array<{ role: string; content: string }> }) {
+  // This body is a fallback — the dashboard uses HTTP mode from the config instead
+  return { message: 'This should not run — check elasticdash.config.ts' }
+}
+```
+
+```ts
+// elasticdash.config.ts — defines how the dashboard calls the route
+export default {
+  testMatch: ['**/*.ai.test.ts'],
+  workflows: {
+    chatHandler: {                               // ← same name as ed_workflows.ts export
+      mode: 'http' as const,
+      url: 'http://localhost:3001/api/chat',
+      method: 'POST' as const,
+      headers: { 'Content-Type': 'application/json' },
+      bodyTemplate: {
+        messages: '{{input.messages}}',
+        sessionId: '{{input.sessionId}}',
+      },
+      responseFormat: 'json' as const,           // or 'vercel-ai-stream'
+    },
+  },
+}
+```
+
+**How it works:**
+
+1. Dashboard generates a unique `runId` and registers it
+2. Dashboard sends your request with `x-elasticdash-run-id` and `x-elasticdash-server` headers
+3. Your route handler reads these headers and calls `initHttpRunContext(runId, server)` or `runWithInitializedHttpContext(runId, server, callback)`
+4. Every `wrapTool`/`wrapAI` call in your route pushes a telemetry event back to the dashboard
+5. Dashboard collects all events and displays them as observations
+
+**Route handler setup (Next.js example):**
+
+```ts
+// app/api/chat/route.ts
+import { NextRequest } from 'next/server'
+
+export async function POST(request: NextRequest) {
+  const edRunId = request.headers.get('x-elasticdash-run-id')
+  const edServer = request.headers.get('x-elasticdash-server')
+
+  if (edRunId && edServer) {
+    // For non-streaming routes:
+    const { initHttpRunContext } = require('elasticdash-sdk')
+    await initHttpRunContext(edRunId, edServer)
+
+    // For streaming routes (preferred — context propagates through als.run):
+    const { runWithInitializedHttpContext } = require('elasticdash-sdk')
+    return runWithInitializedHttpContext(edRunId, edServer, async () => {
+      // All wrapTool/wrapAI calls inside here are captured
+      return doWork()
+    })
+  }
+
+  return doWork()
+}
+```
+
+**Response formats:**
+
+| Format | Config value | Use when |
+|---|---|---|
+| JSON | `responseFormat: 'json'` | Route returns `NextResponse.json(...)` |
+| Vercel AI stream | `responseFormat: 'vercel-ai-stream'` | Route returns `new Response(stream)` with Vercel AI SDK wire protocol |
+
+---
+
+## Common Mistake: Subprocess Wrapper Around an HTTP Call
+
+A frequent anti-pattern is defining a subprocess workflow that calls your own server:
+
+```ts
+// ed_workflows.ts — WRONG: subprocess calling your own server
+export async function chatHandler(input: any) {
+  const response = await fetch('http://localhost:3001/api/chat', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(input),
+  })
+  const result = await response.json()
+  recordToolCall('chatHandler', input, result)
+  return result
+}
+```
+
+This records the top-level result but loses all inner steps. The subprocess worker and the Next.js server are different processes — the `CaptureContext` in the worker doesn't reach `wrapTool`/`wrapAI` calls in the server.
+
+**Fix:** Define the workflow in `elasticdash.config.ts` as HTTP mode, and simplify `ed_workflows.ts` to a stub:
+
+```ts
+// elasticdash.config.ts — add HTTP mode config
+export default {
+  workflows: {
+    chatHandler: {
+      mode: 'http' as const,
+      url: 'http://localhost:3001/api/chat',
+      method: 'POST' as const,
+      headers: { 'Content-Type': 'application/json' },
+      bodyTemplate: {
+        messages: '{{input.messages}}',
+      },
+      responseFormat: 'json' as const,
+    },
+  },
+}
+```
+
+```ts
+// ed_workflows.ts — keep the export so it appears in the dashboard dropdown,
+// but remove the fetch logic (the config handles it now)
+export async function chatHandler(input: { messages: Array<{ role: string; content: string }> }) {
+  return { message: 'Fallback — check elasticdash.config.ts' }
+}
+```
+
+The dashboard's HTTP mode injects ED headers automatically, your route sets up context, and inner steps flow back to the dashboard.
+
+---
+
+## When to Use Each Mode
+
+| Scenario | Mode |
+|---|---|
+| Workflow function runs tools directly (same process) | Subprocess |
+| Workflow calls your own server via HTTP | HTTP |
+| Next.js / Remix / Fastify route handlers | HTTP |
+| Streaming responses (Vercel AI SDK, SSE) | HTTP with `responseFormat: 'vercel-ai-stream'` |
+| Pure AI pipeline with no server (script-based) | Subprocess |
+| `.ai.test.ts` files (programmatic tests) | Subprocess (via test runner) |
+
+---
+
+## Streaming Routes and Context Propagation
+
+For streaming routes that return a `ReadableStream`, use `runWithInitializedHttpContext` inside the stream's `start()` callback to ensure ALS context propagates through all async work:
+
+```ts
+// Recommended pattern for streaming routes
+const stream = new ReadableStream({
+  async start(controller) {
+    if (edRunId && edServer) {
+      const { runWithInitializedHttpContext } = require('elasticdash-sdk')
+      await runWithInitializedHttpContext(edRunId, edServer, async () => {
+        // All wrapTool/wrapAI calls here are captured
+        const result = await myWrappedTool(input)
+        controller.enqueue(encoder.encode(JSON.stringify(result)))
+        controller.close()
+      })
+    }
+  },
+})
+return new Response(stream, { headers: { ... } })
+```
+
+If `runWithInitializedHttpContext` is called outside `start()` (e.g., in the handler scope), the SDK's global fallback ensures context is still available inside `start()`. However, wrapping inside `start()` is the more reliable pattern.
+
+---
+
+## Importing the SDK in Next.js / Turbopack
+
+Turbopack statically analyzes `import()` and `require()` calls, which causes `Module not found` errors for `serverExternalPackages`. Use `eval('require')` or `createRequire` to bypass this:
+
+```ts
+// Option 1: eval('require') — simple, works everywhere
+const { wrapTool } = (eval('require') as (id: string) => any)('elasticdash-sdk')
+
+// Option 2: createRequire — cleaner, no eval
+import { createRequire } from 'node:module'
+const nodeRequire = createRequire(process.cwd() + '/')
+const { wrapTool } = nodeRequire('elasticdash-sdk')
+```
+
+Also add the SDK to `serverExternalPackages` in `next.config.js`:
+
+```js
+module.exports = {
+  serverExternalPackages: ['elasticdash-sdk'],
+}
+```

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "elasticdash-sdk",
+  "version": "0.2.0",
+  "description": "AI-native SDK for ElasticDash workflow testing, tracing, and observability",
+  "type": "module",
+  "bin": {
+    "elasticdash": "./dist/cli.js",
+    "ed": "./dist/cli.js"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.cjs",
+      "default": "./dist/index.js"
+    },
+    "./http": {
+      "types": "./dist/http.d.ts",
+      "default": "./dist/http.js"
+    },
+    "./observability": {
+      "types": "./dist/observability.d.ts",
+      "default": "./dist/observability.js"
+    },
+    "./portal": {
+      "types": "./dist/portal-server.d.ts",
+      "default": "./dist/portal-server.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src",
+    "docs",
+    "package.json",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "node scripts/inline-html.js && tsc && npm run restore:inline-html && node -e \"const{cpSync,mkdirSync}=require('fs');mkdirSync('dist/html',{recursive:true});cpSync('src/html/dashboard.html','dist/html/dashboard.html');\" && node_modules/.bin/esbuild src/index.ts --bundle --platform=node --format=cjs --outfile=dist/index.cjs --packages=external --target=node18 && npm pack",
+    "restore:inline-html": "node scripts/restore-inline-html.js",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit",
+    "dashboard": "node dist/cli.js dashboard",
+    "release": "npm run build && npm publish --tag beta",
+    "release:prod": "npm run build && npm publish"
+  },
+  "dependencies": {
+    "chalk": "^5.3.0",
+    "chokidar": "^5.0.0",
+    "commander": "^12.1.0",
+    "dotenv": "^17.3.1",
+    "esbuild": "^0.27.3",
+    "expect": "^29.7.0",
+    "express": "^5.2.1",
+    "fast-glob": "^3.3.2",
+    "socket.io-client": "^4.8.3",
+    "tsx": "^4.15.6"
+  },
+  "devDependencies": {
+    "@types/express": "^5.0.6",
+    "@types/node": "^20.14.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.4"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  },
+  "author": "ElasticDash <contact@elasticdash.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/ElasticDash/elasticdash-test-js.git"
+  }
+}

package/src/browser-ui.ts

@@ -0,0 +1,281 @@
+import http from 'node:http'
+import path from 'node:path'
+import os from 'node:os'
+import { fileURLToPath } from 'node:url'
+import { spawn } from 'node:child_process'
+
+export type UiEvent =
+  | { type: 'run-start'; payload: { files: string[] } }
+  | { type: 'test-start'; payload: { name: string } }
+  | { type: 'test-finish'; payload: { name: string; passed: boolean; durationMs: number; errorMessage?: string } }
+  | { type: 'run-summary'; payload: { passed: number; failed: number; total: number; durationMs: number; failures: Array<{ name: string; errorMessage?: string }> } }
+
+export interface BrowserUiServer {
+  url: string
+  send(event: UiEvent): void
+  close(): void
+}
+
+interface BrowserUiOptions {
+  port?: number
+  autoOpen?: boolean
+}
+
+const defaultHtml = `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <title>ElasticDash SDK Test Runner</title>
+  <style>
+    :root { font-family: Inter, -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: #0b1021; color: #e8ecf7; }
+    body { margin: 0; padding: 24px; }
+    h1 { margin: 0 0 16px; font-size: 20px; }
+    .summary { display: flex; gap: 12px; margin-bottom: 16px; }
+    .pill { padding: 8px 12px; border-radius: 999px; font-weight: 600; }
+    .pass { background: #12351b; color: #8de0a3; }
+    .fail { background: #351212; color: #f59b9b; }
+    .total { background: #1c2745; color: #cdd7ff; }
+    .tests { margin-top: 12px; }
+    .test { border: 1px solid #1f2a4f; border-radius: 8px; padding: 12px; margin-bottom: 8px; background: #0f1731; }
+    .name { font-weight: 600; }
+    .error { margin-top: 8px; white-space: pre-wrap; color: #f59b9b; font-family: ui-monospace, SFMono-Regular, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', 'Courier New', monospace; }
+    .status { font-weight: 600; }
+  </style>
+</head>
+<body>
+  <h1>ElasticDash SDK Test Runner</h1>
+  <div class="summary">
+    <div class="pill total" id="total">Total: -</div>
+    <div class="pill pass" id="passed">Passed: -</div>
+    <div class="pill fail" id="failed">Failed: -</div>
+  </div>
+  <div id="progress">Waiting for test run...</div>
+  <div class="tests" id="tests"></div>
+  <script>
+    const totalEl = document.getElementById('total');
+    const passedEl = document.getElementById('passed');
+    const failedEl = document.getElementById('failed');
+    const progressEl = document.getElementById('progress');
+    const testsEl = document.getElementById('tests');
+
+    const tests = new Map();
+    let processed = 0;
+    let passedCount = 0;
+    let failedCount = 0;
+    let finalTotal = null;
+
+    function setError(el, message) {
+      const errEl = el.querySelector('.error');
+      let toggle = el.querySelector('.toggle');
+      if (!message) {
+        errEl.textContent = '';
+        errEl.style.display = 'none';
+        if (toggle) toggle.style.display = 'none';
+        return;
+      }
+      if (!toggle) {
+        toggle = document.createElement('button');
+        toggle.className = 'toggle';
+        toggle.textContent = 'Show details';
+        toggle.style.marginTop = '8px';
+        toggle.style.background = '#1c2745';
+        toggle.style.color = '#cdd7ff';
+        toggle.style.border = '1px solid #2a3866';
+        toggle.style.borderRadius = '6px';
+        toggle.style.padding = '6px 10px';
+        toggle.style.cursor = 'pointer';
+        toggle.style.fontWeight = '600';
+        el.appendChild(toggle);
+        toggle.addEventListener('click', () => {
+          const isHidden = errEl.style.display === 'none';
+          errEl.style.display = isHidden ? 'block' : 'none';
+          toggle.textContent = isHidden ? 'Hide details' : 'Show details';
+        });
+      }
+      errEl.textContent = message;
+      errEl.style.display = 'none';
+      toggle.style.display = 'inline-block';
+      toggle.textContent = 'Show details';
+    }
+
+    function renderTest(name) {
+      let el = tests.get(name);
+      if (!el) {
+        el = document.createElement('div');
+        el.className = 'test';
+        el.innerHTML = '<div class="name"></div><div class="status"></div><div class="error"></div>';
+        tests.set(name, el);
+        testsEl.appendChild(el);
+      }
+      return el;
+    }
+
+    function updatePills(passed, failed, total) {
+      totalEl.textContent = 'Total: ' + total;
+      passedEl.textContent = 'Passed: ' + passed;
+      failedEl.textContent = 'Failed: ' + failed;
+    }
+
+    const evtSource = new EventSource('/events');
+    evtSource.onmessage = (ev) => {
+      try {
+        const msg = JSON.parse(ev.data);
+        if (msg.type === 'test-start') {
+          const el = renderTest(msg.payload.name);
+          el.querySelector('.name').textContent = msg.payload.name;
+          el.querySelector('.status').textContent = 'Running...';
+          el.querySelector('.status').style.color = '#cdd7ff';
+          el.querySelector('.error').textContent = '';
+          progressEl.textContent = 'Running tests...';
+        }
+        if (msg.type === 'test-finish') {
+          const el = renderTest(msg.payload.name);
+          el.querySelector('.name').textContent = msg.payload.name;
+          el.querySelector('.status').textContent = msg.payload.passed ? 'Passed' : 'Failed';
+          el.querySelector('.status').style.color = msg.payload.passed ? '#8de0a3' : '#f59b9b';
+          setError(el, msg.payload.errorMessage || '');
+
+          // live tally
+          processed += 1;
+          if (msg.payload.passed) passedCount += 1;
+          else failedCount += 1;
+          const displayTotal = finalTotal !== null ? finalTotal : processed;
+          updatePills(passedCount, failedCount, displayTotal);
+        }
+        if (msg.type === 'run-summary') {
+          finalTotal = msg.payload.total;
+          passedCount = msg.payload.passed;
+          failedCount = msg.payload.failed;
+          processed = msg.payload.total;
+          updatePills(msg.payload.passed, msg.payload.failed, msg.payload.total);
+          progressEl.textContent = 'Finished';
+          msg.payload.failures.forEach(function (f) {
+            const el = renderTest(f.name);
+            el.querySelector('.name').textContent = f.name;
+            el.querySelector('.status').textContent = 'Failed';
+            el.querySelector('.status').style.color = '#f59b9b';
+            setError(el, f.errorMessage || '');
+          });
+        }
+      } catch (e) {
+        console.error('Bad event data', e);
+      }
+    };
+  </script>
+</body>
+</html>`
+
+export async function startBrowserUiServer(opts: BrowserUiOptions = {}): Promise<BrowserUiServer | undefined> {
+  const autoOpen = opts.autoOpen !== false
+  let port = opts.port ?? 4571
+
+  // // Ensure base dir for potential static assets (none now)
+  // const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+  type FlushableResponse = http.ServerResponse & { flush?: () => void; flushHeaders?: () => void }
+  const clients: FlushableResponse[] = []
+  const eventBuffer: UiEvent[] = []
+
+  const handler: http.RequestListener = (req, res) => {
+    if (!req.url) return res.end()
+    if (req.url.startsWith('/events')) {
+      const sseRes = res as FlushableResponse
+      sseRes.writeHead(200, {
+        'Content-Type': 'text/event-stream',
+        Connection: 'keep-alive',
+        'Cache-Control': 'no-cache',
+      })
+      // Prime the connection so browsers render immediately
+      sseRes.flushHeaders?.()
+      sseRes.write(': connected\n\n')
+      sseRes.flush?.()
+      // Replay all previously sent events so late-connecting browsers get the full history
+      for (const e of eventBuffer) {
+        sseRes.write(`data: ${JSON.stringify(e)}\n\n`)
+      }
+      sseRes.flush?.()
+      clients.push(sseRes)
+      req.on('close', () => {
+        const idx = clients.indexOf(sseRes)
+        if (idx >= 0) clients.splice(idx, 1)
+      })
+      return
+    }
+
+    // Serve inline HTML
+    res.writeHead(200, { 'Content-Type': 'text/html' })
+    res.end(defaultHtml)
+  }
+
+  let server: http.Server | undefined
+  let heartbeat: ReturnType<typeof setInterval> | undefined
+  let started = false
+
+  while (!started) {
+    try {
+      server = http.createServer(handler)
+      await new Promise<void>((resolve, reject) => {
+        server!.once('error', reject)
+        server!.listen(port, resolve)
+      })
+      started = true
+    } catch (err) {
+      port += 1
+      if (port > (opts.port ?? 4571) + 10) {
+        console.error('[elasticdash] Browser UI server failed to start:', (err as Error).message)
+        return undefined
+      }
+    }
+  }
+
+  const url = `http://localhost:${port}`
+
+  function send(event: UiEvent): void {
+    eventBuffer.push(event)
+    const payload = `data: ${JSON.stringify(event)}\n\n`
+    for (const client of [...clients]) {
+      client.write(payload)
+      client.flush?.()
+    }
+  }
+
+  function close(): void {
+    if (heartbeat) {
+      clearInterval(heartbeat)
+      heartbeat = undefined
+    }
+    for (const client of clients) {
+      client.end()
+    }
+    clients.length = 0
+    server?.close()
+  }
+
+  if (autoOpen) {
+    openBrowser(url)
+  }
+
+  // Periodic keepalive comments to keep EventSource connections from timing out
+  heartbeat = setInterval(() => {
+    for (const client of [...clients]) {
+      client.write(': keepalive\n\n')
+      client.flush?.()
+    }
+  }, 5000)
+
+  return { url, send, close }
+}
+
+function openBrowser(url: string): void {
+  const platform = os.platform()
+  const command =
+    platform === 'darwin'
+      ? 'open'
+      : platform === 'win32'
+      ? 'cmd'
+      : 'xdg-open'
+
+  const args = platform === 'win32' ? ['/c', 'start', '""', url] : [url]
+  const child = spawn(command, args, { stdio: 'ignore', detached: true })
+  child.unref()
+}

package/src/capture/event.ts

@@ -0,0 +1,30 @@
+export type WorkflowEventType = 'ai' | 'tool' | 'http' | 'db' | 'side_effect' | 'workflow'
+
+export interface WorkflowEvent {
+  id: number
+  type: WorkflowEventType
+  name: string
+  input: unknown
+  output: unknown
+  timestamp: number
+  durationMs: number
+  /** Token usage for LLM (ai) events */
+  usage?: { inputTokens?: number; outputTokens?: number; totalTokens?: number }
+  /** Optional: ID of the agent task that produced this event */
+  agentTaskId?: string
+  /** Optional: Zero-based index of the agent task that produced this event */
+  agentTaskIndex?: number
+  /** Set to true when the original response / output was a stream */
+  streamed?: boolean
+  /** Raw buffered text of a streamed response (used for replay) */
+  streamRaw?: string
+  /** Schema version for forward compatibility (default 1) */
+  schemaVersion?: number
+  /** Optional request-level trace ID for grouping events within a session */
+  traceId?: string
+}
+
+export interface WorkflowTrace {
+  traceId: string
+  events: WorkflowEvent[]
+}

package/src/capture/index.ts

@@ -0,0 +1,3 @@
+export * from './event.js'
+export * from './recorder.js'
+export * from './replay.js'