elasticdash-sdk 0.2.0

package/src/proxy/llm-capture.ts

@@ -0,0 +1,301 @@
+import http from 'node:http'
+import { URL } from 'node:url'
+import { Readable } from 'node:stream'
+import type { LLMStep } from '../trace-adapter/context.js'
+
+const DEFAULT_PORT = 8787
+const HEADER_TRACE_ID = 'x-trace-id'
+
+type Provider = 'openai' | 'gemini' | 'grok' | 'anthropic'
+
+const DEFAULT_UPSTREAM: Record<Provider, string> = {
+  openai: 'https://api.openai.com',
+  gemini: 'https://generativelanguage.googleapis.com',
+  grok: 'https://api.x.ai',
+  anthropic: 'https://api.anthropic.com',
+}
+
+const AI_PATTERNS: Record<Provider, RegExp> = {
+  openai: /\/v1\/(chat\/)?completions/, // also covers legacy /v1/completions
+  gemini: /\/v1beta\/models\/[^/:]+:(generateContent|streamGenerateContent)/,
+  grok: /\/v1\/(chat\/)?completions/,
+  anthropic: /\/v1\/messages/,
+}
+
+function detectProvider(pathname: string): Provider | null {
+  for (const [provider, pattern] of Object.entries(AI_PATTERNS) as Array<[Provider, RegExp]>) {
+    if (pattern.test(pathname)) return provider
+  }
+  return null
+}
+
+function extractModel(provider: Provider, body: Record<string, unknown>, url: string): string {
+  if (provider === 'gemini') {
+    const match = /\/models\/([^/:]+):/.exec(url)
+    return match ? match[1] : 'unknown'
+  }
+  if (provider === 'anthropic') {
+    return typeof body.model === 'string' ? body.model : 'unknown'
+  }
+  return typeof body.model === 'string' ? body.model : 'unknown'
+}
+
+function extractPrompt(provider: Provider, body: Record<string, unknown>): string {
+  if (provider === 'openai' || provider === 'grok') {
+    const messages = body.messages
+    if (Array.isArray(messages)) {
+      return messages
+        .map((m: unknown) => {
+          if (m && typeof m === 'object') {
+            const msg = m as Record<string, unknown>
+            return `${msg.role}: ${msg.content}`
+          }
+          return String(m)
+        })
+        .join('\n')
+    }
+    return typeof body.prompt === 'string' ? body.prompt : ''
+  }
+
+  if (provider === 'gemini') {
+    const contents = body.contents
+    if (Array.isArray(contents)) {
+      return contents
+        .flatMap((c: unknown) => {
+          if (c && typeof c === 'object') {
+            const parts = (c as Record<string, unknown>).parts
+            if (Array.isArray(parts)) {
+              return parts.map((p: unknown) => {
+                if (p && typeof p === 'object') {
+                  return String((p as Record<string, unknown>).text ?? '')
+                }
+                return ''
+              })
+            }
+          }
+          return []
+        })
+        .join('\n')
+    }
+  }
+
+  if (provider === 'anthropic') {
+    const messages = body.messages
+    if (Array.isArray(messages)) {
+      return messages
+        .map((m: unknown) => {
+          if (m && typeof m === 'object') {
+            const msg = m as Record<string, unknown>
+            return `${msg.role ?? 'user'}: ${msg.content ?? ''}`
+          }
+          return String(m)
+        })
+        .join('\n')
+    }
+  }
+
+  return ''
+}
+
+function extractCompletion(provider: Provider, responseBody: Record<string, unknown>): string {
+  if (provider === 'openai' || provider === 'grok') {
+    const choices = responseBody.choices
+    if (Array.isArray(choices) && choices.length > 0) {
+      const first = choices[0] as Record<string, unknown>
+      if (first.message && typeof first.message === 'object') {
+        return String((first.message as Record<string, unknown>).content ?? '')
+      }
+      if (typeof first.text === 'string') return first.text
+    }
+  }
+
+  if (provider === 'gemini') {
+    const candidates = responseBody.candidates
+    if (Array.isArray(candidates) && candidates.length > 0) {
+      const first = candidates[0] as Record<string, unknown>
+      if (first.content && typeof first.content === 'object') {
+        const parts = (first.content as Record<string, unknown>).parts
+        if (Array.isArray(parts) && parts.length > 0) {
+          return String((parts[0] as Record<string, unknown>).text ?? '')
+        }
+      }
+    }
+  }
+
+  if (provider === 'anthropic') {
+    const content = responseBody.content
+    if (Array.isArray(content) && content.length > 0) {
+      const first = content[0] as Record<string, unknown>
+      if (typeof first.text === 'string') return first.text
+      if (first.type === 'text' && typeof first.text === 'string') return first.text
+    }
+  }
+
+  return ''
+}
+
+function cloneHeaders(headers: http.IncomingHttpHeaders): Record<string, string> {
+  const result: Record<string, string> = {}
+  for (const [key, value] of Object.entries(headers)) {
+    if (Array.isArray(value)) {
+      result[key] = value.join(', ')
+    } else if (typeof value === 'string') {
+      result[key] = value
+    }
+  }
+  return result
+}
+
+function normalizeUpstream(provider: Provider, userBase?: string): string {
+  const base = userBase || DEFAULT_UPSTREAM[provider]
+  return base.endsWith('/') ? base.slice(0, -1) : base
+}
+
+type Store = Map<string, LLMStep[]>
+
+function recordStep(store: Store, traceId: string, step: LLMStep): void {
+  if (!store.has(traceId)) {
+    store.set(traceId, [])
+  }
+  store.get(traceId)!.push(step)
+}
+
+async function readBody(req: http.IncomingMessage): Promise<Buffer> {
+  const chunks: Buffer[] = []
+  for await (const chunk of req) {
+    chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk)
+  }
+  return Buffer.concat(chunks)
+}
+
+function sendUpstreamResponse(upstreamRes: Response, res: http.ServerResponse): void {
+  res.statusCode = upstreamRes.status
+  for (const [key, value] of upstreamRes.headers.entries()) {
+    res.setHeader(key, value)
+  }
+  const body = upstreamRes.body
+  if (!body) {
+    res.end()
+    return
+  }
+  const nodeStream = Readable.fromWeb(body as unknown as ReadableStream)
+  nodeStream.pipe(res)
+}
+
+export interface StartedProxy {
+  url: string
+  stop(): Promise<void>
+}
+
+export interface ProxyOptions {
+  port?: number
+  upstream?: Partial<Record<Provider, string>>
+}
+
+export async function startLLMProxy(options: ProxyOptions = {}): Promise<StartedProxy> {
+  const port = options.port ?? DEFAULT_PORT
+  const store: Store = new Map()
+  const upstreamOverride = options.upstream || {}
+
+  const server = http.createServer(async (req, res) => {
+    try {
+      if (!req.url || !req.method) {
+        res.statusCode = 400
+        res.end('Bad request')
+        return
+      }
+
+      const parsed = new URL(req.url, `http://localhost:${port}`)
+
+      if (req.method === 'GET' && parsed.pathname.startsWith('/traces/')) {
+        const traceId = decodeURIComponent(parsed.pathname.replace('/traces/', ''))
+        const steps = store.get(traceId) ?? []
+        store.delete(traceId)
+        res.setHeader('content-type', 'application/json')
+        res.end(JSON.stringify({ steps }))
+        return
+      }
+
+      if (req.method === 'GET' && parsed.pathname === '/health') {
+        res.statusCode = 200
+        res.end('ok')
+        return
+      }
+
+      const bodyBuf = await readBody(req)
+      const bodyText = bodyBuf.toString() || '{}'
+      const requestBody = (() => {
+        try {
+          return JSON.parse(bodyText) as Record<string, unknown>
+        } catch {
+          return {} as Record<string, unknown>
+        }
+      })()
+
+      const provider = detectProvider(parsed.pathname)
+      const traceId = (req.headers[HEADER_TRACE_ID] as string | undefined)?.toString()
+      const isStreaming = requestBody && typeof requestBody === 'object' ? (requestBody as any).stream === true : false
+      const headers = cloneHeaders(req.headers)
+
+      const upstreamBase = provider ? normalizeUpstream(provider, upstreamOverride[provider]) : undefined
+      if (!provider || !upstreamBase) {
+        // Fallback passthrough without capture
+        const passthrough = await fetch(parsed.toString(), {
+          method: req.method,
+          headers,
+          body: bodyBuf.length > 0 ? bodyBuf : undefined,
+        })
+        sendUpstreamResponse(passthrough, res)
+        return
+      }
+
+      const targetUrl = `${upstreamBase}${parsed.pathname}${parsed.search}`
+      const upstreamRes = await fetch(targetUrl, {
+        method: req.method,
+        headers,
+        body: bodyBuf.length > 0 ? bodyBuf : undefined,
+      })
+
+      if (traceId) {
+        const model = extractModel(provider, requestBody, targetUrl)
+        const prompt = extractPrompt(provider, requestBody)
+        if (isStreaming) {
+          recordStep(store, traceId, { model, provider, prompt, completion: '(streamed)' })
+        } else {
+          try {
+            const clone = upstreamRes.clone()
+            const responseBody = (await clone.json()) as Record<string, unknown>
+            const completion = extractCompletion(provider, responseBody)
+            recordStep(store, traceId, { model, provider, prompt, completion })
+          } catch {
+            recordStep(store, traceId, { model, provider, prompt, completion: '' })
+          }
+        }
+      }
+
+      sendUpstreamResponse(upstreamRes, res)
+    } catch (err) {
+      res.statusCode = 500
+      res.end(`proxy error: ${(err as Error).message}`)
+    }
+  })
+
+  await new Promise<void>((resolve) => server.listen(port, resolve))
+
+  return {
+    url: `http://localhost:${port}`,
+    async stop() {
+      await new Promise<void>((resolve) => server.close(() => resolve()))
+    },
+  }
+}
+
+export async function fetchCapturedTrace(proxyUrl: string, traceId: string): Promise<LLMStep[]> {
+  const url = `${proxyUrl.replace(/\/$/, '')}/traces/${encodeURIComponent(traceId)}`
+  const res = await fetch(url)
+  if (!res.ok) {
+    throw new Error(`failed to fetch trace ${traceId} from proxy: ${res.status}`)
+  }
+  const data = (await res.json()) as { steps?: LLMStep[] }
+  return data.steps || []
+}

package/src/reporter.ts

@@ -0,0 +1,81 @@
+import chalk from 'chalk'
+import type { TestResult, FileResult } from './runner.js'
+
+export function reportResults(fileResults: FileResult[]): void {
+  let totalPassed = 0
+  let totalFailed = 0
+  let totalDurationMs = 0
+
+  for (const fileResult of fileResults) {
+    if (fileResults.length > 1) {
+      console.log(chalk.dim(`\n${fileResult.file}`))
+    }
+
+    for (const result of fileResult.results) {
+      printTestResult(result)
+      totalDurationMs += result.durationMs
+      if (result.passed) {
+        totalPassed++
+      } else {
+        totalFailed++
+      }
+    }
+  }
+
+  printSummary(totalPassed, totalFailed, totalDurationMs)
+}
+
+function printTestResult(result: TestResult): void {
+  const duration = chalk.dim(`(${formatDuration(result.durationMs)})`)
+
+  if (result.passed) {
+    console.log(`  ${chalk.green('✓')} ${result.name} ${duration}`)
+  } else {
+    console.log(`  ${chalk.red('✗')} ${result.name} ${duration}`)
+    if (result.error) {
+      const errorLines = formatError(result.error)
+      for (const line of errorLines) {
+        console.log(`    ${chalk.red('→')} ${line}`)
+      }
+    }
+  }
+}
+
+function printSummary(passed: number, failed: number, totalMs: number): void {
+  const total = passed + failed
+  console.log('')
+
+  if (passed > 0) {
+    console.log(chalk.green(`${passed} passed`))
+  }
+  if (failed > 0) {
+    console.log(chalk.red(`${failed} failed`))
+  }
+
+  console.log(chalk.dim(`Total: ${total}`))
+  console.log(chalk.dim(`Duration: ${formatDuration(totalMs)}`))
+}
+
+function formatDuration(ms: number): string {
+  if (ms >= 1000) {
+    return `${(ms / 1000).toFixed(1)}s`
+  }
+  return `${ms}ms`
+}
+
+function formatError(error: Error): string[] {
+  const lines: string[] = []
+  if (error.message) {
+    lines.push(error.message)
+  }
+  if (error.stack) {
+    const stackLines = error.stack
+      .split('\n')
+      .slice(1)
+      .map((l) => l.trim())
+      .filter((l) => l.startsWith('at '))
+      .slice(0, 3)
+    lines.push(...stackLines)
+  }
+  return lines
+}

package/src/runWorkflowSubprocess.ts

@@ -0,0 +1,74 @@
+import { spawn } from 'child_process';
+import { join } from 'path';
+import { Readable } from 'stream';
+
+/**
+ * Run workflow logic in a subprocess using the workflow-runner-worker (fd3 IPC).
+ * This utility is designed for use in Next.js API routes or any parent process.
+ *
+ * @param {object} input - The workflow input (requestBody, userToken, testCaseId, testCaseRunRecordId, etc)
+ * @returns {Promise<any>} - Resolves with the result from the worker, or throws on error.
+ *
+ * Usage (in your Next.js API route):
+ *   import { runWorkflowInSubprocess } from './src/runWorkflowSubprocess';
+ *   const result = await runWorkflowInSubprocess({ ... });
+ *   return NextResponse.json(result);
+ */
+export async function runWorkflowInSubprocess(input: Record<string, any>): Promise<any> {
+  return new Promise((resolve, reject) => {
+    // Path to the workflow worker entry point
+    const workerPath = join(process.cwd(), 'src', 'workflow-runner-worker.ts');
+
+    // Spawn the worker as a subprocess with fd3 pipe
+    const child = spawn(
+      process.execPath,
+      [workerPath],
+      {
+        stdio: ['pipe', 'pipe', 'pipe', 'pipe'],
+        env: { ...process.env },
+      }
+    );
+
+    // Write the input as JSON to the worker's stdin
+    child.stdin.write(JSON.stringify(input));
+    child.stdin.end();
+
+    let resultData = '';
+    let errorData = '';
+
+    // Read result from fd3 (child.stdio[3])
+    if (child.stdio[3] && child.stdio[3] instanceof Readable) {
+      child.stdio[3].setEncoding('utf8');
+      child.stdio[3].on('data', (chunk) => {
+        resultData += chunk;
+      });
+    }
+
+    // Optionally, collect logs from stdout/stderr for debugging
+    child.stdout?.on('data', (chunk) => {
+      // Optionally log or buffer
+    });
+    child.stderr?.on('data', (chunk) => {
+      errorData += chunk;
+    });
+
+    child.on('error', (err) => {
+      reject(err);
+    });
+
+    child.on('close', (code) => {
+      if (resultData) {
+        try {
+          const parsed = JSON.parse(resultData);
+          resolve(parsed);
+        } catch (err) {
+          reject(new Error('Failed to parse worker result: ' + err));
+        }
+      } else if (errorData) {
+        reject(new Error('Worker error: ' + errorData));
+      } else {
+        reject(new Error('Worker exited with code ' + code + ' and no result.'));
+      }
+    });
+  });
+}

package/src/runner.ts

@@ -0,0 +1,178 @@
+import { clearRegistry, getRegistry } from './core/registry.js'
+import { startTraceSession, setCurrentTrace } from './trace-adapter/context.js'
+import { startLLMProxy, fetchCapturedTrace } from './proxy/llm-capture.js'
+import type { RunnerHooks } from './trace-adapter/context.js'
+import { pathToFileURL } from 'node:url'
+import { randomUUID } from 'node:crypto'
+import path from 'node:path'
+
+export interface TestResult {
+  name: string
+  passed: boolean
+  durationMs: number
+  error?: Error
+}
+
+export interface FileResult {
+  file: string
+  results: TestResult[]
+}
+
+export interface RunnerOptions {
+  hooks?: RunnerHooks
+}
+
+export async function runFiles(files: string[], options: RunnerOptions = {}): Promise<FileResult[]> {
+  // Optional local LLM capture proxy (opt-in via env). Default behavior stays unchanged when disabled.
+  const proxyOptIn = process.env.ELASTICDASH_LLM_PROXY === '1' || Boolean(process.env.ELASTICDASH_LLM_PROXY_URL)
+  const proxyPort = Number.parseInt(process.env.ELASTICDASH_LLM_PROXY_PORT || '8787', 10)
+  let proxyUrl = process.env.ELASTICDASH_LLM_PROXY_URL
+  const proxyHandle = proxyOptIn && !proxyUrl ? await startLLMProxy({ port: proxyPort }) : null
+  if (proxyHandle) {
+    proxyUrl = proxyHandle.url
+  }
+
+  const fileResults: FileResult[] = []
+
+  for (const file of files) {
+    const result = await runFile(file, options, { proxyOptIn, proxyUrl })
+    fileResults.push(result)
+  }
+
+  if (proxyHandle) {
+    await proxyHandle.stop()
+  }
+
+  return fileResults
+}
+
+async function runFile(file: string, options: RunnerOptions, proxyCtx: { proxyOptIn: boolean; proxyUrl: string | null | undefined }): Promise<FileResult> {
+  const { hooks = {} } = options
+
+  // 1. Clear the global registry before loading the file
+  clearRegistry()
+
+  // 2. Dynamically import the test file (triggers aiTest() registrations)
+  const resolvedPath = file.startsWith('file://')
+    ? file
+    : pathToFileURL(path.resolve(file)).href
+
+  if (resolvedPath.endsWith('.ts') && typeof (globalThis as any).Deno === 'undefined') {
+    await import('tsx/esm')
+    await import('tsx/cjs')
+  }
+
+  await import(resolvedPath)
+
+  const registry = getRegistry()
+  const results: TestResult[] = []
+
+  // Shared unhandled error trap for this file's test run
+  let currentTestName: string | null = null
+  let pendingUnhandled: Error | undefined
+  const onUnhandled = (reason: unknown) => {
+    if (!pendingUnhandled) pendingUnhandled = reason instanceof Error ? reason : new Error(String(reason))
+  }
+  process.on('unhandledRejection', onUnhandled)
+  process.on('uncaughtException', onUnhandled)
+
+  // 3. Run beforeAll hooks
+  for (const hook of registry.beforeAllHooks) {
+    await hook()
+  }
+
+  // 4. Execute each test sequentially
+  for (const entry of registry.tests) {
+    const { context, finalise } = startTraceSession()
+    const traceId = proxyCtx.proxyOptIn ? randomUUID() : null
+    setCurrentTrace(context.trace)
+    if (traceId) {
+      process.env.ELASTICDASH_TRACE_ID = traceId
+    }
+
+    if (hooks.onTestStart) {
+      await hooks.onTestStart(entry.name)
+    }
+
+    const startTime = Date.now()
+    let passed = false
+    let error: Error | undefined
+
+    // Reset per-test unhandled capture and mark current test name
+    pendingUnhandled = undefined
+    currentTestName = entry.name
+
+    setCurrentTrace(context.trace)
+    try {
+      for (const hook of registry.beforeEachHooks) {
+        await hook()
+      }
+
+      await entry.fn(context)
+      passed = true
+    } catch (err) {
+      error = err instanceof Error ? err : new Error(String(err))
+    } finally {
+      try {
+        for (const hook of registry.afterEachHooks) {
+          await hook()
+        }
+      } catch (afterErr) {
+        if (!error) {
+          error = afterErr instanceof Error ? afterErr : new Error(String(afterErr))
+          passed = false
+        }
+      }
+
+      setCurrentTrace(undefined)
+      if (!error && pendingUnhandled) {
+        error = pendingUnhandled
+        passed = false
+      }
+      currentTestName = null
+    }
+
+    const durationMs = Date.now() - startTime
+
+    if (hooks.onTestFinish) {
+      await hooks.onTestFinish(entry.name, passed, durationMs, error)
+    }
+
+    if (hooks.onTraceComplete) {
+      await hooks.onTraceComplete(entry.name, context.trace)
+    }
+
+    // If proxy mode is enabled, pull captured LLM steps and fold into the trace
+    if (traceId && proxyCtx.proxyUrl) {
+      try {
+        const captured = await fetchCapturedTrace(proxyCtx.proxyUrl, traceId)
+        for (const step of captured) {
+          context.trace.recordLLMStep(step)
+        }
+      } catch (proxyErr) {
+        // Non-fatal: keep test result as-is
+        // eslint-disable-next-line no-console
+        console.warn('[elasticdash] Failed to fetch proxy-captured steps:', proxyErr)
+      }
+    }
+
+    finalise()
+    setCurrentTrace(undefined)
+    if (traceId) {
+      delete process.env.ELASTICDASH_TRACE_ID
+    }
+
+    results.push({ name: entry.name, passed, durationMs, error })
+  }
+
+  // 5. Run afterAll hooks
+  for (const hook of registry.afterAllHooks) {
+    await hook()
+  }
+
+  // Cleanup shared handlers
+  process.off('unhandledRejection', onUnhandled)
+  process.off('uncaughtException', onUnhandled)
+
+  return { file, results }
+}