elasticdash-sdk 0.2.0

package/src/capture/recorder.ts

@@ -0,0 +1,62 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+import { randomUUID } from 'node:crypto'
+import type { WorkflowEvent, WorkflowTrace } from './event.js'
+import type { ReplayController } from './replay.js'
+
+export class TraceRecorder {
+  events: WorkflowEvent[] = []
+  private _counter = 0
+  private _sideEffectCounter = 0
+  private _pending: Set<Promise<void>> = new Set()
+
+  record(event: WorkflowEvent): void {
+    this.events.push(event)
+  }
+
+  /** Register an in-flight async recording promise so flush() can await it. */
+  trackAsync(promise: Promise<void>): void {
+    this._pending.add(promise)
+    promise.finally(() => { this._pending.delete(promise) })
+  }
+
+  /** Await all in-flight async recordings. No-op when none are pending. */
+  async flush(): Promise<void> {
+    await Promise.allSettled([...this._pending])
+  }
+
+  nextId(): number {
+    return ++this._counter
+  }
+
+  /** Separate counter for Date.now / Math.random — never shares IDs with main events. */
+  nextSideEffectId(): number {
+    return ++this._sideEffectCounter
+  }
+
+  toTrace(traceId?: string): WorkflowTrace {
+    return {
+      traceId: traceId ?? randomUUID(),
+      events: [...this.events],
+    }
+  }
+}
+
+export interface CaptureContext {
+  recorder: TraceRecorder
+  replay: ReplayController
+}
+
+const g = globalThis as Record<string, unknown>
+const CAPTURE_ALS_KEY = '__elasticdash_capture_als__'
+const captureAls: AsyncLocalStorage<CaptureContext | undefined> =
+  (g[CAPTURE_ALS_KEY] as AsyncLocalStorage<CaptureContext | undefined>) ??
+  new AsyncLocalStorage<CaptureContext | undefined>()
+if (!g[CAPTURE_ALS_KEY]) g[CAPTURE_ALS_KEY] = captureAls
+
+export function setCaptureContext(ctx: CaptureContext | undefined): void {
+  captureAls.enterWith(ctx)
+}
+
+export function getCaptureContext(): CaptureContext | undefined {
+  return captureAls.getStore()
+}

package/src/capture/replay.ts

@@ -0,0 +1,55 @@
+import type { WorkflowEvent } from './event.js'
+
+export class ReplayController {
+  private historyMap: Map<number, WorkflowEvent>
+  /** Side effects keyed by their assigned sideEffectId, independent of main event IDs */
+  private sideEffectMap: Map<number, WorkflowEvent>
+
+  constructor(
+    public replayMode: boolean,
+    public checkpoint: number,
+    public history: WorkflowEvent[],
+  ) {
+    this.historyMap = new Map(history.map(e => [e.id, e]))
+    this.sideEffectMap = new Map(
+      history.filter(e => e.type === 'side_effect').map(e => [e.id, e]),
+    )
+  }
+
+  shouldReplay(eventId: number): boolean {
+    return this.replayMode && eventId <= this.checkpoint
+  }
+
+  getRecordedEvent(eventId: number): WorkflowEvent | undefined {
+    return this.historyMap.get(eventId)
+  }
+
+  getRecordedResult(eventId: number): unknown {
+    return this.historyMap.get(eventId)?.output
+  }
+
+  /** Returns true if the side effect with this sideEffectId has a recorded value to replay */
+  shouldReplaySideEffect(n: number): boolean {
+    return this.replayMode && this.sideEffectMap.has(n)
+  }
+
+  getSideEffectResult(n: number): unknown {
+    return this.sideEffectMap.get(n)?.output
+  }
+
+  getRecordedSideEffectEvent(n: number): WorkflowEvent | undefined {
+    return this.sideEffectMap.get(n)
+  }
+
+  shouldReplaySideEffectOfType(n: number, expectedName: string): boolean {
+    if (!this.replayMode) return false
+    const event = this.sideEffectMap.get(n)
+    return !!event && event.type === 'side_effect' && event.name === expectedName
+  }
+
+  getSideEffectResultOfType(n: number, expectedName: string): unknown {
+    const event = this.sideEffectMap.get(n)
+    if (!event || event.type !== 'side_effect' || event.name !== expectedName) return undefined
+    return event.output
+  }
+}

package/src/ci/api-client.ts

@@ -0,0 +1,136 @@
+import { randomUUID } from 'node:crypto'
+import { getOriginalFetch } from '../interceptors/http.js'
+import type { APITestGroup } from './types.js'
+
+// ─── API Client ──────────────────────────────────────────────
+// Uses getOriginalFetch() to bypass SDK interceptors.
+
+/** Normalize serverUrl: strip trailing slash and trailing /api to avoid double /api/api paths */
+function normalizeBase(serverUrl: string): string {
+  return serverUrl.replace(/\/+$/, '').replace(/\/api$/, '')
+}
+
+function headers(apiKey: string): Record<string, string> {
+  return {
+    'Content-Type': 'application/json',
+    'api-key': apiKey || '',
+    'X-Correlation-ID': randomUUID(),
+  }
+}
+
+async function apiRequest<T>(
+  url: string,
+  apiKey: string,
+  options: RequestInit = {},
+): Promise<T> {
+  const method = (options.method || 'GET').toUpperCase()
+  console.log(`[elasticdash ci] ${method} ${url}`)
+
+  const res = await getOriginalFetch()(url, {
+    ...options,
+    headers: { ...headers(apiKey), ...(options.headers as Record<string, string> ?? {}) },
+  })
+
+  if (!res.ok) {
+    const text = await res.text().catch(() => '')
+    console.log(`[elasticdash ci] ${method} ${url} → ${res.status} ${text.substring(0, 200)}`)
+    throw new Error(`API ${res.status}: ${text || res.statusText}`)
+  }
+
+  const json = await res.json() as Record<string, unknown>
+  // Backend wraps responses in generalApiResponseSender which sends { result: ... }
+  // Use 'in' check instead of nullish coalescing — result:null is valid data, not "missing"
+  const result = 'result' in json ? json.result : ('data' in json ? json.data : json)
+  console.log(`[elasticdash ci] ${method} ${url} → ${res.status} (result keys: ${result && typeof result === 'object' ? Object.keys(result).join(',') : typeof result})`)
+  return result as T
+}
+
+/**
+ * Fetch all active test groups (with nested tests & expectations) for the project
+ * scoped to the API key.
+ */
+export async function fetchTestGroups(
+  serverUrl: string,
+  apiKey: string,
+  filters?: { workflowName?: string; tags?: string[]; status?: string },
+): Promise<APITestGroup[]> {
+  const base = normalizeBase(serverUrl)
+  const params = new URLSearchParams()
+  if (filters?.workflowName) params.set('workflowName', filters.workflowName)
+  if (filters?.tags?.length) params.set('tags', filters.tags.join(','))
+  if (filters?.status) params.set('status', filters.status)
+
+  const qs = params.toString()
+  const url = `${base}/api/testgroups/by-project${qs ? `?${qs}` : ''}`
+  return apiRequest<APITestGroup[]>(url, apiKey)
+}
+
+/**
+ * Submit a test run result to the backend.
+ */
+export async function submitTestRun(
+  serverUrl: string,
+  apiKey: string,
+  testGroupId: number,
+  payload: Record<string, unknown>,
+): Promise<{ id: number }> {
+  const base = normalizeBase(serverUrl)
+  const url = `${base}/api/testgroups/${testGroupId}/runs`
+  return apiRequest<{ id: number }>(url, apiKey, {
+    method: 'POST',
+    body: JSON.stringify(payload),
+  })
+}
+
+/**
+ * Create a batch grouping multiple test run IDs.
+ */
+export async function createBatch(
+  serverUrl: string,
+  apiKey: string,
+  payload: Record<string, unknown>,
+): Promise<{ id: number }> {
+  const base = normalizeBase(serverUrl)
+  const url = `${base}/api/testgroups/batches`
+  return apiRequest<{ id: number }>(url, apiKey, {
+    method: 'POST',
+    body: JSON.stringify(payload),
+  })
+}
+
+/**
+ * Resolved evaluator configuration from the backend.
+ * Provider/model/apiKey may be null if the user has not configured an evaluator.
+ */
+export interface EvaluatorConfig {
+  provider: string | null
+  model: string | null
+  apiKey: string | null
+}
+
+/** Maps backend llmProviderId to provider name. */
+const LLM_PROVIDER_MAP: Record<number, string> = {
+  1: 'openai',
+  2: 'gemini',
+  3: 'anthropic',
+  4: 'moonshot',
+}
+
+/**
+ * Fetch the user's default evaluator config.
+ * Used by ed-test llm_judge benchmarks when judge_provider/judge_model
+ * are not specified in the test definition.
+ */
+export async function fetchEvaluatorConfig(
+  serverUrl: string,
+  apiKey: string,
+): Promise<EvaluatorConfig> {
+  const base = normalizeBase(serverUrl)
+  const url = `${base}/api/user/settings/llm/default-evaluator`
+  const raw = await apiRequest<{ llmProviderId?: number; model?: string }>(url, apiKey)
+  return {
+    provider: raw?.llmProviderId ? (LLM_PROVIDER_MAP[raw.llmProviderId] ?? null) : null,
+    model: raw?.model ?? null,
+    apiKey: null,
+  }
+}

package/src/ci/benchmark.ts

@@ -0,0 +1,257 @@
+/**
+ * benchmark.ts
+ *
+ * Compares recorded trace measurements against user-defined benchmarks.
+ * Supports duration, token, output-contains, and LLM-as-a-judge evaluations.
+ *
+ * Generated/updated on 2026-04-20.
+ */
+
+import { callProviderLLM } from '../matchers/index.js'
+import { prepareOutputForJudge } from '../core/judge-utils.js'
+import type { TestMeasurement } from './measurement.js'
+import type { TestBenchmarks } from './test-registry.js'
+import type { EvaluatorConfig } from './api-client.js'
+
+export type MetricName = 'duration_ms' | 'tokens_total' | 'output_contains' | 'output_not_contains' | 'llm_judge'
+
+export interface MetricResult {
+  name: MetricName
+  value: number
+  threshold: number
+  passed: boolean
+  detail?: string
+}
+
+export interface BenchmarkResult {
+  passed: boolean
+  failure_reason?: string
+  metrics: MetricResult[]
+}
+
+/** Maps backend provider names to SDK provider names used by callProviderLLM. */
+const PROVIDER_NAME_MAP: Record<string, string> = {
+  anthropic: 'claude',
+  moonshot: 'kimi',
+}
+
+/** Default model for each provider, used when no explicit model is set or
+ *  when the evaluator config model doesn't belong to the resolved provider. */
+const DEFAULT_PROVIDER_MODELS: Record<string, string> = {
+  openai: 'gpt-4o',
+  claude: 'claude-sonnet-4-20250514',
+  gemini: 'gemini-2.0-flash',
+  grok: 'grok-3',
+  kimi: 'moonshot-v1-auto',
+}
+
+/** Known model prefixes per provider — used to check if a model belongs to a provider. */
+const PROVIDER_MODEL_PREFIXES: Record<string, string[]> = {
+  openai: ['gpt-', 'o1-', 'o3-', 'o4-', 'chatgpt-', 'omni-'],
+  claude: ['claude-'],
+  gemini: ['gemini-'],
+  grok: ['grok-'],
+  kimi: ['moonshot-', 'kimi-'],
+}
+
+/** Check if a model name belongs to the given provider. */
+function isModelForProvider(model: string, provider: string): boolean {
+  const prefixes = PROVIDER_MODEL_PREFIXES[provider]
+  if (!prefixes) return false
+  return prefixes.some(p => model.toLowerCase().startsWith(p))
+}
+
+/** Normalize provider name from backend format to SDK format. */
+function normalizeSdkProvider(provider: string): string {
+  return PROVIDER_NAME_MAP[provider] ?? provider
+}
+
+/**
+ * Compare a measurement against benchmarks. Async because llm_judge requires
+ * an LLM call. The step's output is needed for output_contains/llm_judge checks.
+ *
+ * @param evaluatorConfig - Optional backend evaluator config used as fallback
+ *   when the test does not specify judge_provider/judge_model.
+ */
+export async function compareBenchmarks(
+  measurement: TestMeasurement,
+  benchmarks: TestBenchmarks,
+  stepOutput?: unknown,
+  evaluatorConfig?: EvaluatorConfig | null,
+): Promise<BenchmarkResult> {
+  const metrics: MetricResult[] = []
+  let firstFailure: string | undefined
+
+  if (benchmarks.max_duration_ms !== undefined) {
+    const passed = measurement.duration_ms <= benchmarks.max_duration_ms
+    metrics.push({
+      name: 'duration_ms',
+      value: measurement.duration_ms,
+      threshold: benchmarks.max_duration_ms,
+      passed,
+    })
+    if (!passed && !firstFailure) {
+      firstFailure = `duration_ms (${measurement.duration_ms}) exceeded max threshold (${benchmarks.max_duration_ms})`
+    }
+  }
+
+  if (benchmarks.max_tokens_total !== undefined) {
+    const value = measurement.tokens_total ?? 0
+    const passed = value <= benchmarks.max_tokens_total
+    metrics.push({
+      name: 'tokens_total',
+      value,
+      threshold: benchmarks.max_tokens_total,
+      passed,
+    })
+    if (!passed && !firstFailure) {
+      firstFailure = `tokens_total (${value}) exceeded max threshold (${benchmarks.max_tokens_total})`
+    }
+  }
+
+  // ─── Output contains ────────────────────────────────────────
+  if (benchmarks.output_contains !== undefined) {
+    const outputStr = stringifyOutput(stepOutput)
+    const passed = outputStr.toLowerCase().includes(benchmarks.output_contains.toLowerCase())
+    metrics.push({
+      name: 'output_contains',
+      value: passed ? 1 : 0,
+      threshold: 1,
+      passed,
+      detail: passed ? undefined : `output does not contain "${benchmarks.output_contains}"`,
+    })
+    if (!passed && !firstFailure) {
+      firstFailure = `output does not contain "${benchmarks.output_contains}"`
+    }
+  }
+
+  // ─── Output not contains ────────────────────────────────────
+  if (benchmarks.output_not_contains !== undefined) {
+    const outputStr = stringifyOutput(stepOutput)
+    const passed = !outputStr.toLowerCase().includes(benchmarks.output_not_contains.toLowerCase())
+    metrics.push({
+      name: 'output_not_contains',
+      value: passed ? 1 : 0,
+      threshold: 1,
+      passed,
+      detail: passed ? undefined : `output unexpectedly contains "${benchmarks.output_not_contains}"`,
+    })
+    if (!passed && !firstFailure) {
+      firstFailure = `output unexpectedly contains "${benchmarks.output_not_contains}"`
+    }
+  }
+
+  // ─── LLM-as-a-judge ────────────────────────────────────────
+  if (benchmarks.llm_judge) {
+    const judge = benchmarks.llm_judge
+    const outputStr = stringifyOutput(stepOutput)
+    const threshold = judge.judge_score_threshold ?? 7
+
+    // Resolve provider/model: test definition takes priority, then backend
+    // evaluator config, then fall back to 'openai' default.
+    const resolvedProvider = normalizeSdkProvider(
+      judge.judge_provider ?? evaluatorConfig?.provider ?? 'openai'
+    )
+    // Model resolution: judge_model > evaluatorConfig.model (if compatible) > provider default
+    let resolvedModel = judge.judge_model ?? undefined
+    if (!resolvedModel && evaluatorConfig?.model) {
+      // Only use the evaluator config model if it belongs to the resolved provider
+      if (isModelForProvider(evaluatorConfig.model, resolvedProvider)) {
+        resolvedModel = evaluatorConfig.model
+      }
+    }
+    if (!resolvedModel) {
+      resolvedModel = DEFAULT_PROVIDER_MODELS[resolvedProvider]
+    }
+
+    // If the backend provided an API key and we're using its provider,
+    // set it in the environment so callProviderLLM can pick it up.
+    const envKeyMap: Record<string, string> = {
+      openai: 'OPENAI_API_KEY',
+      claude: 'ANTHROPIC_API_KEY',
+      gemini: 'GEMINI_API_KEY',
+      grok: 'GROK_API_KEY',
+      kimi: 'KIMI_API_KEY',
+    }
+    const envKey = envKeyMap[resolvedProvider]
+    let restoreEnv: (() => void) | undefined
+    if (evaluatorConfig?.apiKey && envKey && !judge.judge_provider && !process.env[envKey]) {
+      const prev = process.env[envKey]
+      process.env[envKey] = evaluatorConfig.apiKey
+      restoreEnv = () => {
+        if (prev === undefined) delete process.env[envKey]
+        else process.env[envKey] = prev
+      }
+    }
+
+    try {
+      console.log(`  [llm_judge] provider=${resolvedProvider}, model=${resolvedModel ?? '(default)'}`)
+      const preparedOutput = prepareOutputForJudge(outputStr, judge.judge_prompt)
+      const evalPrompt = `${judge.judge_prompt}\n\n<output>\n${preparedOutput}\n</output>\n\nBased on the evaluation criteria above, score this output on a scale of 0-10. Respond with only the number.`
+
+      const result = await callProviderLLM(
+        evalPrompt,
+        { provider: resolvedProvider as 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi', model: resolvedModel },
+        'You are an expert test judge. Return only a number between 0 and 10.',
+        4096,
+        0,
+      )
+
+      restoreEnv?.()
+
+      console.log(`  [llm_judge] raw response: "${result.content}"`)
+      const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '')
+      if (isNaN(score)) {
+        metrics.push({
+          name: 'llm_judge',
+          value: 0,
+          threshold,
+          passed: false,
+          detail: `Could not parse score from LLM response: "${result.content}"`,
+        })
+        if (!firstFailure) {
+          firstFailure = `llm_judge: could not parse score from response`
+        }
+      } else {
+        const passed = score >= threshold
+        metrics.push({
+          name: 'llm_judge',
+          value: score,
+          threshold,
+          passed,
+          detail: `Score: ${score}/${threshold}`,
+        })
+        if (!passed && !firstFailure) {
+          firstFailure = `llm_judge score (${score}) below threshold (${threshold})`
+        }
+      }
+    } catch (err) {
+      restoreEnv?.()
+      const errMsg = err instanceof Error ? err.message : String(err)
+      metrics.push({
+        name: 'llm_judge',
+        value: 0,
+        threshold,
+        passed: false,
+        detail: `LLM judge error: ${errMsg}`,
+      })
+      if (!firstFailure) {
+        firstFailure = `llm_judge error: ${errMsg}`
+      }
+    }
+  }
+
+  const allPassed = metrics.every(m => m.passed)
+  return {
+    passed: allPassed,
+    failure_reason: firstFailure,
+    metrics,
+  }
+}
+
+/** Converts step output to a string for text-based assertions. */
+function stringifyOutput(output: unknown): string {
+  if (output === null || output === undefined) return ''
+  if (typeof output === 'string') return output
+  return JSON.stringify(output)
+}