elasticdash-sdk 0.2.0

package/src/ci/test-loader.ts

@@ -0,0 +1,187 @@
+import { readFile } from 'node:fs/promises'
+import { resolve, dirname } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { discoverTestFiles } from './test-discovery.js'
+import { getTestRegistry, clearTestRegistry } from './test-registry.js'
+import type { TestDefinition } from './test-registry.js'
+import type { DiskTrace } from './trace-schema.js'
+
+export interface ValidatedTest extends TestDefinition {
+  resolvedTracePath: string
+  traceData: DiskTrace
+}
+
+export interface ValidationError {
+  file: string
+  testName?: string
+  message: string
+}
+
+export interface LoadTestsResult {
+  tests: ValidatedTest[]
+  errors: ValidationError[]
+}
+
+export async function loadTests(options?: { cwd?: string }): Promise<LoadTestsResult> {
+  const cwd = options?.cwd ?? process.cwd()
+  const files = await discoverTestFiles({ cwd })
+
+  const allDefinitions: TestDefinition[] = []
+  const errors: ValidationError[] = []
+
+  // Phase 1: import each test file and collect definitions
+  for (const file of files) {
+    clearTestRegistry()
+    try {
+      // Cache-bust so re-imports within the same process re-execute the module
+      const fileUrl = pathToFileURL(file).href + `?t=${Date.now()}`
+      await import(fileUrl)
+    } catch (err) {
+      errors.push({
+        file,
+        message: `Failed to import test file: ${err instanceof Error ? err.message : String(err)}`,
+      })
+      continue
+    }
+    const defs = getTestRegistry()
+    for (const def of defs) {
+      def._sourceFile = file
+    }
+    allDefinitions.push(...defs)
+  }
+  clearTestRegistry()
+
+  // Phase 2: validate definitions
+  const tests: ValidatedTest[] = []
+  const seenNames = new Map<string, string>() // name → source file
+
+  for (const def of allDefinitions) {
+    const file = def._sourceFile!
+
+    // Validate name
+    if (!def.name || typeof def.name !== 'string' || def.name.trim().length === 0) {
+      errors.push({ file, testName: def.name, message: 'Test name must be a non-empty string' })
+      continue
+    }
+
+    // Check uniqueness
+    if (seenNames.has(def.name)) {
+      errors.push({
+        file,
+        testName: def.name,
+        message: `Duplicate test name "${def.name}" (also defined in ${seenNames.get(def.name)})`,
+      })
+      continue
+    }
+    seenNames.set(def.name, file)
+
+    // Resolve trace path relative to the test file's directory
+    const resolvedTracePath = resolve(dirname(file), def.trace)
+
+    // Read and parse trace file
+    let traceData: DiskTrace
+    try {
+      const raw = await readFile(resolvedTracePath, 'utf-8')
+      traceData = JSON.parse(raw) as DiskTrace
+    } catch (err) {
+      errors.push({
+        file,
+        testName: def.name,
+        message: `Cannot read/parse trace file "${def.trace}": ${err instanceof Error ? err.message : String(err)}`,
+      })
+      continue
+    }
+
+    // Validate trace has steps array
+    if (!Array.isArray(traceData.steps)) {
+      errors.push({ file, testName: def.name, message: `Trace file "${def.trace}" has no steps array` })
+      continue
+    }
+
+    // Validate target
+    if (!def.target || !def.target.step_id || !def.target.type) {
+      errors.push({ file, testName: def.name, message: 'Target must have type and step_id' })
+      continue
+    }
+
+    const matchedStep = traceData.steps.find(s => s.step_id === def.target.step_id)
+    if (!matchedStep) {
+      errors.push({
+        file,
+        testName: def.name,
+        message: `step_id "${def.target.step_id}" not found in trace "${def.trace}"`,
+      })
+      continue
+    }
+
+    if (matchedStep.type !== def.target.type) {
+      errors.push({
+        file,
+        testName: def.name,
+        message: `target.type "${def.target.type}" does not match step type "${matchedStep.type}" for step_id "${def.target.step_id}"`,
+      })
+      continue
+    }
+
+    // Validate benchmarks
+    if (!def.benchmarks || typeof def.benchmarks !== 'object') {
+      errors.push({ file, testName: def.name, message: 'Benchmarks must be specified' })
+      continue
+    }
+
+    const hasMaxDuration = def.benchmarks.max_duration_ms !== undefined
+    const hasMaxTokens = def.benchmarks.max_tokens_total !== undefined
+    const hasOutputContains = def.benchmarks.output_contains !== undefined
+    const hasOutputNotContains = def.benchmarks.output_not_contains !== undefined
+    const hasLLMJudge = def.benchmarks.llm_judge !== undefined
+
+    if (!hasMaxDuration && !hasMaxTokens && !hasOutputContains && !hasOutputNotContains && !hasLLMJudge) {
+      errors.push({ file, testName: def.name, message: 'At least one benchmark must be specified' })
+      continue
+    }
+
+    let valid = true
+
+    if (hasMaxDuration) {
+      if (typeof def.benchmarks.max_duration_ms !== 'number' || def.benchmarks.max_duration_ms <= 0) {
+        errors.push({ file, testName: def.name, message: 'max_duration_ms must be a positive number' })
+        valid = false
+      }
+    }
+
+    if (hasMaxTokens) {
+      if (typeof def.benchmarks.max_tokens_total !== 'number' || def.benchmarks.max_tokens_total <= 0) {
+        errors.push({ file, testName: def.name, message: 'max_tokens_total must be a positive number' })
+        valid = false
+      }
+      if (def.target.type !== 'ai_call') {
+        errors.push({ file, testName: def.name, message: 'max_tokens_total can only be used with ai_call targets' })
+        valid = false
+      }
+    }
+
+    if (!valid) continue
+
+    // Validate run (optional but must be a function if present)
+    if (def.run !== undefined && typeof def.run !== 'function') {
+      errors.push({ file, testName: def.name, message: 'run must be a function' })
+      continue
+    }
+
+    // Validate timeout_ms (optional but must be a positive number if present)
+    if (def.timeout_ms !== undefined) {
+      if (typeof def.timeout_ms !== 'number' || def.timeout_ms <= 0) {
+        errors.push({ file, testName: def.name, message: 'timeout_ms must be a positive number' })
+        continue
+      }
+    }
+
+    tests.push({
+      ...def,
+      resolvedTracePath,
+      traceData,
+    })
+  }
+
+  return { tests, errors }
+}

package/src/ci/test-registry.ts

@@ -0,0 +1,62 @@
+export interface TestTarget {
+  type: 'tool_call' | 'ai_call'
+  step_id: string
+}
+
+export interface LLMJudgeBenchmark {
+  /** The prompt sent to the LLM judge to evaluate the step's output. */
+  judge_prompt: string
+  /** Minimum score (0-10) required to pass. Defaults to 7. */
+  judge_score_threshold?: number
+  /** LLM provider to use for judging. Defaults to 'openai'. */
+  judge_provider?: 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi'
+  /** Model override for the judge LLM. */
+  judge_model?: string
+}
+
+export interface TestBenchmarks {
+  max_duration_ms?: number
+  max_tokens_total?: number
+  /** Assert that the step's output contains this substring. */
+  output_contains?: string
+  /** Assert that the step's output does NOT contain this substring. */
+  output_not_contains?: string
+  /** LLM-as-a-judge evaluation of the step's output quality. */
+  llm_judge?: LLMJudgeBenchmark
+}
+
+export interface TestDefinition {
+  name: string
+  trace: string
+  target: TestTarget
+  benchmarks: TestBenchmarks
+  /** Custom input that overrides the trace's recorded input. Can be a static value or an async function for dynamic resolution (e.g. fetching from a database or API). */
+  input?: unknown | (() => Promise<unknown> | unknown)
+  /** The function that invokes the workflow under test. Receives the resolved input (custom or from trace) as its argument. Required for execution (Phase 3). */
+  run?: (input?: unknown) => Promise<void>
+  /** Per-test timeout in milliseconds. Defaults to 60000. */
+  timeout_ms?: number
+  /** Set internally by the loader to the absolute path of the source file */
+  _sourceFile?: string
+}
+
+// Use a globalThis-backed registry so that the CLI (global install) and the
+// test file (local node_modules) share the same array even when they resolve
+// to different module instances.
+const g = globalThis as Record<string, unknown>
+const ED_TEST_REGISTRY_KEY = '__elasticdash_ed_test_registry__'
+const registry: TestDefinition[] =
+  (g[ED_TEST_REGISTRY_KEY] as TestDefinition[]) ?? []
+if (!g[ED_TEST_REGISTRY_KEY]) g[ED_TEST_REGISTRY_KEY] = registry
+
+export function defineTest(def: Omit<TestDefinition, '_sourceFile'>): void {
+  registry.push({ ...def })
+}
+
+export function getTestRegistry(): TestDefinition[] {
+  return [...registry]
+}
+
+export function clearTestRegistry(): void {
+  registry.length = 0
+}

package/src/ci/trace-schema.ts

@@ -0,0 +1,96 @@
+import { readFileSync } from 'node:fs'
+import { join, dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import type { WorkflowEvent } from '../capture/event.js'
+
+export let SDK_VERSION = 'unknown'
+try {
+  // Works in both ESM (import.meta.url) and CJS (__dirname) contexts
+  const base = typeof __dirname !== 'undefined' ? __dirname : dirname(fileURLToPath(import.meta.url))
+  const pkg = JSON.parse(readFileSync(join(base, '..', '..', 'package.json'), 'utf-8'))
+  SDK_VERSION = pkg.version
+} catch {
+  // Bundled context — version not critical
+}
+
+export interface DiskTraceStep {
+  step_id: string
+  type: 'tool_call' | 'ai_call'
+  name: string
+  input: unknown
+  output: unknown
+  started_at: string
+  ended_at: string
+  duration_ms: number
+  tokens?: { input: number; output: number; total: number } | null
+}
+
+export interface DiskTrace {
+  trace_id: string
+  created_at: string
+  sdk_version: string
+  workflow: {
+    name: string
+    input: unknown
+    output: unknown
+  }
+  steps: DiskTraceStep[]
+}
+
+function mapEventType(type: string): 'tool_call' | 'ai_call' | null {
+  if (type === 'tool') return 'tool_call'
+  if (type === 'ai') return 'ai_call'
+  return null
+}
+
+export function workflowEventsToDiskTrace(events: WorkflowEvent[], traceId: string): DiskTrace {
+  const workflowEvent = events.find(e => e.type === 'workflow')
+
+  const counters: Record<string, number> = { tool_call: 0, ai_call: 0 }
+  const steps: DiskTraceStep[] = []
+
+  for (const event of events) {
+    const mappedType = mapEventType(event.type)
+    if (!mappedType) continue
+
+    const index = counters[mappedType]++
+    const stepId = `${mappedType}_${index}`
+
+    const startedAt = new Date(event.timestamp).toISOString()
+    const endedAt = new Date(event.timestamp + event.durationMs).toISOString()
+
+    let tokens: DiskTraceStep['tokens'] = null
+    if (mappedType === 'ai_call' && event.usage) {
+      tokens = {
+        input: event.usage.inputTokens ?? 0,
+        output: event.usage.outputTokens ?? 0,
+        total: event.usage.totalTokens ?? ((event.usage.inputTokens ?? 0) + (event.usage.outputTokens ?? 0)),
+      }
+    }
+
+    steps.push({
+      step_id: stepId,
+      type: mappedType,
+      name: event.name,
+      input: event.input,
+      output: event.output,
+      started_at: startedAt,
+      ended_at: endedAt,
+      duration_ms: event.durationMs,
+      tokens,
+    })
+  }
+
+  const now = new Date()
+  return {
+    trace_id: traceId,
+    created_at: now.toISOString(),
+    sdk_version: SDK_VERSION,
+    workflow: {
+      name: workflowEvent?.name ?? 'unknown',
+      input: workflowEvent?.input ?? null,
+      output: workflowEvent?.output ?? null,
+    },
+    steps,
+  }
+}

package/src/ci/trace-writer.ts

@@ -0,0 +1,107 @@
+import { mkdir, writeFile } from 'node:fs/promises'
+import { mkdirSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { randomBytes } from 'node:crypto'
+import type { WorkflowEvent } from '../capture/event.js'
+import { workflowEventsToDiskTrace } from './trace-schema.js'
+import type { DiskTrace } from './trace-schema.js'
+
+const TRACE_DIR = '.ed_traces'
+
+const SENSITIVE_KEYS = /^(authorization|api_key|apikey|password|secret|token)$/i
+
+export function isTraceCaptureEnabled(): boolean {
+  const val = process.env.ELASTICDASH_CAPTURE_TRACE
+  if (val === '1') return true
+  // Log once on first check so users can diagnose env var issues
+  if (!_captureCheckLogged) {
+    _captureCheckLogged = true
+    if (val !== undefined) {
+      console.log(`[elasticdash] ELASTICDASH_CAPTURE_TRACE=${JSON.stringify(val)} (expected "1" to enable)`)
+    }
+  }
+  return false
+}
+let _captureCheckLogged = false
+
+export function generateTraceFilename(): string {
+  const now = new Date()
+  const ts = now.toISOString()
+    .replace(/\.\d{3}Z$/, '')   // remove milliseconds
+    .replace(/:/g, '-')          // filesystem-safe
+  const suffix = randomBytes(2).toString('hex') // 4 hex chars
+  return `${ts}_${suffix}.json`
+}
+
+export function scrubSecrets(obj: unknown): unknown {
+  if (obj === null || obj === undefined) return obj
+  if (Array.isArray(obj)) return obj.map(item => scrubSecrets(item))
+  if (typeof obj === 'object') {
+    const result: Record<string, unknown> = {}
+    for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
+      if (SENSITIVE_KEYS.test(key)) {
+        result[key] = '[REDACTED]'
+      } else {
+        result[key] = scrubSecrets(value)
+      }
+    }
+    return result
+  }
+  return obj
+}
+
+export async function writeTraceToDisk(trace: DiskTrace, cwd?: string): Promise<void> {
+  const dir = join(cwd ?? process.cwd(), TRACE_DIR)
+  try {
+    await mkdir(dir, { recursive: true })
+    const scrubbed = scrubSecrets(trace) as DiskTrace
+    const filename = `${scrubbed.trace_id}.json`
+    const filepath = join(dir, filename)
+    await writeFile(filepath, JSON.stringify(scrubbed, null, 2), 'utf-8')
+    console.log(`[elasticdash] Trace written: ${filepath} (${trace.steps.length} steps)`)
+  } catch (err) {
+    console.warn(`[elasticdash] Failed to write trace file: ${err instanceof Error ? err.message : String(err)}`)
+  }
+}
+
+/**
+ * Synchronous write for use in process.on('exit') handlers where async is not possible.
+ */
+export function writeTraceToDiskSync(trace: DiskTrace, cwd?: string): void {
+  const dir = join(cwd ?? process.cwd(), TRACE_DIR)
+  try {
+    mkdirSync(dir, { recursive: true })
+    const scrubbed = scrubSecrets(trace) as DiskTrace
+    const filename = `${scrubbed.trace_id}.json`
+    const filepath = join(dir, filename)
+    writeFileSync(filepath, JSON.stringify(scrubbed, null, 2), 'utf-8')
+  } catch (err) {
+    // In sync exit handler, console.warn may not flush — best effort
+    try { console.warn(`[elasticdash] Failed to write trace file: ${err instanceof Error ? err.message : String(err)}`) } catch {}
+  }
+}
+
+export async function maybeCaptureTrace(events: WorkflowEvent[], traceId?: string, cwd?: string): Promise<void> {
+  if (!isTraceCaptureEnabled()) return
+  if (events.length === 0) return
+
+  const filename = generateTraceFilename()
+  const id = filename.replace(/\.json$/, '')
+  const trace = workflowEventsToDiskTrace(events, traceId ?? id)
+  trace.trace_id = id
+  await writeTraceToDisk(trace, cwd)
+}
+
+/**
+ * Synchronous version of maybeCaptureTrace for process exit handlers.
+ */
+export function maybeCaptureTraceSync(events: WorkflowEvent[], traceId?: string, cwd?: string): void {
+  if (!isTraceCaptureEnabled()) return
+  if (events.length === 0) return
+
+  const filename = generateTraceFilename()
+  const id = filename.replace(/\.json$/, '')
+  const trace = workflowEventsToDiskTrace(events, traceId ?? id)
+  trace.trace_id = id
+  writeTraceToDiskSync(trace, cwd)
+}

package/src/ci/types.ts

@@ -0,0 +1,115 @@
+// ─── CI Runner Types ─────────────────────────────────────────
+
+export interface CIRunConfig {
+  serverUrl: string
+  apiKey: string
+  workflowName?: string
+  tags?: string[]
+  triggeredBy?: 'ci' | 'api'
+  gitBranch?: string
+  gitCommit?: string
+  gitCommitMessage?: string
+  gitPrNumber?: number
+  gitPrUrl?: string
+}
+
+export interface CITestResult {
+  testGroupId: number
+  testGroupName: string
+  testId: number
+  testName: string | null
+  testType: string
+  passed: boolean
+  runId: number | null
+  singleRuns: CISingleRunResult[]
+  expectationResults: CIExpectationResult[]
+  error?: string
+  durationMs: number
+}
+
+export interface CISingleRunResult {
+  runIndex: number
+  passed: boolean
+  durationMs: number
+  inputTokens: number
+  outputTokens: number
+  totalTokens: number
+  output: unknown
+  trace: unknown
+  error?: string
+}
+
+export interface CIExpectationResult {
+  expectationId: number
+  type: string
+  passed: boolean
+  detail?: string
+  perRun?: Record<number, { passed: boolean; detail?: string }>
+}
+
+export interface CIRunSummary {
+  total: number
+  passed: number
+  failed: number
+  skipped: number
+  durationMs: number
+  batchId: number | null
+  results: CITestResult[]
+}
+
+// ─── API Response Types ─────────────────────────────────────
+
+export interface APITestGroup {
+  id: number
+  name: string
+  description: string | null
+  project_id: number
+  workflow_name: string
+  trace_file: unknown
+  status: string
+  tags: string[]
+  tests: APITestGroupTest[]
+}
+
+export interface APITestGroupTest {
+  id: number
+  test_group_id: number
+  name: string | null
+  description: string | null
+  test_type: 'single-step' | 'full-flow'
+  target_step_index: number | null
+  target_step_type: string | null
+  target_step_name: string | null
+  mock_input: unknown
+  workflow_input: unknown
+  frozen_events: unknown[]
+  tool_mocks: Record<string, unknown>
+  prompt_mocks: Record<string, string>
+  run_count: number
+  pass_threshold: string
+  timeout_ms: number
+  sort_order: number
+  expectations: APIExpectation[]
+}
+
+export interface APIExpectation {
+  id: number
+  test_group_test_id: number
+  type: string
+  judge_prompt: string | null
+  judge_model: string | null
+  judge_provider: string | null
+  judge_score_threshold: number | null
+  max_total_tokens: number | null
+  max_tokens_per_run: number | null
+  max_duration_ms: number | null
+  max_total_duration_ms: number | null
+  contains_text: string | null
+  not_contains_text: string | null
+  case_insensitive: boolean
+  json_schema: unknown
+  similarity_threshold: number | null
+  required_tools: string[]
+  forbidden_tools: string[]
+  tool_call_rules: unknown
+}