elasticdash-test 0.1.18 → 0.1.19-alpha-2

package/src/ci/executor.ts

@@ -0,0 +1,668 @@
+import { executePortalTask, checkToolAvailability, checkAIAvailability } from '../portal-executor.js'
+import { scanTools } from '../execution/tool-runner.js'
+import { callProviderLLM } from '../matchers/index.js'
+import type { ToolInfo } from '../execution/tool-runner.js'
+import type {
+  APITestGroupTest,
+  APIExpectation,
+  CISingleRunResult,
+  CIExpectationResult,
+} from './types.js'
+
+// ─── Test Executor ───────────────────────────────────────────
+// Executes a single TestGroupTest: runs it N times, evaluates expectations,
+// determines pass/fail based on pass_threshold.
+
+interface ExecutionResult {
+  passed: boolean
+  singleRuns: CISingleRunResult[]
+  expectationResults: CIExpectationResult[]
+  durationMs: number
+}
+
+/**
+ * Execute a test (single-step or full-flow) according to its configuration.
+ */
+export async function executeTest(
+  test: APITestGroupTest,
+  cwd: string,
+): Promise<ExecutionResult> {
+  const tools = scanTools(cwd)
+  const runCount = test.run_count || 1
+  const timeoutMs = test.timeout_ms || 30000
+  const startTime = Date.now()
+
+  const singleRuns: CISingleRunResult[] = []
+
+  for (let i = 0; i < runCount; i++) {
+    const runStart = Date.now()
+    let result: CISingleRunResult
+
+    try {
+      const runPromise = executeSingleRun(test, cwd, tools, i)
+      const timeoutPromise = new Promise<never>((_, reject) =>
+        setTimeout(() => reject(new Error(`Test timed out after ${timeoutMs}ms`)), timeoutMs)
+      )
+      result = await Promise.race([runPromise, timeoutPromise])
+    } catch (err) {
+      result = {
+        runIndex: i + 1,
+        passed: false,
+        durationMs: Date.now() - runStart,
+        inputTokens: 0,
+        outputTokens: 0,
+        totalTokens: 0,
+        output: null,
+        trace: null,
+        error: err instanceof Error ? err.message : String(err),
+      }
+    }
+
+    singleRuns.push(result)
+  }
+
+  // Evaluate expectations against the single runs
+  const expectationResults = await evaluateExpectations(test.expectations, singleRuns)
+
+  // Determine overall pass/fail
+  const passed = determinePassFail(test.pass_threshold, singleRuns, expectationResults)
+
+  return {
+    passed,
+    singleRuns,
+    expectationResults,
+    durationMs: Date.now() - startTime,
+  }
+}
+
+// ─── Single Run Execution ────────────────────────────────────
+
+async function executeSingleRun(
+  test: APITestGroupTest,
+  cwd: string,
+  tools: ToolInfo[],
+  runIndex: number,
+): Promise<CISingleRunResult> {
+  const start = Date.now()
+
+  if (test.test_type === 'single-step') {
+    return executeSingleStep(test, cwd, tools, runIndex, start)
+  }
+
+  return executeFullFlow(test, cwd, tools, runIndex, start)
+}
+
+async function executeSingleStep(
+  test: APITestGroupTest,
+  cwd: string,
+  tools: ToolInfo[],
+  runIndex: number,
+  start: number,
+): Promise<CISingleRunResult> {
+  const stepType = test.target_step_type // 'ai' or 'tool'
+  const stepName = test.target_step_name
+
+  if (!stepType || !stepName) {
+    return {
+      runIndex: runIndex + 1,
+      passed: false,
+      durationMs: Date.now() - start,
+      inputTokens: 0, outputTokens: 0, totalTokens: 0,
+      output: null, trace: null,
+      error: 'Single-step test requires target_step_type and target_step_name.',
+    }
+  }
+
+  // Pre-validate availability
+  const availability = stepType === 'ai'
+    ? checkAIAvailability(undefined, stepName)
+    : checkToolAvailability(stepName, cwd, tools)
+
+  if (!availability.available) {
+    return {
+      runIndex: runIndex + 1,
+      passed: false,
+      durationMs: Date.now() - start,
+      inputTokens: 0, outputTokens: 0, totalTokens: 0,
+      output: null, trace: null,
+      error: `Step unavailable: ${availability.reason}`,
+    }
+  }
+
+  // Execute via portal executor (reuses existing infrastructure)
+  const result = await executePortalTask(
+    {
+      taskId: `ci-${test.id}-run-${runIndex}`,
+      type: stepType === 'ai' ? 'ai' : 'tool',
+      name: stepName,
+      input: test.mock_input,
+      frozenEvents: test.frozen_events as any[],
+    },
+    cwd,
+    tools,
+  )
+
+  return {
+    runIndex: runIndex + 1,
+    passed: result.ok,
+    durationMs: result.durationMs,
+    inputTokens: result.usage?.inputTokens ?? 0,
+    outputTokens: result.usage?.outputTokens ?? 0,
+    totalTokens: result.usage?.totalTokens ?? 0,
+    output: result.output,
+    trace: null,
+    error: result.error,
+  }
+}
+
+async function executeFullFlow(
+  test: APITestGroupTest,
+  cwd: string,
+  tools: ToolInfo[],
+  runIndex: number,
+  start: number,
+): Promise<CISingleRunResult> {
+  // Full-flow tests require running the entire workflow.
+  // We leverage the existing workflow runner infrastructure.
+  // The workflow is loaded from ed_workflows.ts and executed with workflow_input.
+
+  try {
+    const { runWorkflow } = await import('../workflow-runner.js')
+    const { resolveRuntimeModule } = await import('../execution/tool-runner.js')
+    const { pathToFileURL } = await import('node:url')
+
+    const workflowModulePath = resolveRuntimeModule(cwd, 'ed_workflows')
+    if (!workflowModulePath) {
+      return {
+        runIndex: runIndex + 1,
+        passed: false,
+        durationMs: Date.now() - start,
+        inputTokens: 0, outputTokens: 0, totalTokens: 0,
+        output: null, trace: null,
+        error: 'Cannot find ed_workflows.ts/js in workspace root.',
+      }
+    }
+
+    // Import the workflow module dynamically
+    const workflowModule = await import(pathToFileURL(workflowModulePath).href)
+
+    // Find a suitable workflow function to execute
+    // Convention: use the first exported async function, or match by test group's workflow name
+    const workflowFns = Object.entries(workflowModule).filter(
+      ([, val]) => typeof val === 'function'
+    ) as [string, (...args: unknown[]) => Promise<unknown>][]
+
+    if (workflowFns.length === 0) {
+      return {
+        runIndex: runIndex + 1,
+        passed: false,
+        durationMs: Date.now() - start,
+        inputTokens: 0, outputTokens: 0, totalTokens: 0,
+        output: null, trace: null,
+        error: 'No workflow functions found in ed_workflows.',
+      }
+    }
+
+    // Prefer matching by name if the test has a target_step_name
+    const targetFn = test.target_step_name
+      ? workflowFns.find(([name]) => name === test.target_step_name)?.[1]
+      : workflowFns[0][1]
+
+    const fn = targetFn ?? workflowFns[0][1]
+
+    const frozenEvents = Array.isArray(test.frozen_events) ? test.frozen_events : []
+
+    const { result, trace } = await runWorkflow(
+      () => {
+        const input = test.workflow_input
+        return fn(input) as Promise<unknown>
+      },
+      {
+        replayMode: frozenEvents.length > 0,
+        history: frozenEvents as any[],
+        interceptHttp: true,
+        interceptSideEffects: true,
+      },
+    )
+
+    // Extract usage from trace events
+    let inputTokens = 0, outputTokens = 0, totalTokens = 0
+    if (trace?.events) {
+      for (const evt of trace.events) {
+        if (evt.type === 'ai' && evt.usage) {
+          inputTokens += (evt.usage as any).inputTokens ?? (evt.usage as any).input ?? 0
+          outputTokens += (evt.usage as any).outputTokens ?? (evt.usage as any).output ?? 0
+          totalTokens += (evt.usage as any).totalTokens ?? (evt.usage as any).total ?? 0
+        }
+      }
+    }
+
+    return {
+      runIndex: runIndex + 1,
+      passed: true,
+      durationMs: Date.now() - start,
+      inputTokens, outputTokens, totalTokens,
+      output: result,
+      trace: trace ?? null,
+    }
+  } catch (err) {
+    return {
+      runIndex: runIndex + 1,
+      passed: false,
+      durationMs: Date.now() - start,
+      inputTokens: 0, outputTokens: 0, totalTokens: 0,
+      output: null, trace: null,
+      error: err instanceof Error ? err.message : String(err),
+    }
+  }
+}
+
+// ─── Expectation Evaluation ──────────────────────────────────
+
+async function evaluateExpectations(
+  expectations: APIExpectation[],
+  singleRuns: CISingleRunResult[],
+): Promise<CIExpectationResult[]> {
+  const results: CIExpectationResult[] = []
+
+  for (const exp of expectations) {
+    const result = await evaluateExpectation(exp, singleRuns)
+    results.push(result)
+  }
+
+  return results
+}
+
+async function evaluateExpectation(
+  exp: APIExpectation,
+  singleRuns: CISingleRunResult[],
+): Promise<CIExpectationResult> {
+  switch (exp.type) {
+    case 'token-budget':
+      return evaluateTokenBudget(exp, singleRuns)
+    case 'latency-budget':
+      return evaluateLatencyBudget(exp, singleRuns)
+    case 'output-contains':
+      return evaluateOutputContains(exp, singleRuns)
+    case 'output-schema':
+      return evaluateOutputSchema(exp, singleRuns)
+    case 'tool-called':
+      return evaluateToolCalled(exp, singleRuns)
+    case 'determinism':
+      return evaluateDeterminism(exp, singleRuns)
+    case 'llm-judge':
+      return evaluateLLMJudge(exp, singleRuns)
+    default:
+      return {
+        expectationId: exp.id,
+        type: exp.type,
+        passed: false,
+        detail: `Unknown expectation type: ${exp.type}`,
+      }
+  }
+}
+
+function evaluateTokenBudget(exp: APIExpectation, runs: CISingleRunResult[]): CIExpectationResult {
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+  let allPassed = true
+
+  for (const run of runs) {
+    let passed = true
+    const details: string[] = []
+
+    if (exp.max_tokens_per_run != null && run.totalTokens > exp.max_tokens_per_run) {
+      passed = false
+      details.push(`total ${run.totalTokens} > max ${exp.max_tokens_per_run}`)
+    }
+    perRun[run.runIndex] = { passed, detail: details.join('; ') || undefined }
+    if (!passed) allPassed = false
+  }
+
+  if (exp.max_total_tokens != null) {
+    const total = runs.reduce((sum, r) => sum + r.totalTokens, 0)
+    if (total > exp.max_total_tokens) {
+      allPassed = false
+    }
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'token-budget',
+    passed: allPassed,
+    detail: allPassed ? undefined : 'Token budget exceeded.',
+    perRun,
+  }
+}
+
+function evaluateLatencyBudget(exp: APIExpectation, runs: CISingleRunResult[]): CIExpectationResult {
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+  let allPassed = true
+
+  for (const run of runs) {
+    let passed = true
+    if (exp.max_duration_ms != null && run.durationMs > exp.max_duration_ms) {
+      passed = false
+    }
+    perRun[run.runIndex] = { passed, detail: passed ? undefined : `${run.durationMs}ms > ${exp.max_duration_ms}ms` }
+    if (!passed) allPassed = false
+  }
+
+  if (exp.max_total_duration_ms != null) {
+    const total = runs.reduce((sum, r) => sum + r.durationMs, 0)
+    if (total > exp.max_total_duration_ms) {
+      allPassed = false
+    }
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'latency-budget',
+    passed: allPassed,
+    detail: allPassed ? undefined : 'Latency budget exceeded.',
+    perRun,
+  }
+}
+
+function evaluateOutputContains(exp: APIExpectation, runs: CISingleRunResult[]): CIExpectationResult {
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+  let allPassed = true
+
+  for (const run of runs) {
+    const outputStr = typeof run.output === 'string'
+      ? run.output
+      : JSON.stringify(run.output ?? '')
+
+    const haystack = exp.case_insensitive ? outputStr.toLowerCase() : outputStr
+    let passed = true
+
+    if (exp.contains_text) {
+      const needle = exp.case_insensitive ? exp.contains_text.toLowerCase() : exp.contains_text
+      if (!haystack.includes(needle)) passed = false
+    }
+    if (exp.not_contains_text) {
+      const needle = exp.case_insensitive ? exp.not_contains_text.toLowerCase() : exp.not_contains_text
+      if (haystack.includes(needle)) passed = false
+    }
+
+    perRun[run.runIndex] = { passed, detail: passed ? undefined : 'Output text check failed.' }
+    if (!passed) allPassed = false
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'output-contains',
+    passed: allPassed,
+    detail: allPassed ? undefined : 'Output contains check failed.',
+    perRun,
+  }
+}
+
+function evaluateOutputSchema(exp: APIExpectation, runs: CISingleRunResult[]): CIExpectationResult {
+  // Basic JSON schema validation: check that output is valid JSON matching the schema's type/required fields
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+  let allPassed = true
+
+  const schema = exp.json_schema as Record<string, unknown> | null
+
+  for (const run of runs) {
+    if (!schema) {
+      perRun[run.runIndex] = { passed: true }
+      continue
+    }
+
+    let output = run.output
+    if (typeof output === 'string') {
+      try { output = JSON.parse(output) } catch {
+        perRun[run.runIndex] = { passed: false, detail: 'Output is not valid JSON.' }
+        allPassed = false
+        continue
+      }
+    }
+
+    // Check type
+    const schemaType = schema.type as string | undefined
+    let passed = true
+
+    if (schemaType === 'object' && (typeof output !== 'object' || output === null || Array.isArray(output))) {
+      passed = false
+    } else if (schemaType === 'array' && !Array.isArray(output)) {
+      passed = false
+    } else if (schemaType === 'string' && typeof output !== 'string') {
+      passed = false
+    }
+
+    // Check required fields
+    if (passed && schemaType === 'object' && Array.isArray(schema.required)) {
+      for (const key of schema.required as string[]) {
+        if (!(key in (output as Record<string, unknown>))) {
+          passed = false
+          break
+        }
+      }
+    }
+
+    perRun[run.runIndex] = { passed, detail: passed ? undefined : 'Output does not match schema.' }
+    if (!passed) allPassed = false
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'output-schema',
+    passed: allPassed,
+    detail: allPassed ? undefined : 'Output schema check failed.',
+    perRun,
+  }
+}
+
+function evaluateToolCalled(exp: APIExpectation, runs: CISingleRunResult[]): CIExpectationResult {
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+  let allPassed = true
+
+  for (const run of runs) {
+    // Extract tool calls from trace
+    const trace = run.trace as { events?: Array<{ type: string; name: string }> } | null
+    const toolNames = trace?.events
+      ?.filter((e) => e.type === 'tool')
+      .map((e) => e.name) ?? []
+
+    let passed = true
+
+    // Check required tools
+    if (exp.required_tools?.length) {
+      for (const required of exp.required_tools) {
+        if (!toolNames.includes(required)) {
+          passed = false
+          break
+        }
+      }
+    }
+
+    // Check forbidden tools
+    if (exp.forbidden_tools?.length) {
+      for (const forbidden of exp.forbidden_tools) {
+        if (toolNames.includes(forbidden)) {
+          passed = false
+          break
+        }
+      }
+    }
+
+    perRun[run.runIndex] = { passed, detail: passed ? undefined : `Tools called: [${toolNames.join(', ')}]` }
+    if (!passed) allPassed = false
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'tool-called',
+    passed: allPassed,
+    detail: allPassed ? undefined : 'Tool call check failed.',
+    perRun,
+  }
+}
+
+function evaluateDeterminism(exp: APIExpectation, runs: CISingleRunResult[]): CIExpectationResult {
+  if (runs.length < 2) {
+    return {
+      expectationId: exp.id,
+      type: 'determinism',
+      passed: true,
+      detail: 'Only one run — determinism check skipped.',
+    }
+  }
+
+  // Compare outputs pairwise
+  const outputs = runs.map((r) =>
+    typeof r.output === 'string' ? r.output : JSON.stringify(r.output ?? '')
+  )
+
+  const reference = outputs[0]
+  let allSame = true
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+
+  for (let i = 0; i < runs.length; i++) {
+    const same = outputs[i] === reference
+    perRun[runs[i].runIndex] = { passed: same, detail: same ? undefined : 'Output differs from run 1.' }
+    if (!same) allSame = false
+  }
+
+  // If similarity_threshold is set, do a basic character-level similarity check
+  if (!allSame && exp.similarity_threshold != null) {
+    const threshold = exp.similarity_threshold
+    let allAboveThreshold = true
+    for (let i = 1; i < outputs.length; i++) {
+      const similarity = computeStringSimilarity(reference, outputs[i])
+      if (similarity < threshold) {
+        allAboveThreshold = false
+        perRun[runs[i].runIndex] = {
+          passed: false,
+          detail: `Similarity ${(similarity * 100).toFixed(1)}% < ${(threshold * 100).toFixed(1)}%`,
+        }
+      } else {
+        perRun[runs[i].runIndex] = { passed: true }
+      }
+    }
+    return {
+      expectationId: exp.id,
+      type: 'determinism',
+      passed: allAboveThreshold,
+      detail: allAboveThreshold ? undefined : 'Outputs are not sufficiently similar.',
+      perRun,
+    }
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'determinism',
+    passed: allSame,
+    detail: allSame ? undefined : 'Outputs are not identical across runs.',
+    perRun,
+  }
+}
+
+function computeStringSimilarity(a: string, b: string): number {
+  if (a === b) return 1
+  if (!a.length || !b.length) return 0
+  // Simple character overlap ratio
+  const longer = a.length >= b.length ? a : b
+  const shorter = a.length < b.length ? a : b
+  let matches = 0
+  for (let i = 0; i < shorter.length; i++) {
+    if (shorter[i] === longer[i]) matches++
+  }
+  return matches / longer.length
+}
+
+async function evaluateLLMJudge(exp: APIExpectation, runs: CISingleRunResult[]): Promise<CIExpectationResult> {
+  if (!exp.judge_prompt) {
+    return {
+      expectationId: exp.id,
+      type: 'llm-judge',
+      passed: false,
+      detail: 'LLM judge expectation requires judge_prompt.',
+    }
+  }
+
+  const perRun: Record<number, { passed: boolean; detail?: string }> = {}
+  let allPassed = true
+  const threshold = exp.judge_score_threshold ?? 7
+
+  for (const run of runs) {
+    const outputStr = typeof run.output === 'string'
+      ? run.output
+      : JSON.stringify(run.output ?? '')
+
+    const evalPrompt = `${exp.judge_prompt}
+
+Output to evaluate:
+${outputStr}
+
+Score this output on a scale of 0-10. Respond with only the number.`
+
+    try {
+      const provider = (exp.judge_provider as 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi') || 'openai'
+      const result = await callProviderLLM(
+        evalPrompt,
+        { provider, model: exp.judge_model ?? undefined },
+        'You are an expert test judge. Return only a number between 0 and 10.',
+        16,
+        0,
+      )
+
+      const score = parseFloat(result.content.match(/-?\d+(?:\.\d+)?/)?.[0] ?? '')
+      if (isNaN(score)) {
+        perRun[run.runIndex] = { passed: false, detail: `Could not parse score from: "${result.content}"` }
+        allPassed = false
+      } else {
+        const passed = score >= threshold
+        perRun[run.runIndex] = { passed, detail: `Score: ${score}/${threshold}` }
+        if (!passed) allPassed = false
+      }
+    } catch (err) {
+      perRun[run.runIndex] = {
+        passed: false,
+        detail: `LLM judge error: ${err instanceof Error ? err.message : String(err)}`,
+      }
+      allPassed = false
+    }
+  }
+
+  return {
+    expectationId: exp.id,
+    type: 'llm-judge',
+    passed: allPassed,
+    detail: allPassed ? undefined : 'LLM judge check failed.',
+    perRun,
+  }
+}
+
+// ─── Pass/Fail Threshold ─────────────────────────────────────
+
+function determinePassFail(
+  passThreshold: string,
+  singleRuns: CISingleRunResult[],
+  expectationResults: CIExpectationResult[],
+): boolean {
+  // Check single run pass/fail
+  const runsPassed = singleRuns.filter((r) => r.passed).length
+  const totalRuns = singleRuns.length
+
+  let runsPass: boolean
+  switch (passThreshold) {
+    case 'all':
+      runsPass = runsPassed === totalRuns
+      break
+    case 'majority':
+      runsPass = runsPassed > totalRuns / 2
+      break
+    case 'any':
+      runsPass = runsPassed > 0
+      break
+    default:
+      runsPass = runsPassed === totalRuns
+  }
+
+  // All expectations must pass regardless of threshold
+  const expectationsPass = expectationResults.every((e) => e.passed)
+
+  return runsPass && expectationsPass
+}