elasticdash-sdk 0.2.0

package/src/ci/git-info.ts

@@ -0,0 +1,127 @@
+import { execSync } from 'node:child_process'
+
+// ─── CI Environment Auto-Detection ──────────────────────────
+
+export interface GitInfo {
+  branch?: string
+  commit?: string
+  commitMessage?: string
+  baseBranch?: string
+  prNumber?: number
+  prUrl?: string
+  repo?: string
+  ciProvider?: string
+  ciRunUrl?: string
+}
+
+/**
+ * Auto-detect git info from CI environment variables.
+ * Supports GitHub Actions, GitLab CI, CircleCI, Buildkite, and local git.
+ */
+export function detectGitInfo(): GitInfo {
+  const env = process.env
+
+  // GitHub Actions
+  if (env.GITHUB_ACTIONS === 'true') {
+    const prNumber = env.GITHUB_EVENT_NAME === 'pull_request'
+      ? parseInt(env.GITHUB_REF?.match(/refs\/pull\/(\d+)/)?.[1] ?? '', 10) || undefined
+      : undefined
+
+    const repo = env.GITHUB_REPOSITORY
+    const prUrl = prNumber && repo
+      ? `https://github.com/${repo}/pull/${prNumber}`
+      : undefined
+
+    const serverUrl = env.GITHUB_SERVER_URL || 'https://github.com'
+    const ciRunUrl = repo && env.GITHUB_RUN_ID
+      ? `${serverUrl}/${repo}/actions/runs/${env.GITHUB_RUN_ID}`
+      : undefined
+
+    return {
+      branch: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME,
+      commit: env.GITHUB_SHA,
+      commitMessage: env.GITHUB_COMMIT_MESSAGE,
+      baseBranch: env.GITHUB_BASE_REF || undefined,
+      prNumber,
+      prUrl,
+      repo,
+      ciProvider: 'github-actions',
+      ciRunUrl,
+    }
+  }
+
+  // GitLab CI
+  if (env.GITLAB_CI === 'true') {
+    const prNumber = env.CI_MERGE_REQUEST_IID
+      ? parseInt(env.CI_MERGE_REQUEST_IID, 10) || undefined
+      : undefined
+
+    return {
+      branch: env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME || env.CI_COMMIT_BRANCH,
+      commit: env.CI_COMMIT_SHA,
+      commitMessage: env.CI_COMMIT_MESSAGE,
+      baseBranch: env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME || undefined,
+      prNumber,
+      prUrl: env.CI_MERGE_REQUEST_PROJECT_URL && prNumber
+        ? `${env.CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${prNumber}`
+        : undefined,
+      repo: env.CI_PROJECT_PATH,
+      ciProvider: 'gitlab-ci',
+      ciRunUrl: env.CI_JOB_URL,
+    }
+  }
+
+  // CircleCI
+  if (env.CIRCLECI === 'true') {
+    const prNumber = env.CIRCLE_PULL_REQUEST
+      ? parseInt(env.CIRCLE_PULL_REQUEST.split('/').pop() ?? '', 10) || undefined
+      : undefined
+
+    return {
+      branch: env.CIRCLE_BRANCH,
+      commit: env.CIRCLE_SHA1,
+      prNumber,
+      repo: env.CIRCLE_PROJECT_REPONAME,
+      ciProvider: 'circleci',
+      ciRunUrl: env.CIRCLE_BUILD_URL,
+    }
+  }
+
+  // Buildkite
+  if (env.BUILDKITE === 'true') {
+    const prNumber = env.BUILDKITE_PULL_REQUEST && env.BUILDKITE_PULL_REQUEST !== 'false'
+      ? parseInt(env.BUILDKITE_PULL_REQUEST, 10) || undefined
+      : undefined
+
+    return {
+      branch: env.BUILDKITE_BRANCH,
+      commit: env.BUILDKITE_COMMIT,
+      baseBranch: env.BUILDKITE_PULL_REQUEST_BASE_BRANCH || undefined,
+      prNumber,
+      repo: env.BUILDKITE_REPO,
+      ciProvider: 'buildkite',
+      ciRunUrl: env.BUILDKITE_BUILD_URL,
+    }
+  }
+
+  // Local fallback: shell out to git
+  return detectLocalGitInfo()
+}
+
+function detectLocalGitInfo(): GitInfo {
+  try {
+    const commit = execSync('git rev-parse HEAD', { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
+    const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }).trim()
+    return {
+      branch: branch || undefined,
+      commit: commit || undefined,
+      ciProvider: 'local',
+    }
+  } catch {
+    return {
+      branch: 'unknown',
+      commit: 'unknown',
+      ciProvider: 'local',
+    }
+  }
+}

package/src/ci/index.ts

@@ -0,0 +1,5 @@
+export { runCI } from './runner.js'
+export { fetchTestGroups, submitTestRun, createBatch } from './api-client.js'
+export { detectGitInfo } from './git-info.js'
+export type { CIRunConfig, CIRunSummary, CITestResult, CISingleRunResult, CIExpectationResult } from './types.js'
+export type { GitInfo } from './git-info.js'

package/src/ci/measurement.ts

@@ -0,0 +1,25 @@
+import type { EdReplayContext } from './replay.js'
+
+export interface TestMeasurement {
+  duration_ms: number
+  tokens_input?: number
+  tokens_output?: number
+  tokens_total?: number
+}
+
+export function collectMeasurement(ctx: EdReplayContext): TestMeasurement | null {
+  const m = ctx.targetMeasurement
+  if (!m) return null
+
+  const result: TestMeasurement = {
+    duration_ms: m.duration_ms,
+  }
+
+  if (m.tokens) {
+    result.tokens_input = m.tokens.input
+    result.tokens_output = m.tokens.output
+    result.tokens_total = m.tokens.total
+  }
+
+  return result
+}

package/src/ci/replay.ts

@@ -0,0 +1,127 @@
+import { createHash } from 'node:crypto'
+import { AsyncLocalStorage } from 'node:async_hooks'
+import type { DiskTrace, DiskTraceStep } from './trace-schema.js'
+
+// ─── Types ──────────────────────────────────────────────────
+
+export interface ReplayMeasurement {
+  duration_ms: number
+  tokens?: { input: number; output: number; total: number } | null
+}
+
+export interface EdReplayContext {
+  steps: DiskTraceStep[]
+  consumed: boolean[]
+  targetStepId: string
+  targetMeasurement: ReplayMeasurement | null
+}
+
+export class ReplayMissError extends Error {
+  constructor(public callType: string, public callName: string) {
+    super(`replay miss: ${callType}::${callName}`)
+    this.name = 'ReplayMissError'
+  }
+}
+
+// ─── ALS-backed context ─────────────────────────────────────
+
+const g = globalThis as Record<string, unknown>
+const ED_REPLAY_ALS_KEY = '__elasticdash_ed_replay_als__'
+const edReplayAls: AsyncLocalStorage<EdReplayContext | undefined> =
+  (g[ED_REPLAY_ALS_KEY] as AsyncLocalStorage<EdReplayContext | undefined>) ??
+  new AsyncLocalStorage<EdReplayContext | undefined>()
+if (!g[ED_REPLAY_ALS_KEY]) g[ED_REPLAY_ALS_KEY] = edReplayAls
+
+export function getEdReplayContext(): EdReplayContext | undefined {
+  return edReplayAls.getStore()
+}
+
+export function installReplay(ctx: EdReplayContext): void {
+  edReplayAls.enterWith(ctx)
+}
+
+export function uninstallReplay(): void {
+  edReplayAls.enterWith(undefined)
+}
+
+// ─── Context creation ───────────────────────────────────────
+
+export function createReplayContext(trace: DiskTrace, targetStepId: string): EdReplayContext {
+  return {
+    steps: trace.steps,
+    consumed: new Array(trace.steps.length).fill(false),
+    targetStepId,
+    targetMeasurement: null,
+  }
+}
+
+// ─── Input canonicalization & match key ─────────────────────
+
+export function canonicalizeInput(input: unknown): string {
+  return JSON.stringify(sortKeys(input))
+}
+
+function sortKeys(value: unknown): unknown {
+  if (value === null || value === undefined) return value
+  if (Array.isArray(value)) return value.map(sortKeys)
+  if (typeof value === 'number') {
+    // Round floats to 6 decimal places
+    if (!Number.isInteger(value)) {
+      return Math.round(value * 1e6) / 1e6
+    }
+    return value
+  }
+  if (typeof value === 'string') {
+    // Normalize whitespace
+    return value.replace(/\s+/g, ' ').trim()
+  }
+  if (typeof value === 'object') {
+    const sorted: Record<string, unknown> = {}
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      sorted[key] = sortKeys((value as Record<string, unknown>)[key])
+    }
+    return sorted
+  }
+  return value
+}
+
+export function computeMatchKey(type: 'tool_call' | 'ai_call', name: string, input: unknown): string {
+  const canonical = canonicalizeInput(input)
+  const hash = createHash('sha256').update(canonical).digest('hex').slice(0, 16)
+  return `${type}::${name}::${hash}`
+}
+
+// ─── Replay call ────────────────────────────────────────────
+
+export function replayCall(
+  ctx: EdReplayContext,
+  type: 'tool_call' | 'ai_call',
+  name: string,
+  input: unknown,
+): { output: unknown; measurement: ReplayMeasurement | null } {
+  const runtimeKey = computeMatchKey(type, name, input)
+
+  // Find next unconsumed step with matching key
+  for (let i = 0; i < ctx.steps.length; i++) {
+    if (ctx.consumed[i]) continue
+    const step = ctx.steps[i]
+    const stepKey = computeMatchKey(step.type, step.name, step.input)
+    if (stepKey === runtimeKey) {
+      ctx.consumed[i] = true
+
+      const measurement: ReplayMeasurement = {
+        duration_ms: step.duration_ms,
+        tokens: step.tokens ?? null,
+      }
+
+      // Capture measurement if this is the target step
+      if (step.step_id === ctx.targetStepId) {
+        ctx.targetMeasurement = measurement
+      }
+
+      return { output: step.output, measurement }
+    }
+  }
+
+  throw new ReplayMissError(type, name)
+}

package/src/ci/reporters/default.ts

@@ -0,0 +1,50 @@
+import chalk from 'chalk'
+import type { Reporter } from './index.js'
+import type { EdTestResult, EdTestRunResult } from '../ed-runner.js'
+
+export class DefaultReporter implements Reporter {
+  onTestStart(name: string): void {
+    process.stdout.write(chalk.gray(`  ${name} ... `))
+  }
+
+  onTestResult(result: EdTestResult): void {
+    if (result.status === 'pass') {
+      console.log(chalk.green('PASS'))
+      if (result.benchmarkResult) {
+        for (const m of result.benchmarkResult.metrics) {
+          console.log(chalk.gray(`      ${m.name}: ${m.value} (threshold: ${m.threshold}) `) + chalk.green('✓'))
+        }
+      }
+    } else {
+      console.log(chalk.red('FAIL'))
+      if (result.benchmarkResult) {
+        for (const m of result.benchmarkResult.metrics) {
+          const icon = m.passed ? chalk.green('✓') : chalk.red('✗')
+          console.log(chalk.gray(`      ${m.name}: ${m.value} (threshold: ${m.threshold}) `) + icon)
+        }
+      }
+      if (result.failureReason) {
+        console.log(chalk.red(`      → ${result.failureReason}`))
+      }
+    }
+    console.log()
+  }
+
+  onRunComplete(runResult: EdTestRunResult, uploadUrl?: string): void {
+    const passed = runResult.results.filter(r => r.status === 'pass').length
+    const failed = runResult.results.filter(r => r.status === 'fail').length
+    const total = runResult.results.length
+
+    console.log(chalk.white.bold('─'.repeat(45)))
+
+    const parts: string[] = []
+    if (passed > 0) parts.push(chalk.green(`${passed} passed`))
+    if (failed > 0) parts.push(chalk.red(`${failed} failed`))
+    console.log(`${parts.join(', ')}, ${total} total`)
+
+    if (uploadUrl) {
+      console.log(chalk.gray(`Uploaded to ${uploadUrl}`))
+    }
+    console.log()
+  }
+}

package/src/ci/reporters/index.ts

@@ -0,0 +1,21 @@
+import type { EdTestResult, EdTestRunResult } from '../ed-runner.js'
+import { DefaultReporter } from './default.js'
+import { JsonReporter } from './json.js'
+import { JunitReporter } from './junit.js'
+
+export interface Reporter {
+  onTestStart(name: string): void
+  onTestResult(result: EdTestResult): void
+  onRunComplete(runResult: EdTestRunResult, uploadUrl?: string): void
+}
+
+export function createReporter(name: 'default' | 'json' | 'junit'): Reporter {
+  switch (name) {
+    case 'json':
+      return new JsonReporter()
+    case 'junit':
+      return new JunitReporter()
+    default:
+      return new DefaultReporter()
+  }
+}

package/src/ci/reporters/json.ts

@@ -0,0 +1,18 @@
+import type { Reporter } from './index.js'
+import type { EdTestResult, EdTestRunResult } from '../ed-runner.js'
+import { buildUploadPayload } from '../upload-client.js'
+
+export class JsonReporter implements Reporter {
+  onTestStart(_name: string): void {
+    // silent
+  }
+
+  onTestResult(_result: EdTestResult): void {
+    // silent — output everything at the end
+  }
+
+  onRunComplete(runResult: EdTestRunResult, _uploadUrl?: string): void {
+    const payload = buildUploadPayload(runResult)
+    console.log(JSON.stringify(payload, null, 2))
+  }
+}

package/src/ci/reporters/junit.ts

@@ -0,0 +1,61 @@
+import type { Reporter } from './index.js'
+import type { EdTestResult, EdTestRunResult } from '../ed-runner.js'
+
+function escapeXml(str: string): string {
+  return str
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+export class JunitReporter implements Reporter {
+  onTestStart(_name: string): void {
+    // silent
+  }
+
+  onTestResult(_result: EdTestResult): void {
+    // silent — output everything at the end
+  }
+
+  onRunComplete(runResult: EdTestRunResult, _uploadUrl?: string): void {
+    const results = runResult.results
+    const passed = results.filter(r => r.status === 'pass').length
+    const failed = results.filter(r => r.status === 'fail').length
+    const totalTime = results.reduce((sum, r) => sum + r.durationMs, 0) / 1000
+
+    const lines: string[] = [
+      '<?xml version="1.0" encoding="UTF-8"?>',
+      `<testsuites tests="${results.length}" failures="${failed}" errors="0" time="${totalTime.toFixed(3)}">`,
+      `  <testsuite name="ed_tests" tests="${results.length}" failures="${failed}" errors="0" time="${totalTime.toFixed(3)}">`,
+    ]
+
+    for (const r of results) {
+      const time = (r.durationMs / 1000).toFixed(3)
+      // Use duration_ms measurement as the duration attribute if available
+      const metricTime = r.measurement ? (r.measurement.duration_ms / 1000).toFixed(3) : time
+
+      lines.push(`    <testcase name="${escapeXml(r.testName)}" time="${metricTime}">`)
+
+      if (r.status === 'fail') {
+        const isReplayMiss = r.failureReason?.startsWith('replay miss:')
+        const isTimeout = r.failureReason?.startsWith('test timed out')
+        const isExecError = r.failureReason?.startsWith('execution error:')
+        const type = isReplayMiss ? 'ReplayMissError'
+          : isTimeout ? 'TimeoutError'
+          : isExecError ? 'ExecutionError'
+          : 'BenchmarkFailure'
+
+        lines.push(`      <failure type="${type}" message="${escapeXml(r.failureReason ?? 'test failed')}">${escapeXml(r.failureReason ?? '')}</failure>`)
+      }
+
+      lines.push('    </testcase>')
+    }
+
+    lines.push('  </testsuite>')
+    lines.push('</testsuites>')
+
+    console.log(lines.join('\n'))
+  }
+}

package/src/ci/runner.ts

@@ -0,0 +1,208 @@
+import chalk from 'chalk'
+import { fetchTestGroups, submitTestRun, createBatch } from './api-client.js'
+import { executeTest } from './executor.js'
+import { detectGitInfo } from './git-info.js'
+import type { CIRunConfig, CIRunSummary, CITestResult } from './types.js'
+
+// ─── CI Runner ───────────────────────────────────────────────
+// Orchestrates the full CI/CD test flow:
+//   1. Fetch test groups from backend
+//   2. Execute each test
+//   3. Submit results
+//   4. Create batch
+//   5. Print summary
+
+export async function runCI(config: CIRunConfig): Promise<CIRunSummary> {
+  const { serverUrl, apiKey } = config
+  const cwd = process.cwd()
+
+  // Debug: log config so CI logs show what was received
+  console.log(`[elasticdash ci] serverUrl: ${serverUrl || '(not set)'}`)
+  console.log(`[elasticdash ci] apiKey: ${apiKey ? apiKey.substring(0, 7) + '...' + apiKey.substring(apiKey.length - 4) : '(not set)'}`)
+
+  // Merge explicit git info with auto-detected CI env
+  const detected = detectGitInfo()
+  const gitBranch = config.gitBranch || detected.branch
+  const gitCommit = config.gitCommit || detected.commit
+  const gitCommitMessage = config.gitCommitMessage || detected.commitMessage
+  const gitPrNumber = config.gitPrNumber || detected.prNumber
+  const gitPrUrl = config.gitPrUrl || detected.prUrl
+  const triggeredBy = config.triggeredBy || 'ci'
+
+  const overallStart = Date.now()
+
+  // Step 1: Fetch test groups
+  console.log(chalk.cyan('\n[elasticdash ci] Fetching test groups...'))
+
+  const testGroups = await fetchTestGroups(serverUrl, apiKey, {
+    workflowName: config.workflowName,
+    tags: config.tags,
+  })
+
+  if (testGroups.length === 0) {
+    console.log(chalk.yellow('[elasticdash ci] No active test groups found.'))
+    return {
+      total: 0, passed: 0, failed: 0, skipped: 0,
+      durationMs: Date.now() - overallStart,
+      batchId: null,
+      results: [],
+    }
+  }
+
+  const totalTests = testGroups.reduce((sum, g) => sum + g.tests.length, 0)
+  console.log(chalk.cyan(`[elasticdash ci] Found ${testGroups.length} test group(s), ${totalTests} test(s) total.\n`))
+
+  // Step 2 & 3: Execute and submit
+  const allResults: CITestResult[] = []
+  const runIds: number[] = []
+
+  for (const group of testGroups) {
+    console.log(chalk.white.bold(`  ${group.name}`) + chalk.gray(` (${group.tests.length} tests)`))
+
+    for (const test of group.tests) {
+      const testLabel = test.name || `${test.test_type}:${test.target_step_name || 'unnamed'}`
+      process.stdout.write(chalk.gray(`    ${testLabel} ... `))
+
+      const testStart = Date.now()
+      let testResult: CITestResult
+
+      try {
+        const execution = await executeTest(test, cwd)
+
+        // Submit result to backend
+        const payload = {
+          testGroupTestId: test.id,
+          triggeredBy,
+          passed: execution.passed,
+          summary: execution.passed ? 'Passed' : 'Failed',
+          gitBranch, gitCommit, gitCommitMessage, gitPrNumber, gitPrUrl,
+          singleRuns: execution.singleRuns,
+          expectationResults: execution.expectationResults,
+          startedAt: new Date(testStart).toISOString(),
+          completedAt: new Date().toISOString(),
+        }
+
+        let runId: number | null = null
+        try {
+          const submitted = await submitTestRun(serverUrl, apiKey, group.id, payload)
+          runId = submitted.id
+          runIds.push(runId)
+        } catch (submitErr) {
+          // Non-fatal: log but don't fail the test
+          console.error(chalk.yellow(`\n    [warn] Failed to submit result: ${submitErr instanceof Error ? submitErr.message : String(submitErr)}`))
+        }
+
+        testResult = {
+          testGroupId: group.id,
+          testGroupName: group.name,
+          testId: test.id,
+          testName: test.name,
+          testType: test.test_type,
+          passed: execution.passed,
+          runId,
+          singleRuns: execution.singleRuns,
+          expectationResults: execution.expectationResults,
+          durationMs: execution.durationMs,
+        }
+      } catch (err) {
+        testResult = {
+          testGroupId: group.id,
+          testGroupName: group.name,
+          testId: test.id,
+          testName: test.name,
+          testType: test.test_type,
+          passed: false,
+          runId: null,
+          singleRuns: [],
+          expectationResults: [],
+          error: err instanceof Error ? err.message : String(err),
+          durationMs: Date.now() - testStart,
+        }
+      }
+
+      allResults.push(testResult)
+
+      // Print result
+      const durationStr = chalk.gray(`(${testResult.durationMs}ms)`)
+      if (testResult.passed) {
+        console.log(chalk.green('PASS') + ` ${durationStr}`)
+      } else {
+        console.log(chalk.red('FAIL') + ` ${durationStr}`)
+        if (testResult.error) {
+          console.log(chalk.red(`      ${testResult.error}`))
+        }
+        // Show errors from individual single runs
+        for (const run of testResult.singleRuns) {
+          if (!run.passed && run.error) {
+            console.log(chalk.red(`      [run ${run.runIndex}] ${run.error}`))
+          }
+        }
+        // Show failed expectations
+        for (const exp of testResult.expectationResults) {
+          if (!exp.passed) {
+            console.log(chalk.red(`      [${exp.type}] ${exp.detail || 'failed'}`))
+          }
+        }
+      }
+    }
+
+    console.log() // blank line between groups
+  }
+
+  // Step 4: Create batch
+  let batchId: number | null = null
+  if (runIds.length > 0) {
+    try {
+      const passedCount = allResults.filter((r) => r.passed).length
+      const allPassed = passedCount === allResults.length
+
+      const batch = await createBatch(serverUrl, apiKey, {
+        testGroupRunIds: runIds,
+        status: allPassed ? 'success' : 'failed',
+        passed: allPassed,
+        summary: `CI: ${passedCount}/${allResults.length} passed`,
+        gitBranch, gitCommit,
+        startedAt: new Date(overallStart).toISOString(),
+        completedAt: new Date().toISOString(),
+      })
+      batchId = batch.id
+    } catch {
+      // Non-fatal
+    }
+  }
+
+  // Step 5: Print summary
+  const passed = allResults.filter((r) => r.passed).length
+  const failed = allResults.filter((r) => !r.passed).length
+  const durationMs = Date.now() - overallStart
+
+  console.log(chalk.white.bold('─'.repeat(50)))
+  console.log(chalk.white.bold('Summary'))
+  console.log(chalk.white.bold('─'.repeat(50)))
+  console.log(`  Total:    ${allResults.length}`)
+  console.log(`  ${chalk.green(`Passed:   ${passed}`)}`)
+  if (failed > 0) {
+    console.log(`  ${chalk.red(`Failed:   ${failed}`)}`)
+  }
+  console.log(`  Duration: ${(durationMs / 1000).toFixed(1)}s`)
+  if (batchId) {
+    console.log(`  Batch ID: ${batchId}`)
+  }
+  console.log(chalk.white.bold('─'.repeat(50)))
+
+  if (failed > 0) {
+    console.log(chalk.red(`\n[elasticdash ci] ${failed} test(s) failed.\n`))
+  } else {
+    console.log(chalk.green(`\n[elasticdash ci] All tests passed.\n`))
+  }
+
+  return {
+    total: allResults.length,
+    passed,
+    failed,
+    skipped: 0,
+    durationMs,
+    batchId,
+    results: allResults,
+  }
+}

package/src/ci/test-discovery.ts

@@ -0,0 +1,16 @@
+import fg from 'fast-glob'
+import { resolve } from 'node:path'
+
+export interface DiscoverOptions {
+  cwd?: string
+}
+
+export async function discoverTestFiles(options?: DiscoverOptions): Promise<string[]> {
+  const cwd = options?.cwd ?? process.cwd()
+  const files = await fg('**/ed_tests.{ts,js}', {
+    cwd,
+    absolute: true,
+    ignore: ['**/node_modules/**', '**/.git/**', '**/.ed_traces/**'],
+  })
+  return files.sort()
+}