elasticdash-test 0.1.18 → 0.1.19-alpha-2

package/src/ci/git-info.ts

@@ -0,0 +1,66 @@
+// ─── CI Environment Auto-Detection ──────────────────────────
+
+export interface GitInfo {
+  branch?: string
+  commit?: string
+  commitMessage?: string
+  prNumber?: number
+  prUrl?: string
+}
+
+/**
+ * Auto-detect git info from CI environment variables.
+ * Supports GitHub Actions, GitLab CI, and generic CI.
+ */
+export function detectGitInfo(): GitInfo {
+  const env = process.env
+
+  // GitHub Actions
+  if (env.GITHUB_ACTIONS === 'true') {
+    const prNumber = env.GITHUB_EVENT_NAME === 'pull_request'
+      ? parseInt(env.GITHUB_REF?.match(/refs\/pull\/(\d+)/)?.[1] ?? '', 10) || undefined
+      : undefined
+
+    const repo = env.GITHUB_REPOSITORY // e.g. "owner/repo"
+    const prUrl = prNumber && repo
+      ? `https://github.com/${repo}/pull/${prNumber}`
+      : undefined
+
+    return {
+      branch: env.GITHUB_HEAD_REF || env.GITHUB_REF_NAME,
+      commit: env.GITHUB_SHA,
+      commitMessage: env.GITHUB_COMMIT_MESSAGE,
+      prNumber,
+      prUrl,
+    }
+  }
+
+  // GitLab CI
+  if (env.GITLAB_CI === 'true') {
+    const prNumber = env.CI_MERGE_REQUEST_IID
+      ? parseInt(env.CI_MERGE_REQUEST_IID, 10) || undefined
+      : undefined
+
+    return {
+      branch: env.CI_MERGE_REQUEST_SOURCE_BRANCH_NAME || env.CI_COMMIT_BRANCH,
+      commit: env.CI_COMMIT_SHA,
+      commitMessage: env.CI_COMMIT_MESSAGE,
+      prNumber,
+      prUrl: env.CI_MERGE_REQUEST_PROJECT_URL && prNumber
+        ? `${env.CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${prNumber}`
+        : undefined,
+    }
+  }
+
+  // Generic CI (CircleCI, Travis, Bitbucket Pipelines, etc.)
+  return {
+    branch: env.CIRCLE_BRANCH || env.TRAVIS_BRANCH || env.BITBUCKET_BRANCH || env.CI_BRANCH,
+    commit: env.CIRCLE_SHA1 || env.TRAVIS_COMMIT || env.BITBUCKET_COMMIT || env.CI_COMMIT_SHA,
+    commitMessage: env.CI_COMMIT_MESSAGE,
+    prNumber: env.CIRCLE_PULL_REQUEST
+      ? parseInt(env.CIRCLE_PULL_REQUEST.split('/').pop() ?? '', 10) || undefined
+      : env.TRAVIS_PULL_REQUEST && env.TRAVIS_PULL_REQUEST !== 'false'
+        ? parseInt(env.TRAVIS_PULL_REQUEST, 10) || undefined
+        : undefined,
+  }
+}

package/src/ci/index.ts

@@ -0,0 +1,5 @@
+export { runCI } from './runner.js'
+export { fetchTestGroups, submitTestRun, createBatch } from './api-client.js'
+export { detectGitInfo } from './git-info.js'
+export type { CIRunConfig, CIRunSummary, CITestResult, CISingleRunResult, CIExpectationResult } from './types.js'
+export type { GitInfo } from './git-info.js'

package/src/ci/runner.ts

@@ -0,0 +1,198 @@
+import chalk from 'chalk'
+import { fetchTestGroups, submitTestRun, createBatch } from './api-client.js'
+import { executeTest } from './executor.js'
+import { detectGitInfo } from './git-info.js'
+import type { CIRunConfig, CIRunSummary, CITestResult } from './types.js'
+
+// ─── CI Runner ───────────────────────────────────────────────
+// Orchestrates the full CI/CD test flow:
+//   1. Fetch test groups from backend
+//   2. Execute each test
+//   3. Submit results
+//   4. Create batch
+//   5. Print summary
+
+export async function runCI(config: CIRunConfig): Promise<CIRunSummary> {
+  const { serverUrl, apiKey } = config
+  const cwd = process.cwd()
+
+  // Merge explicit git info with auto-detected CI env
+  const detected = detectGitInfo()
+  const gitBranch = config.gitBranch || detected.branch
+  const gitCommit = config.gitCommit || detected.commit
+  const gitCommitMessage = config.gitCommitMessage || detected.commitMessage
+  const gitPrNumber = config.gitPrNumber || detected.prNumber
+  const gitPrUrl = config.gitPrUrl || detected.prUrl
+  const triggeredBy = config.triggeredBy || 'ci'
+
+  const overallStart = Date.now()
+
+  // Step 1: Fetch test groups
+  console.log(chalk.cyan('\n[elasticdash ci] Fetching test groups...'))
+
+  const testGroups = await fetchTestGroups(serverUrl, apiKey, {
+    workflowName: config.workflowName,
+    tags: config.tags,
+  })
+
+  if (testGroups.length === 0) {
+    console.log(chalk.yellow('[elasticdash ci] No active test groups found.'))
+    return {
+      total: 0, passed: 0, failed: 0, skipped: 0,
+      durationMs: Date.now() - overallStart,
+      batchId: null,
+      results: [],
+    }
+  }
+
+  const totalTests = testGroups.reduce((sum, g) => sum + g.tests.length, 0)
+  console.log(chalk.cyan(`[elasticdash ci] Found ${testGroups.length} test group(s), ${totalTests} test(s) total.\n`))
+
+  // Step 2 & 3: Execute and submit
+  const allResults: CITestResult[] = []
+  const runIds: number[] = []
+
+  for (const group of testGroups) {
+    console.log(chalk.white.bold(`  ${group.name}`) + chalk.gray(` (${group.tests.length} tests)`))
+
+    for (const test of group.tests) {
+      const testLabel = test.name || `${test.test_type}:${test.target_step_name || 'unnamed'}`
+      process.stdout.write(chalk.gray(`    ${testLabel} ... `))
+
+      const testStart = Date.now()
+      let testResult: CITestResult
+
+      try {
+        const execution = await executeTest(test, cwd)
+
+        // Submit result to backend
+        const payload = {
+          testGroupTestId: test.id,
+          triggeredBy,
+          passed: execution.passed,
+          summary: execution.passed ? 'Passed' : 'Failed',
+          gitBranch, gitCommit, gitCommitMessage, gitPrNumber, gitPrUrl,
+          singleRuns: execution.singleRuns,
+          expectationResults: execution.expectationResults,
+          startedAt: new Date(testStart).toISOString(),
+          completedAt: new Date().toISOString(),
+        }
+
+        let runId: number | null = null
+        try {
+          const submitted = await submitTestRun(serverUrl, apiKey, group.id, payload)
+          runId = submitted.id
+          runIds.push(runId)
+        } catch (submitErr) {
+          // Non-fatal: log but don't fail the test
+          console.error(chalk.yellow(`\n    [warn] Failed to submit result: ${submitErr instanceof Error ? submitErr.message : String(submitErr)}`))
+        }
+
+        testResult = {
+          testGroupId: group.id,
+          testGroupName: group.name,
+          testId: test.id,
+          testName: test.name,
+          testType: test.test_type,
+          passed: execution.passed,
+          runId,
+          singleRuns: execution.singleRuns,
+          expectationResults: execution.expectationResults,
+          durationMs: execution.durationMs,
+        }
+      } catch (err) {
+        testResult = {
+          testGroupId: group.id,
+          testGroupName: group.name,
+          testId: test.id,
+          testName: test.name,
+          testType: test.test_type,
+          passed: false,
+          runId: null,
+          singleRuns: [],
+          expectationResults: [],
+          error: err instanceof Error ? err.message : String(err),
+          durationMs: Date.now() - testStart,
+        }
+      }
+
+      allResults.push(testResult)
+
+      // Print result
+      const durationStr = chalk.gray(`(${testResult.durationMs}ms)`)
+      if (testResult.passed) {
+        console.log(chalk.green('PASS') + ` ${durationStr}`)
+      } else {
+        console.log(chalk.red('FAIL') + ` ${durationStr}`)
+        if (testResult.error) {
+          console.log(chalk.red(`      ${testResult.error}`))
+        }
+        // Show failed expectations
+        for (const exp of testResult.expectationResults) {
+          if (!exp.passed) {
+            console.log(chalk.red(`      [${exp.type}] ${exp.detail || 'failed'}`))
+          }
+        }
+      }
+    }
+
+    console.log() // blank line between groups
+  }
+
+  // Step 4: Create batch
+  let batchId: number | null = null
+  if (runIds.length > 0) {
+    try {
+      const passedCount = allResults.filter((r) => r.passed).length
+      const allPassed = passedCount === allResults.length
+
+      const batch = await createBatch(serverUrl, apiKey, {
+        testGroupRunIds: runIds,
+        status: allPassed ? 'success' : 'failed',
+        passed: allPassed,
+        summary: `CI: ${passedCount}/${allResults.length} passed`,
+        gitBranch, gitCommit,
+        startedAt: new Date(overallStart).toISOString(),
+        completedAt: new Date().toISOString(),
+      })
+      batchId = batch.id
+    } catch {
+      // Non-fatal
+    }
+  }
+
+  // Step 5: Print summary
+  const passed = allResults.filter((r) => r.passed).length
+  const failed = allResults.filter((r) => !r.passed).length
+  const durationMs = Date.now() - overallStart
+
+  console.log(chalk.white.bold('─'.repeat(50)))
+  console.log(chalk.white.bold('Summary'))
+  console.log(chalk.white.bold('─'.repeat(50)))
+  console.log(`  Total:    ${allResults.length}`)
+  console.log(`  ${chalk.green(`Passed:   ${passed}`)}`)
+  if (failed > 0) {
+    console.log(`  ${chalk.red(`Failed:   ${failed}`)}`)
+  }
+  console.log(`  Duration: ${(durationMs / 1000).toFixed(1)}s`)
+  if (batchId) {
+    console.log(`  Batch ID: ${batchId}`)
+  }
+  console.log(chalk.white.bold('─'.repeat(50)))
+
+  if (failed > 0) {
+    console.log(chalk.red(`\n[elasticdash ci] ${failed} test(s) failed.\n`))
+  } else {
+    console.log(chalk.green(`\n[elasticdash ci] All tests passed.\n`))
+  }
+
+  return {
+    total: allResults.length,
+    passed,
+    failed,
+    skipped: 0,
+    durationMs,
+    batchId,
+    results: allResults,
+  }
+}

package/src/ci/types.ts

@@ -0,0 +1,115 @@
+// ─── CI Runner Types ─────────────────────────────────────────
+
+export interface CIRunConfig {
+  serverUrl: string
+  apiKey: string
+  workflowName?: string
+  tags?: string[]
+  triggeredBy?: 'ci' | 'api'
+  gitBranch?: string
+  gitCommit?: string
+  gitCommitMessage?: string
+  gitPrNumber?: number
+  gitPrUrl?: string
+}
+
+export interface CITestResult {
+  testGroupId: number
+  testGroupName: string
+  testId: number
+  testName: string | null
+  testType: string
+  passed: boolean
+  runId: number | null
+  singleRuns: CISingleRunResult[]
+  expectationResults: CIExpectationResult[]
+  error?: string
+  durationMs: number
+}
+
+export interface CISingleRunResult {
+  runIndex: number
+  passed: boolean
+  durationMs: number
+  inputTokens: number
+  outputTokens: number
+  totalTokens: number
+  output: unknown
+  trace: unknown
+  error?: string
+}
+
+export interface CIExpectationResult {
+  expectationId: number
+  type: string
+  passed: boolean
+  detail?: string
+  perRun?: Record<number, { passed: boolean; detail?: string }>
+}
+
+export interface CIRunSummary {
+  total: number
+  passed: number
+  failed: number
+  skipped: number
+  durationMs: number
+  batchId: number | null
+  results: CITestResult[]
+}
+
+// ─── API Response Types ─────────────────────────────────────
+
+export interface APITestGroup {
+  id: number
+  name: string
+  description: string | null
+  project_id: number
+  workflow_name: string
+  trace_file: unknown
+  status: string
+  tags: string[]
+  tests: APITestGroupTest[]
+}
+
+export interface APITestGroupTest {
+  id: number
+  test_group_id: number
+  name: string | null
+  description: string | null
+  test_type: 'single-step' | 'full-flow'
+  target_step_index: number | null
+  target_step_type: string | null
+  target_step_name: string | null
+  mock_input: unknown
+  workflow_input: unknown
+  frozen_events: unknown[]
+  tool_mocks: Record<string, unknown>
+  prompt_mocks: Record<string, string>
+  run_count: number
+  pass_threshold: string
+  timeout_ms: number
+  sort_order: number
+  expectations: APIExpectation[]
+}
+
+export interface APIExpectation {
+  id: number
+  test_group_test_id: number
+  type: string
+  judge_prompt: string | null
+  judge_model: string | null
+  judge_provider: string | null
+  judge_score_threshold: number | null
+  max_total_tokens: number | null
+  max_tokens_per_run: number | null
+  max_duration_ms: number | null
+  max_total_duration_ms: number | null
+  contains_text: string | null
+  not_contains_text: string | null
+  case_insensitive: boolean
+  json_schema: unknown
+  similarity_threshold: number | null
+  required_tools: string[]
+  forbidden_tools: string[]
+  tool_call_rules: unknown
+}

package/src/cli.ts

@@ -346,6 +346,61 @@ async function bootstrap(): Promise<void> {
       process.once('SIGTERM', cleanup)
     })
 
+  // elasticdash ci
+  program
+    .command('ci')
+    .description('Run CI/CD tests — fetch test groups from backend, execute tests, submit results')
+    .option('--server <url>', 'ElasticDash backend API URL', process.env.ELASTICDASH_API_URL)
+    .option('--api-key <key>', 'Project API key', process.env.ELASTICDASH_API_KEY)
+    .option('--workflow <name>', 'Filter test groups by workflow name')
+    .option('--tags <tags>', 'Filter test groups by tags (comma-separated)')
+    .option('--triggered-by <source>', 'Trigger source', 'ci')
+    .option('--git-branch <branch>', 'Git branch name')
+    .option('--git-commit <sha>', 'Git commit SHA')
+    .option('--git-commit-message <msg>', 'Git commit message')
+    .option('--git-pr-number <number>', 'Pull request number', (v) => Number(v))
+    .option('--git-pr-url <url>', 'Pull request URL')
+    .action(async (options: {
+      server?: string
+      apiKey?: string
+      workflow?: string
+      tags?: string
+      triggeredBy?: string
+      gitBranch?: string
+      gitCommit?: string
+      gitCommitMessage?: string
+      gitPrNumber?: number
+      gitPrUrl?: string
+    }) => {
+      const serverUrl = options.server
+      if (!serverUrl) {
+        console.error('[elasticdash] Error: --server or ELASTICDASH_API_URL is required')
+        process.exit(1)
+      }
+
+      const apiKey = options.apiKey
+      if (!apiKey) {
+        console.error('[elasticdash] Error: --api-key or ELASTICDASH_API_KEY is required')
+        process.exit(1)
+      }
+
+      const { runCI } = await import('./ci/runner.js')
+      const summary = await runCI({
+        serverUrl,
+        apiKey,
+        workflowName: options.workflow,
+        tags: options.tags ? options.tags.split(',').map(s => s.trim()).filter(Boolean) : undefined,
+        triggeredBy: (options.triggeredBy as 'ci' | 'api') || 'ci',
+        gitBranch: options.gitBranch,
+        gitCommit: options.gitCommit,
+        gitCommitMessage: options.gitCommitMessage,
+        gitPrNumber: options.gitPrNumber,
+        gitPrUrl: options.gitPrUrl,
+      })
+
+      process.exit(summary.failed > 0 ? 1 : 0)
+    })
+
   // elasticdash portal
   program
     .command('portal')

package/src/dashboard-server.ts

@@ -3,7 +3,7 @@ import path from 'node:path'
 import { existsSync, readFileSync, readdirSync, statSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'
 import { spawn } from 'node:child_process'
 import { pathToFileURL, fileURLToPath } from 'url'
-import { randomUUID } from 'node:crypto'
+import { randomUUID, createCipheriv, createDecipheriv, randomBytes } from 'node:crypto'
 import { callProviderLLM } from './matchers/index.js'
 import { startTraceSession } from './trace-adapter/context.js'
 import type { WorkflowTrace, WorkflowEvent } from './capture/event.js'
@@ -144,18 +144,52 @@ interface ValidateWorkflowResult {
   error?: string
 }
 
+// ─── Snapshot Encryption (opt-in via ELASTICDASH_SNAPSHOT_ENCRYPTION_KEY) ────
+
+const SNAPSHOT_ENCRYPTION_KEY = process.env.ELASTICDASH_SNAPSHOT_ENCRYPTION_KEY
+  ? Buffer.from(process.env.ELASTICDASH_SNAPSHOT_ENCRYPTION_KEY, 'hex')
+  : null
+
+function encryptSnapshot(data: string): string {
+  if (!SNAPSHOT_ENCRYPTION_KEY || SNAPSHOT_ENCRYPTION_KEY.length !== 32) return data
+  const iv = randomBytes(16)
+  const cipher = createCipheriv('aes-256-gcm', SNAPSHOT_ENCRYPTION_KEY, iv)
+  const encrypted = Buffer.concat([cipher.update(data, 'utf8'), cipher.final()])
+  const authTag = cipher.getAuthTag()
+  return `${iv.toString('hex')}:${authTag.toString('hex')}:${encrypted.toString('hex')}`
+}
+
+function decryptSnapshot(data: string): string {
+  if (!SNAPSHOT_ENCRYPTION_KEY || SNAPSHOT_ENCRYPTION_KEY.length !== 32) return data
+  const parts = data.split(':')
+  if (parts.length !== 3) return data // not encrypted, return as-is
+  const [ivHex, authTagHex, encryptedHex] = parts
+  try {
+    const iv = Buffer.from(ivHex, 'hex')
+    const authTag = Buffer.from(authTagHex, 'hex')
+    const encrypted = Buffer.from(encryptedHex, 'hex')
+    const decipher = createDecipheriv('aes-256-gcm', SNAPSHOT_ENCRYPTION_KEY, iv)
+    decipher.setAuthTag(authTag)
+    return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8')
+  } catch {
+    return data // decryption failed, return raw (may be unencrypted legacy data)
+  }
+}
+
 function saveSnapshot(cwd: string, workflowTrace: WorkflowTrace): string {
   const dir = path.join(cwd, '.temp', 'snapshots')
   mkdirSync(dir, { recursive: true })
   const id = workflowTrace.traceId
-  writeFileSync(path.join(dir, `${id}.json`), JSON.stringify(workflowTrace), 'utf8')
+  const content = encryptSnapshot(JSON.stringify(workflowTrace))
+  writeFileSync(path.join(dir, `${id}.json`), content, 'utf8')
   return id
 }
 
 function loadSnapshot(cwd: string, snapshotId: string): WorkflowTrace | null {
   try {
     const file = path.join(cwd, '.temp', 'snapshots', `${snapshotId}.json`)
-    return JSON.parse(readFileSync(file, 'utf8')) as WorkflowTrace
+    const raw = readFileSync(file, 'utf8')
+    return JSON.parse(decryptSnapshot(raw)) as WorkflowTrace
   } catch {
     return null
   }

package/src/index.ts

@@ -104,6 +104,13 @@ export { startPortalServer } from './portal-server.js'
 export { executePortalTask } from './portal-executor.js'
 export type { PortalTask, PortalTaskResult, PortalServerOptions, PortalServerHandle, PortalStatus } from './types/portal.js'
 
+// CI runner (programmatic API)
+export { runCI } from './ci/runner.js'
+export { fetchTestGroups, submitTestRun, createBatch } from './ci/api-client.js'
+export { detectGitInfo } from './ci/git-info.js'
+export type { CIRunConfig, CIRunSummary, CITestResult, CISingleRunResult, CIExpectationResult } from './ci/types.js'
+export type { GitInfo } from './ci/git-info.js'
+
 // ─── Eager auto-init ────────────────────────────────────────
 // When ELASTICDASH_API_KEY is set, automatically initialise observability mode
 // at import time so the socket connection is established immediately (e.g. for

package/src/portal-server.ts

@@ -180,6 +180,13 @@ export async function startPortalServer(options: PortalServerOptions): Promise<P
     if (isAllowedOrigin(req, allowedHosts)) {
       // If API key is configured but not provided, still reject
       if (apiKey) {
+        console.warn(JSON.stringify({
+          event: 'auth.failure',
+          reason: 'invalid_api_key',
+          ip: req.ip || req.headers['x-forwarded-for'] || 'unknown',
+          userAgent: req.headers['user-agent'] || 'unknown',
+          timestamp: new Date().toISOString(),
+        }))
         res.status(401).json({ ok: false, error: 'unauthorized — valid API key required' })
         return
       }
@@ -188,7 +195,14 @@ export async function startPortalServer(options: PortalServerOptions): Promise<P
 
     // Origin not in allowlist and no valid API key
     const origin = req.headers.origin ?? req.headers.referer ?? req.ip ?? 'unknown'
-    console.warn(`[elasticdash portal] Blocked request from disallowed origin: ${origin}`)
+    console.warn(JSON.stringify({
+      event: 'auth.failure',
+      reason: 'origin_not_allowed',
+      origin,
+      ip: req.ip || req.headers['x-forwarded-for'] || 'unknown',
+      userAgent: req.headers['user-agent'] || 'unknown',
+      timestamp: new Date().toISOString(),
+    }))
     res.status(403).json({ ok: false, error: 'forbidden — origin not in allowlist' })
   }
 

package/src/telemetry-batcher.ts

@@ -1,4 +1,5 @@
 import { Readable } from 'node:stream'
+import { randomUUID } from 'node:crypto'
 import type { WorkflowEvent } from './capture/event.js'
 import { getOriginalFetch } from './interceptors/http.js'
 import { debugLog } from './utils/debug.js'
@@ -95,7 +96,10 @@ export class TelemetryBatcher {
     }
     const maxRetries = 3
     const url = `${this.serverUrl}/api/observability/events`
-    const headers: Record<string, string> = { 'Content-Type': 'application/json' }
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'X-Correlation-ID': randomUUID(),
+    }
     if (this.apiKey) headers['Authorization'] = `Bearer ${this.apiKey}`
 
     try {