elasticdash-sdk 0.2.0

package/src/ci/upload-client.ts

@@ -0,0 +1,300 @@
+import { randomUUID } from 'node:crypto'
+import { mkdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { getOriginalFetch } from '../interceptors/http.js'
+import { notifyLicenseError } from '../utils/license-error.js'
+import { detectGitInfo } from './git-info.js'
+import type { EdTestRunResult, EdTestResult, EdSingleRunResult } from './ed-runner.js'
+import type { BenchmarkResult } from './benchmark.js'
+
+// ─── Upload payload types (aligned with POST /api/v1/test-runs/create) ──
+
+export interface UploadPayload {
+  schema_version: 1
+  run: {
+    run_id: string
+    started_at: string
+    finished_at: string
+    sdk_version: string
+    git: {
+      commit_sha: string
+      branch: string
+      base_branch?: string
+      pr_number?: number
+      repo?: string
+    }
+    ci: {
+      provider?: string
+      run_url?: string
+    }
+  }
+  results: UploadTestResult[]
+}
+
+export interface UploadTestResult {
+  test_id: string
+  test_name: string
+  status: 'pass' | 'fail'
+  failure_reason?: string
+  trace_ref?: string
+  target?: { type: string; step_id: string }
+  duration_ms: number
+  reruns_count: number
+  input?: unknown
+  output?: unknown
+  metrics: UploadMetric[]
+  reruns?: UploadRerun[]
+}
+
+export interface UploadMetric {
+  name: 'duration_ms' | 'tokens_total' | 'tokens_input' | 'tokens_output' | 'output_contains' | 'output_not_contains' | 'llm_judge'
+  value: number
+  passed: boolean
+  benchmark?: { type: string; threshold: number }
+  /** Per-rerun measurements (when reruns > 1) */
+  measurements?: { rerun_index: number; value: number; passed: boolean }[]
+}
+
+export interface UploadRerun {
+  rerun_index: number
+  input?: unknown
+  output?: unknown
+  started_at?: string
+  finished_at?: string
+}
+
+// ─── Payload assembly ───────────────────────────────────────
+
+export function buildUploadPayload(runResult: EdTestRunResult): UploadPayload {
+  const gitInfo = detectGitInfo()
+
+  return {
+    schema_version: 1,
+    run: {
+      run_id: runResult.runId,
+      started_at: runResult.startedAt,
+      finished_at: runResult.finishedAt,
+      sdk_version: runResult.sdkVersion,
+      git: {
+        commit_sha: gitInfo.commit || 'unknown',
+        branch: gitInfo.branch || 'unknown',
+        base_branch: gitInfo.baseBranch,
+        pr_number: gitInfo.prNumber,
+        repo: gitInfo.repo,
+      },
+      ci: {
+        provider: gitInfo.ciProvider,
+        run_url: gitInfo.ciRunUrl,
+      },
+    },
+    results: runResult.results.map(r => mapTestResult(r)),
+  }
+}
+
+function buildMetricsFromRun(run: EdSingleRunResult, benchmarkResult?: BenchmarkResult | null): UploadMetric[] {
+  const metrics: UploadMetric[] = []
+  const br = run.benchmarkResult ?? benchmarkResult
+
+  if (br) {
+    for (const m of br.metrics) {
+      metrics.push({
+        name: m.name,
+        value: m.value,
+        passed: m.passed,
+        benchmark: { type: 'max', threshold: m.threshold },
+      })
+    }
+  }
+
+  if (run.measurement) {
+    if (!metrics.some(m => m.name === 'duration_ms')) {
+      metrics.push({ name: 'duration_ms', value: run.measurement.duration_ms, passed: true })
+    }
+    if (run.measurement.tokens_input !== undefined && !metrics.some(m => m.name === 'tokens_input')) {
+      metrics.push({ name: 'tokens_input', value: run.measurement.tokens_input, passed: true })
+    }
+    if (run.measurement.tokens_output !== undefined && !metrics.some(m => m.name === 'tokens_output')) {
+      metrics.push({ name: 'tokens_output', value: run.measurement.tokens_output, passed: true })
+    }
+    if (run.measurement.tokens_total !== undefined && !metrics.some(m => m.name === 'tokens_total')) {
+      metrics.push({ name: 'tokens_total', value: run.measurement.tokens_total, passed: true })
+    }
+  }
+
+  return metrics
+}
+
+function mapTestResult(result: EdTestResult): UploadTestResult {
+  const singleRuns = result.singleRuns || []
+  const rerunsCount = singleRuns.length || 1
+
+  // Single run (or fallback) — flat metrics, no per-rerun data
+  if (rerunsCount <= 1) {
+    const run = singleRuns[0]
+    const metrics = run
+      ? buildMetricsFromRun(run, result.benchmarkResult)
+      : buildMetricsFromRun(result as any, result.benchmarkResult)
+
+    return {
+      test_id: result.testId,
+      test_name: result.testName,
+      status: result.status,
+      failure_reason: result.failureReason,
+      trace_ref: result.traceRef,
+      target: result.target,
+      duration_ms: result.durationMs,
+      reruns_count: 1,
+      input: result.input,
+      output: result.output,
+      metrics,
+    }
+  }
+
+  // Multiple runs — use measurements[] format with rerun_index
+  // Collect all unique metric names across all runs
+  const metricNameSet = new Set<string>()
+  const perRunMetrics: UploadMetric[][] = singleRuns.map(run => {
+    const m = buildMetricsFromRun(run, result.benchmarkResult)
+    m.forEach(metric => metricNameSet.add(metric.name))
+    return m
+  })
+
+  // Build metrics with measurements[] arrays
+  const metrics: UploadMetric[] = []
+  for (const name of metricNameSet) {
+    const measurements: { rerun_index: number; value: number; passed: boolean }[] = []
+    let benchmark: { type: string; threshold: number } | undefined
+
+    for (let i = 0; i < singleRuns.length; i++) {
+      const metric = perRunMetrics[i].find(m => m.name === name)
+      if (metric) {
+        measurements.push({ rerun_index: i, value: metric.value, passed: metric.passed })
+        if (!benchmark && metric.benchmark) benchmark = metric.benchmark
+      }
+    }
+
+    // Aggregate: use first run's value as the top-level value
+    const firstMeasurement = measurements[0]
+    metrics.push({
+      name: name as UploadMetric['name'],
+      value: firstMeasurement?.value ?? 0,
+      passed: measurements.every(m => m.passed),
+      benchmark,
+      measurements,
+    })
+  }
+
+  // Build reruns[] snapshots
+  const reruns: UploadRerun[] = singleRuns.map((run, i) => ({
+    rerun_index: i,
+    output: run.output,
+    started_at: run.startedAt,
+    finished_at: run.finishedAt,
+  }))
+
+  return {
+    test_id: result.testId,
+    test_name: result.testName,
+    status: result.status,
+    failure_reason: result.failureReason,
+    trace_ref: result.traceRef,
+    target: result.target,
+    duration_ms: result.durationMs,
+    reruns_count: rerunsCount,
+    input: result.input,
+    output: result.output,
+    metrics,
+    reruns,
+  }
+}
+
+// ─── Upload with retries ────────────────────────────────────
+
+export interface UploadOptions {
+  serverUrl: string
+  apiKey: string
+}
+
+export interface UploadResponse {
+  // Backend applies snake2Camel conversion, so response arrives in camelCase
+  runId: string
+  status: 'ingested' | 'already_ingested'
+  resultsCount: number
+}
+
+export async function uploadResults(
+  payload: UploadPayload,
+  options: UploadOptions,
+): Promise<UploadResponse> {
+  const url = `${options.serverUrl.replace(/\/+$/, '')}/api/v1/test-runs/create`
+  const headers: Record<string, string> = {
+    'Content-Type': 'application/json',
+    'api-key': options.apiKey,
+    'X-Correlation-ID': randomUUID(),
+  }
+
+  const body = JSON.stringify(payload)
+  console.log(`[elasticdash] Uploading to ${url}, api-key=${options.apiKey ? options.apiKey.slice(0, 10) + '...' : '(none)'}`)
+  const maxRetries = 3
+  const backoffMs = [1000, 2000, 4000]
+
+  let lastError: Error | undefined
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    try {
+      const res = await getOriginalFetch()(url, { method: 'POST', headers, body })
+
+      if (res.ok) {
+        const json = await res.json() as { success: boolean; result: UploadResponse }
+        return json.result
+      }
+
+      const status = res.status
+      const text = await res.text().catch(() => '')
+
+      // Don't retry 4xx — payload is malformed
+      if (status >= 400 && status < 500) {
+        if (status === 402) {
+          notifyLicenseError(status, 'ci-upload')
+          throw new Error(
+            `Upload failed with 402: no available license for your ElasticDash account. ` +
+            `Please go to https://app.elasticdash.com to buy a plan.`
+          )
+        }
+        throw new Error(`Upload failed with ${status}: ${text.substring(0, 200)}`)
+      }
+
+      // 5xx — retry
+      lastError = new Error(`Upload failed with ${status}: ${text.substring(0, 200)}`)
+    } catch (err) {
+      if (err instanceof Error && err.message.startsWith('Upload failed with 4')) {
+        throw err // Don't retry 4xx
+      }
+      lastError = err instanceof Error ? err : new Error(String(err))
+    }
+
+    // Wait before retry (skip on last attempt)
+    if (attempt < maxRetries) {
+      await new Promise(resolve => setTimeout(resolve, backoffMs[attempt]))
+    }
+  }
+
+  throw lastError ?? new Error('Upload failed after retries')
+}
+
+// ─── Persist failed uploads ─────────────────────────────────
+
+export async function persistFailedUpload(
+  payload: UploadPayload,
+  error: string,
+  cwd?: string,
+): Promise<void> {
+  const dir = join(cwd ?? process.cwd(), '.ed_traces', 'failed_uploads')
+  try {
+    await mkdir(dir, { recursive: true })
+    const filepath = join(dir, `${payload.run.run_id}.json`)
+    await writeFile(filepath, JSON.stringify({ payload, error, timestamp: new Date().toISOString() }, null, 2), 'utf-8')
+  } catch {
+    // Best-effort — don't throw
+  }
+}