elasticdash-test 0.1.17 → 0.1.18-alpha

package/src/portal-server.ts

@@ -0,0 +1,290 @@
+import express from 'express'
+import http from 'node:http'
+import { executePortalTask } from './portal-executor.js'
+import { scanTools } from './execution/tool-runner.js'
+import type { ToolInfo } from './execution/tool-runner.js'
+import type {
+  PortalTask,
+  PortalTaskResult,
+  PortalServerOptions,
+  PortalServerHandle,
+  PortalStatus,
+} from './types/portal.js'
+import { debugLog } from './utils/debug.js'
+
+// ---------------------------------------------------------------------------
+// Origin allowlist
+// ---------------------------------------------------------------------------
+
+function extractHost(url: string): string | null {
+  try {
+    return new URL(url).host
+  } catch {
+    return null
+  }
+}
+
+function buildAllowedHosts(backendUrl: string, extra?: string[]): Set<string> {
+  const hosts = new Set<string>()
+  // Always allow localhost variants
+  hosts.add('localhost')
+  hosts.add('127.0.0.1')
+  hosts.add('::1')
+  // Allow the configured backend
+  const backendHost = extractHost(backendUrl)
+  if (backendHost) hosts.add(backendHost)
+  // Allow explicit extra origins
+  if (extra) {
+    for (const origin of extra) {
+      const h = extractHost(origin)
+      if (h) hosts.add(h)
+      else hosts.add(origin) // treat raw hostname as-is
+    }
+  }
+  return hosts
+}
+
+function isAllowedOrigin(req: express.Request, allowedHosts: Set<string>): boolean {
+  // Determine the caller's host from multiple headers
+  // 1. Origin header (set by browsers and some HTTP clients)
+  const origin = req.headers.origin
+  if (origin) {
+    const h = extractHost(origin)
+    return h !== null && isHostAllowed(h, allowedHosts)
+  }
+  // 2. Referer header fallback
+  const referer = req.headers.referer
+  if (referer) {
+    const h = extractHost(referer)
+    return h !== null && isHostAllowed(h, allowedHosts)
+  }
+  // 3. X-Forwarded-For / remote IP — check if it's a local request
+  const remoteIp = req.ip ?? req.socket?.remoteAddress ?? ''
+  if (remoteIp === '127.0.0.1' || remoteIp === '::1' || remoteIp === '::ffff:127.0.0.1') {
+    return true
+  }
+  // 4. No origin info — this is a direct server-to-server call.
+  //    If API key auth is configured and passes, allow it.
+  //    If no API key is configured, reject unknown-origin requests.
+  return false
+}
+
+function isHostAllowed(host: string, allowedHosts: Set<string>): boolean {
+  // Exact match (includes port, e.g. "localhost:4573")
+  if (allowedHosts.has(host)) return true
+  // Match without port
+  const hostWithoutPort = host.split(':')[0]
+  if (allowedHosts.has(hostWithoutPort)) return true
+  // Match subdomains (e.g. "api.elasticdash.com" matches "elasticdash.com")
+  for (const allowed of allowedHosts) {
+    if (hostWithoutPort.endsWith('.' + allowed)) return true
+  }
+  return false
+}
+
+// ---------------------------------------------------------------------------
+// Server
+// ---------------------------------------------------------------------------
+
+export async function startPortalServer(options: PortalServerOptions): Promise<PortalServerHandle> {
+  const port = options.port ?? 4574
+  const backendUrl = options.backendUrl.replace(/\/$/, '')
+  const apiKey = options.apiKey
+  const cwd = options.cwd ?? process.cwd()
+
+  // Build allowed-origin set from backendUrl + explicit allowlist + localhost
+  const allowedHosts = buildAllowedHosts(backendUrl, options.allowedOrigins)
+  console.log(`[elasticdash portal] Allowed origins: ${[...allowedHosts].join(', ')}`)
+
+  // Scan tools at startup
+  const tools: ToolInfo[] = scanTools(cwd)
+  console.log(`[elasticdash portal] Scanned ${tools.length} tools: ${tools.map(t => t.name).join(', ') || '(none)'}`)
+
+  // Queue state
+  const queue: PortalTask[] = []
+  let processing: string | null = null
+  let completed = 0
+  let failed = 0
+  let draining = false
+
+  // -------------------------------------------------------------------------
+  // Queue processor
+  // -------------------------------------------------------------------------
+
+  async function processQueue(): Promise<void> {
+    if (draining || processing) return
+    if (queue.length === 0) return
+
+    const task = queue.shift()!
+    processing = task.taskId
+    console.log(`[elasticdash portal] Processing task ${task.taskId} (type=${task.type}, name=${task.name}) — ${queue.length} remaining`)
+
+    const result = await executePortalTask(task, cwd, tools)
+
+    if (result.ok) {
+      completed++
+      console.log(`[elasticdash portal] Task ${task.taskId} completed (${result.durationMs}ms)`)
+    } else {
+      failed++
+      console.log(`[elasticdash portal] Task ${task.taskId} failed: ${result.error}`)
+    }
+
+    processing = null
+
+    // Deliver result to backend
+    await deliverResult(result)
+
+    // Process next task
+    processQueue().catch(() => {})
+  }
+
+  async function deliverResult(result: PortalTaskResult): Promise<void> {
+    const url = `${backendUrl}/api/portal/results/${result.taskId}`
+    const headers: Record<string, string> = { 'Content-Type': 'application/json' }
+    if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        const res = await fetch(url, {
+          method: 'POST',
+          headers,
+          body: JSON.stringify(result),
+        })
+        if (res.ok || res.status < 500) {
+          debugLog(`[elasticdash portal] Result delivered for task ${result.taskId} (status ${res.status})`)
+          return
+        }
+        debugLog(`[elasticdash portal] Result delivery failed (status ${res.status}), attempt ${attempt + 1}/3`)
+      } catch (e) {
+        debugLog(`[elasticdash portal] Result delivery error, attempt ${attempt + 1}/3: ${e instanceof Error ? e.message : String(e)}`)
+      }
+      if (attempt < 2) {
+        await new Promise(r => setTimeout(r, Math.pow(2, attempt) * 1000))
+      }
+    }
+    console.warn(`[elasticdash portal] WARNING: Failed to deliver result for task ${result.taskId} after 3 retries`)
+  }
+
+  // -------------------------------------------------------------------------
+  // Security middleware: origin allowlist + API key auth
+  // -------------------------------------------------------------------------
+
+  function securityMiddleware(req: express.Request, res: express.Response, next: express.NextFunction): void {
+    // Check API key first — a valid key overrides origin checks
+    if (apiKey) {
+      const authHeader = req.headers.authorization
+      if (authHeader === `Bearer ${apiKey}`) return next()
+    }
+
+    // Check origin allowlist
+    if (isAllowedOrigin(req, allowedHosts)) {
+      // If API key is configured but not provided, still reject
+      if (apiKey) {
+        res.status(401).json({ ok: false, error: 'unauthorized — valid API key required' })
+        return
+      }
+      return next()
+    }
+
+    // Origin not in allowlist and no valid API key
+    const origin = req.headers.origin ?? req.headers.referer ?? req.ip ?? 'unknown'
+    console.warn(`[elasticdash portal] Blocked request from disallowed origin: ${origin}`)
+    res.status(403).json({ ok: false, error: 'forbidden — origin not in allowlist' })
+  }
+
+  // -------------------------------------------------------------------------
+  // Express app
+  // -------------------------------------------------------------------------
+
+  const app = express()
+  app.use(express.json({ limit: '10mb' }))
+
+  // POST /api/portal/tasks — enqueue a single task
+  app.post('/api/portal/tasks', securityMiddleware, (req, res) => {
+    const task = req.body as PortalTask
+    if (!task?.taskId || !task?.type || !task?.name) {
+      res.status(400).json({ ok: false, error: 'Missing required fields: taskId, type, name' })
+      return
+    }
+    queue.push(task)
+    const position = queue.length
+    debugLog(`[elasticdash portal] Task ${task.taskId} enqueued at position ${position}`)
+    res.status(202).json({ ok: true, taskId: task.taskId, position })
+    processQueue().catch(() => {})
+  })
+
+  // POST /api/portal/tasks/batch — enqueue multiple tasks
+  app.post('/api/portal/tasks/batch', securityMiddleware, (req, res) => {
+    const body = req.body as { tasks?: PortalTask[] }
+    if (!Array.isArray(body?.tasks) || body.tasks.length === 0) {
+      res.status(400).json({ ok: false, error: 'Missing or empty tasks array' })
+      return
+    }
+    const results: Array<{ taskId: string; position: number }> = []
+    for (const task of body.tasks) {
+      if (!task?.taskId || !task?.type || !task?.name) continue
+      queue.push(task)
+      results.push({ taskId: task.taskId, position: queue.length })
+    }
+    debugLog(`[elasticdash portal] Batch enqueued ${results.length} tasks`)
+    res.status(202).json({ ok: true, tasks: results })
+    processQueue().catch(() => {})
+  })
+
+  // GET /api/portal/status — health check
+  app.get('/api/portal/status', (_req, res) => {
+    const status: PortalStatus = {
+      ok: true,
+      queueLength: queue.length,
+      processing,
+      completed,
+      failed,
+    }
+    res.json(status)
+  })
+
+  // DELETE /api/portal/tasks/:taskId — cancel a pending task
+  app.delete('/api/portal/tasks/:taskId', securityMiddleware, (req, res) => {
+    const taskId = req.params.taskId
+    const index = queue.findIndex(t => t.taskId === taskId)
+    if (index === -1) {
+      res.status(404).json({ ok: false, error: 'Task not found in queue (may be already processing or completed)' })
+      return
+    }
+    queue.splice(index, 1)
+    res.json({ ok: true, taskId })
+  })
+
+  // -------------------------------------------------------------------------
+  // Start server
+  // -------------------------------------------------------------------------
+
+  const server = http.createServer(app)
+
+  await new Promise<void>((resolve, reject) => {
+    server.on('error', reject)
+    server.listen(port, () => resolve())
+  })
+
+  const url = `http://localhost:${port}`
+
+  return {
+    port,
+    url,
+    close: async () => {
+      draining = true
+      // Wait for current task to finish if processing
+      if (processing) {
+        console.log(`[elasticdash portal] Waiting for current task ${processing} to finish...`)
+        await new Promise<void>((resolve) => {
+          const check = setInterval(() => {
+            if (!processing) { clearInterval(check); resolve() }
+          }, 200)
+        })
+      }
+      await new Promise<void>((resolve, reject) => {
+        server.close((err) => err ? reject(err) : resolve())
+      })
+    },
+  }
+}

package/src/telemetry-batcher.ts

@@ -0,0 +1,143 @@
+import type { WorkflowEvent } from './capture/event.js'
+import { debugLog } from './utils/debug.js'
+import { redactPayload } from './utils/redact.js'
+
+export interface TriggerStep {
+  eventId: number
+  eventType: 'ai' | 'tool'
+  eventName: string
+  originalEventDbId: number
+  input: unknown
+  model?: string
+  provider?: string
+}
+
+export interface TriggerSignal {
+  triggerId: number
+  runCount: number
+  steps: TriggerStep[]
+}
+
+export interface TelemetryBatcherOptions {
+  serverUrl: string
+  apiKey?: string
+  sessionId: string
+  serviceId: string
+  batchIntervalMs?: number
+  maxBatchSize?: number
+  redactKeys?: string[]
+  onTrigger?: (trigger: TriggerSignal) => Promise<void>
+}
+
+export class TelemetryBatcher {
+  private buffer: WorkflowEvent[] = []
+  private timer: ReturnType<typeof setInterval> | null = null
+  private readonly serverUrl: string
+  private readonly apiKey: string | undefined
+  private readonly sessionId: string
+  private readonly serviceId: string
+  private readonly maxBatchSize: number
+  private readonly redactKeys: string[]
+  private readonly onTrigger: ((trigger: TriggerSignal) => Promise<void>) | undefined
+  private shuttingDown = false
+
+  constructor(opts: TelemetryBatcherOptions) {
+    this.serverUrl = opts.serverUrl.replace(/\/$/, '')
+    this.apiKey = opts.apiKey
+    this.sessionId = opts.sessionId
+    this.serviceId = opts.serviceId
+    this.maxBatchSize = opts.maxBatchSize ?? 50
+    this.redactKeys = opts.redactKeys ?? []
+    this.onTrigger = opts.onTrigger
+
+    const intervalMs = opts.batchIntervalMs ?? 2000
+    this.timer = setInterval(() => { this.flush().catch(() => {}) }, intervalMs)
+    // Allow the process to exit even if the timer is still scheduled
+    if (this.timer && typeof this.timer === 'object' && 'unref' in this.timer) {
+      this.timer.unref()
+    }
+  }
+
+  enqueue(event: WorkflowEvent): void {
+    if (this.shuttingDown) return
+    const redacted = this.redactKeys.length > 0
+      ? { ...event, input: redactPayload(event.input, this.redactKeys), output: redactPayload(event.output, this.redactKeys) }
+      : event
+    this.buffer.push(redacted as WorkflowEvent)
+    if (this.buffer.length >= this.maxBatchSize) {
+      this.flush().catch(() => {})
+    }
+  }
+
+  async flush(): Promise<void> {
+    if (this.buffer.length === 0) return
+    const batch = this.buffer.splice(0, this.buffer.length)
+    await this.send(batch, 0)
+  }
+
+  private async send(batch: WorkflowEvent[], attempt: number): Promise<void> {
+    const maxRetries = 3
+    const url = `${this.serverUrl}/api/observability/events`
+    const headers: Record<string, string> = { 'Content-Type': 'application/json' }
+    if (this.apiKey) headers['Authorization'] = `Bearer ${this.apiKey}`
+
+    try {
+      const res = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          sessionId: this.sessionId,
+          serviceId: this.serviceId,
+          events: batch,
+        }),
+      })
+
+      if (res.status === 429 || res.status >= 500) {
+        if (attempt < maxRetries) {
+          const delayMs = Math.pow(2, attempt) * 1000
+          debugLog(`[elasticdash] Telemetry flush failed (${res.status}), retrying in ${delayMs}ms (attempt ${attempt + 1}/${maxRetries})`)
+          await new Promise((r) => setTimeout(r, delayMs))
+          return this.send(batch, attempt + 1)
+        }
+        debugLog(`[elasticdash] Dropping ${batch.length} events after ${maxRetries} retries (last status: ${res.status})`)
+        return
+      }
+
+      debugLog(`[elasticdash] Flushed ${batch.length} events (status ${res.status})`)
+
+      // Parse response for trigger signal
+      if (res.ok && this.onTrigger) {
+        try {
+          const body = await res.json() as { trigger?: TriggerSignal }
+          if (body.trigger && typeof body.trigger.triggerId === 'number' && Array.isArray(body.trigger.steps)) {
+            debugLog(`[elasticdash] Trigger received: id=${body.trigger.triggerId} steps=${body.trigger.steps.length} runCount=${body.trigger.runCount}`)
+            // Fire-and-forget — don't block the flush pipeline
+            this.onTrigger(body.trigger).catch((err) => {
+              debugLog(`[elasticdash] Trigger execution failed: ${err instanceof Error ? err.message : String(err)}`)
+            })
+          }
+        } catch {
+          // Response parsing failed — not critical, ignore
+        }
+      }
+    } catch (err) {
+      if (attempt < maxRetries) {
+        const delayMs = Math.pow(2, attempt) * 1000
+        debugLog(`[elasticdash] Telemetry flush error, retrying in ${delayMs}ms (attempt ${attempt + 1}/${maxRetries})`)
+        await new Promise((r) => setTimeout(r, delayMs))
+        return this.send(batch, attempt + 1)
+      }
+      debugLog(`[elasticdash] Dropping ${batch.length} events after ${maxRetries} retries: ${err instanceof Error ? err.message : String(err)}`)
+    }
+  }
+
+  async shutdown(): Promise<void> {
+    if (this.shuttingDown) return
+    this.shuttingDown = true
+    if (this.timer) {
+      clearInterval(this.timer)
+      this.timer = null
+    }
+    await this.flush()
+  }
+}

package/src/trigger-executor.ts

@@ -0,0 +1,121 @@
+import type { TriggerSignal } from './telemetry-batcher.js'
+import { executePortalTask, checkToolAvailability, checkAIAvailability } from './portal-executor.js'
+import { scanTools } from './execution/tool-runner.js'
+import { debugLog } from './utils/debug.js'
+
+interface StepRunResult {
+  runIndex: number
+  input: unknown
+  output: unknown
+  durationMs: number
+  error?: string
+  usageInputTokens?: number
+  usageOutputTokens?: number
+  usageTotalTokens?: number
+}
+
+interface StepResult {
+  originalEventDbId: number
+  eventType: string
+  eventName: string
+  available: boolean
+  unavailableReason?: string
+  runs: StepRunResult[]
+}
+
+/**
+ * Executes a trigger received from the backend's event batch response.
+ *
+ * For each step:
+ * 1. Pre-validates availability (tool exists? API key set?)
+ * 2. If unavailable: reports `available: false` with reason, skips execution
+ * 3. If available: re-executes `runCount` times, collects results
+ * 4. POSTs all results to `POST /api/observability/triggers/:triggerId/results`
+ */
+export async function executeTrigger(
+  serverUrl: string,
+  apiKey: string | undefined,
+  trigger: TriggerSignal,
+): Promise<void> {
+  const cwd = process.cwd()
+  const tools = scanTools(cwd)
+
+  debugLog(`[elasticdash] Executing trigger ${trigger.triggerId}: ${trigger.steps.length} steps × ${trigger.runCount} runs`)
+
+  const stepResults: StepResult[] = []
+
+  for (const step of trigger.steps) {
+    // Pre-validate availability
+    const availability = step.eventType === 'ai'
+      ? checkAIAvailability(step.provider, step.model ?? step.eventName)
+      : checkToolAvailability(step.eventName, cwd, tools)
+
+    if (!availability.available) {
+      debugLog(`[elasticdash] Trigger ${trigger.triggerId} step=${step.eventName} unavailable: ${availability.reason}`)
+      stepResults.push({
+        originalEventDbId: step.originalEventDbId,
+        eventType: step.eventType,
+        eventName: step.eventName,
+        available: false,
+        unavailableReason: availability.reason,
+        runs: [],
+      })
+      continue
+    }
+
+    // Execute runs
+    const runs: StepRunResult[] = []
+
+    for (let i = 0; i < trigger.runCount; i++) {
+      const result = await executePortalTask(
+        {
+          taskId: `trigger-${trigger.triggerId}-${step.eventName}-${i}`,
+          type: step.eventType === 'ai' ? 'ai' : 'tool',
+          name: step.eventName,
+          input: step.input,
+          model: step.eventType === 'ai' ? (step.model ?? step.eventName) : undefined,
+          provider: step.provider,
+        },
+        cwd,
+        tools,
+      )
+
+      runs.push({
+        runIndex: i,
+        input: step.input,
+        output: result.output,
+        durationMs: result.durationMs,
+        error: result.error,
+        usageInputTokens: result.usage?.inputTokens,
+        usageOutputTokens: result.usage?.outputTokens,
+        usageTotalTokens: result.usage?.totalTokens,
+      })
+
+      debugLog(`[elasticdash] Trigger ${trigger.triggerId} step=${step.eventName} run=${i} ok=${result.ok}`)
+    }
+
+    stepResults.push({
+      originalEventDbId: step.originalEventDbId,
+      eventType: step.eventType,
+      eventName: step.eventName,
+      available: true,
+      runs,
+    })
+  }
+
+  // POST results to backend
+  const url = `${serverUrl.replace(/\/$/, '')}/api/observability/triggers/${trigger.triggerId}/results`
+  const headers: Record<string, string> = { 'Content-Type': 'application/json' }
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+
+  try {
+    const res = await fetch(url, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify({ steps: stepResults }),
+    })
+    debugLog(`[elasticdash] Trigger ${trigger.triggerId} results posted (status ${res.status})`)
+  } catch (err) {
+    debugLog(`[elasticdash] Trigger ${trigger.triggerId} results POST failed: ${err instanceof Error ? err.message : String(err)}`)
+  }
+}

package/src/types/portal.ts

@@ -0,0 +1,67 @@
+export interface PortalTask {
+  /** Unique task ID assigned by the backend */
+  taskId: string
+  /** What to rerun */
+  type: 'tool' | 'ai'
+  /** Tool name or model name */
+  name: string
+  /** Tool arguments or LLM prompt/messages */
+  input: unknown
+  /** Model name (for AI tasks) */
+  model?: string
+  /** LLM provider: openai, anthropic, gemini, grok, etc. */
+  provider?: string
+  /** LLM generation parameters */
+  modelParameters?: {
+    temperature?: number
+    max_tokens?: number
+  }
+  /** Passthrough metadata (test group ID, expectation IDs, etc.) */
+  metadata?: Record<string, unknown>
+}
+
+export interface PortalTaskResult {
+  taskId: string
+  ok: boolean
+  output: unknown
+  error?: string
+  durationMs: number
+  usage?: {
+    inputTokens?: number
+    outputTokens?: number
+    totalTokens?: number
+  }
+  /** Echoed from the original task */
+  metadata?: Record<string, unknown>
+}
+
+export interface PortalServerOptions {
+  /** Port to listen on (default 4574) */
+  port?: number
+  /** Backend URL to POST results to */
+  backendUrl: string
+  /** Auth token for incoming and outgoing requests */
+  apiKey?: string
+  /** Project root directory (default process.cwd()) */
+  cwd?: string
+  /**
+   * Allowed origin domains that may send requests to this portal.
+   * By default only the `backendUrl` domain and localhost are allowed.
+   * Provide additional origins (e.g. 'https://app.elasticdash.com') to extend.
+   */
+  allowedOrigins?: string[]
+}
+
+export interface PortalServerHandle {
+  port: number
+  url: string
+  close: () => Promise<void>
+}
+
+export interface PortalStatus {
+  ok: boolean
+  queueLength: number
+  processing: string | null
+  completed: number
+  failed: number
+}

package/src/utils/debug.ts

@@ -0,0 +1,8 @@
+const DEBUG_KEY = 'ELASTICDASH_DEBUG'
+
+/** Log only when ELASTICDASH_DEBUG=1 is set. Drop-in replacement for console.log in interceptors. */
+export function debugLog(...args: unknown[]): void {
+  if (typeof process !== 'undefined' && process.env?.[DEBUG_KEY] === '1') {
+    console.log(...args)
+  }
+}

package/src/utils/redact.ts

@@ -0,0 +1,25 @@
+/**
+ * Deep-clones a value and replaces any object property whose key matches
+ * one of `keys` (case-insensitive) with "[REDACTED]".
+ * Returns the original value when `keys` is empty.
+ */
+export function redactPayload(value: unknown, keys: string[]): unknown {
+  if (keys.length === 0) return value
+  const lowerKeys = new Set(keys.map((k) => k.toLowerCase()))
+  return redact(value, lowerKeys)
+}
+
+function redact(value: unknown, keys: Set<string>): unknown {
+  if (value === null || value === undefined) return value
+  if (typeof value !== 'object') return value
+
+  if (Array.isArray(value)) {
+    return value.map((item) => redact(item, keys))
+  }
+
+  const out: Record<string, unknown> = {}
+  for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+    out[k] = keys.has(k.toLowerCase()) ? '[REDACTED]' : redact(v, keys)
+  }
+  return out
+}