elasticdash-sdk 0.2.0

package/src/trace-adapter/context.ts

@@ -0,0 +1,156 @@
+export interface LLMStep {
+  model: string
+  provider?: string   // 'openai' | 'gemini' | 'grok' | undefined
+  prompt?: string
+  completion?: string
+  contains?: string
+  workflowEventId?: number
+  durationMs?: number
+}
+
+export interface ToolCall {
+  name: string
+  args?: Record<string, unknown>
+  result?: unknown
+  workflowEventId?: number
+  durationMs?: number
+}
+
+export type CustomStepKind = 'rag' | 'code' | 'fixed' | 'custom'
+
+export interface CustomStep {
+  kind: CustomStepKind
+  name?: string
+  tags?: string[]
+  payload?: unknown
+  result?: unknown
+  metadata?: Record<string, unknown>
+  contains?: string
+}
+
+export interface TraceStep {
+  type: 'llm' | 'tool' | 'custom'
+  timestamp: number
+  durationMs: number
+  data: Record<string, unknown>
+}
+
+export interface TraceHandle {
+  /** All recorded steps in this trace session */
+  getSteps(): TraceStep[]
+  /** Only LLM inference steps */
+  getLLMSteps(): LLMStep[]
+  /** Only tool-call steps */
+  getToolCalls(): ToolCall[]
+  /** Only custom steps (RAG, code, fixed, etc.) */
+  getCustomSteps(): CustomStep[]
+  /** Record an LLM step (used by stubs / real adapter) */
+  recordLLMStep(step: LLMStep): void
+  /** Record a tool call (used by stubs / real adapter) */
+  recordToolCall(call: ToolCall): void
+  /** Record a custom step (e.g., RAG, code) */
+  recordCustomStep(step: CustomStep): void
+}
+
+export interface AITestContext {
+  trace: TraceHandle
+}
+
+// --- AsyncLocalStorage-backed current trace (parallel-safe) ---
+import { AsyncLocalStorage } from 'node:async_hooks'
+import { rawDateNow } from '../interceptors/side-effects.js'
+
+const g = globalThis as Record<string, unknown>
+const TRACE_ALS_KEY = '__elasticdash_trace_als__'
+const traceAls: AsyncLocalStorage<TraceHandle | undefined> =
+  (g[TRACE_ALS_KEY] as AsyncLocalStorage<TraceHandle | undefined>) ??
+  new AsyncLocalStorage<TraceHandle | undefined>()
+if (!g[TRACE_ALS_KEY]) g[TRACE_ALS_KEY] = traceAls
+
+export function setCurrentTrace(trace: TraceHandle | undefined): void {
+  traceAls.enterWith(trace)
+}
+
+export function getCurrentTrace(): TraceHandle | undefined {
+  return traceAls.getStore()
+}
+
+/** Extension points for runner hooks (scaffold for future backend integration) */
+export interface RunnerHooks {
+  onTestStart?(name: string): void | Promise<void>
+  onTestFinish?(name: string, passed: boolean, durationMs: number, error?: Error): void | Promise<void>
+  onTraceComplete?(name: string, trace: TraceHandle): void | Promise<void>
+}
+
+/**
+ * Create a stubbed trace handle for a single test execution.
+ * Later this can be replaced with a real ElasticDash backend call.
+ */
+export function createTraceHandle(): TraceHandle {
+  const steps: TraceStep[] = []
+  const llmSteps: LLMStep[] = []
+  const toolCalls: ToolCall[] = []
+  const customSteps: CustomStep[] = []
+
+  return {
+    getSteps() {
+      return steps
+    },
+
+    getLLMSteps() {
+      return llmSteps
+    },
+
+    getToolCalls() {
+      return toolCalls
+    },
+
+    getCustomSteps() {
+      return customSteps
+    },
+
+    recordLLMStep(step: LLMStep) {
+      llmSteps.push(step)
+      steps.push({
+        type: 'llm',
+        timestamp: rawDateNow(),
+        durationMs: step.durationMs ?? 0,
+        data: step as unknown as Record<string, unknown>,
+      })
+    },
+
+    recordToolCall(call: ToolCall) {
+      toolCalls.push(call)
+      steps.push({
+        type: 'tool',
+        timestamp: rawDateNow(),
+        durationMs: call.durationMs ?? 0,
+        data: call as unknown as Record<string, unknown>,
+      })
+    },
+
+    recordCustomStep(step: CustomStep) {
+      customSteps.push(step)
+      steps.push({
+        type: 'custom',
+        timestamp: rawDateNow(),
+        durationMs: 0,
+        data: step as unknown as Record<string, unknown>,
+      })
+    },
+  }
+}
+
+/**
+ * Start a trace session before a test and return the context + a finalise fn.
+ */
+export function startTraceSession(): { context: AITestContext; finalise: () => void } {
+  const trace = createTraceHandle()
+  const context: AITestContext = { trace }
+  return {
+    context,
+    finalise() {
+      // Placeholder: flush / send to ElasticDash backend here in the future
+    },
+  }
+}

package/src/tracing.ts

@@ -0,0 +1,62 @@
+// src/tracing.ts
+// ElasticDash tool call recording utility
+
+/**
+ * Records a tool call for workflow tracing. Safe to call in any environment.
+ * If not running inside the ElasticDash runner, this is a no-op.
+ *
+ * @param name - The tool name
+ * @param args - The tool arguments (object or array)
+ * @param result - The tool result (or error)
+ */
+import { getCurrentTrace } from './trace-adapter/context.js'
+import { getCaptureContext } from './capture/recorder.js'
+import { rawDateNow } from './interceptors/side-effects.js'
+
+const TOOL_WRAPPER_ACTIVE_KEY = '__elasticdash_tool_wrapper_active__'
+
+function wrapperRecordingActive(): boolean {
+  return (globalThis as Record<string, unknown>)[TOOL_WRAPPER_ACTIVE_KEY] === true
+}
+
+export function recordToolCall(name: string, args: any, result: any, durationMs = 0) {
+  if (!(globalThis as any).__ELASTICDASH_WORKER__) return
+  try {
+    // Avoid double-recording when a replay-aware tool wrapper is already active.
+    if (wrapperRecordingActive()) return
+
+    const trace = getCurrentTrace()
+    if (!trace || typeof trace.recordToolCall !== 'function') return
+
+    const ctx = getCaptureContext()
+    if (!ctx) {
+      trace.recordToolCall({ name, args, result, durationMs })
+      return
+    }
+
+    const { recorder, replay } = ctx
+    const id = recorder.nextId()
+
+    if (replay.shouldReplay(id)) {
+      const historical = replay.getRecordedEvent(id)
+      if (historical) recorder.record(historical)
+      const replayed = replay.getRecordedResult(id)
+      trace.recordToolCall({ name, args, result: replayed, workflowEventId: id })
+      return
+    }
+
+    const output = result instanceof Error ? { error: String(result) } : result
+    recorder.record({
+      id,
+      type: 'tool',
+      name,
+      input: args,
+      output,
+      timestamp: rawDateNow(),
+      durationMs,
+    })
+    trace.recordToolCall({ name, args, result: output, workflowEventId: id, durationMs })
+  } catch {
+    // Never throw, always swallow errors
+  }
+}

package/src/trigger-executor.ts

@@ -0,0 +1,171 @@
+import type { TriggerSignal, FrozenEvent } from './telemetry-batcher.js'
+import { executePortalTask, checkToolAvailability, checkAIAvailability } from './portal-executor.js'
+import { getOriginalFetch } from './interceptors/http.js'
+import { getObservabilityContext } from './interceptors/telemetry-push.js'
+import { scanTools } from './execution/tool-runner.js'
+import { debugLog } from './utils/debug.js'
+
+/** Track trigger IDs that are currently executing or already completed to prevent duplicate execution. */
+const handledTriggers = new Set<number>()
+
+interface StepRunResult {
+  runIndex: number
+  input: unknown
+  output: unknown
+  durationMs: number
+  error?: string
+  usageInputTokens?: number
+  usageOutputTokens?: number
+  usageTotalTokens?: number
+}
+
+interface StepResult {
+  originalEventDbId: number
+  eventType: string
+  eventName: string
+  available: boolean
+  unavailableReason?: string
+  runs: StepRunResult[]
+}
+
+/**
+ * Executes a trigger received from the backend's event batch response.
+ *
+ * For each step:
+ * 1. Pre-validates availability (tool exists? API key set?)
+ * 2. If unavailable: reports `available: false` with reason, skips execution
+ * 3. If available: re-executes `runCount` times, collects results
+ * 4. POSTs each step's result individually to avoid payload size limits
+ */
+export async function executeTrigger(
+  serverUrl: string,
+  apiKey: string | undefined,
+  trigger: TriggerSignal,
+): Promise<void> {
+  // Feature flag: allow users to disable rerun acceptance per project
+  const acceptReruns = process.env.ELASTICDASH_ACCEPT_RERUNS
+  if (acceptReruns !== undefined && ['false', '0', 'no'].includes(acceptReruns.toLowerCase())) {
+    debugLog(`[elasticdash] Trigger ${trigger.triggerId} rejected: ELASTICDASH_ACCEPT_RERUNS=${acceptReruns}`)
+    return
+  }
+
+  // Dedup guard: skip if this trigger is already being handled
+  if (handledTriggers.has(trigger.triggerId)) {
+    debugLog(`[elasticdash] Trigger ${trigger.triggerId} already handled, skipping duplicate`)
+    return
+  }
+  handledTriggers.add(trigger.triggerId)
+
+  // Mark observability context as rerun so captured events are flagged
+  const obsCtx = getObservabilityContext()
+  if (obsCtx) obsCtx.isRerun = true
+
+  const cwd = process.cwd()
+  const tools = scanTools(cwd)
+  const frozenEvents = trigger.frozenEvents || []
+  const totalSteps = trigger.steps.length
+
+  debugLog(`[elasticdash] Executing trigger ${trigger.triggerId}: ${totalSteps} steps × ${trigger.runCount} runs, ${frozenEvents.length} frozenEvents`)
+
+  if (!serverUrl) {
+    debugLog(`[elasticdash] Trigger ${trigger.triggerId} aborted: serverUrl is empty`)
+    if (obsCtx) obsCtx.isRerun = false
+    return
+  }
+  const baseUrl = serverUrl.replace(/\/$/, '')
+  const headers: Record<string, string> = { 'Content-Type': 'application/json' }
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`
+
+  for (let stepIndex = 0; stepIndex < totalSteps; stepIndex++) {
+    const step = trigger.steps[stepIndex]
+
+    // Pre-validate availability
+    const availability = step.eventType === 'ai'
+      ? checkAIAvailability(step.provider, step.model ?? step.eventName)
+      : checkToolAvailability(step.eventName, cwd, tools)
+
+    let stepResult: StepResult
+
+    if (!availability.available) {
+      debugLog(`[elasticdash] Trigger ${trigger.triggerId} step=${step.eventName} unavailable: ${availability.reason}`)
+      stepResult = {
+        originalEventDbId: step.originalEventDbId,
+        eventType: step.eventType,
+        eventName: step.eventName,
+        available: false,
+        unavailableReason: availability.reason,
+        runs: [],
+      }
+    } else {
+      // Execute runs
+      const runs: StepRunResult[] = []
+
+      for (let i = 0; i < trigger.runCount; i++) {
+        const result = await executePortalTask(
+          {
+            taskId: `trigger-${trigger.triggerId}-${step.eventName}-${i}`,
+            type: step.eventType === 'ai' ? 'ai' : 'tool',
+            name: step.eventName,
+            input: step.input,
+            model: step.eventType === 'ai' ? (step.model ?? step.eventName) : undefined,
+            provider: step.provider,
+            frozenEvents,
+          },
+          cwd,
+          tools,
+        )
+
+        if (!result.ok) {
+          debugLog(`[elasticdash] Trigger ${trigger.triggerId} step=${step.eventName} run=${i} FAILED:\n${result.error}`)
+        }
+
+        runs.push({
+          runIndex: i,
+          input: step.input,
+          output: result.output,
+          durationMs: result.durationMs,
+          error: result.error,
+          usageInputTokens: result.usage?.inputTokens,
+          usageOutputTokens: result.usage?.outputTokens,
+          usageTotalTokens: result.usage?.totalTokens,
+        })
+
+        debugLog(`[elasticdash] Trigger ${trigger.triggerId} step=${step.eventName} run=${i} ok=${result.ok}`)
+      }
+
+      stepResult = {
+        originalEventDbId: step.originalEventDbId,
+        eventType: step.eventType,
+        eventName: step.eventName,
+        available: true,
+        runs,
+      }
+    }
+
+    // POST rerun results directly under the original event record
+    const url = `${baseUrl}/api/observability/events/${step.originalEventDbId}/reruns`
+    try {
+      const res = await getOriginalFetch()(url, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          triggerId: trigger.triggerId,
+          eventType: stepResult.eventType,
+          eventName: stepResult.eventName,
+          available: stepResult.available,
+          unavailableReason: stepResult.unavailableReason,
+          runs: stepResult.runs,
+          stepIndex,
+          totalSteps,
+        }),
+      })
+      debugLog(`[elasticdash] Trigger ${trigger.triggerId} step ${stepIndex + 1}/${totalSteps} posted to event ${step.originalEventDbId} (status ${res.status})`)
+    } catch (err) {
+      debugLog(`[elasticdash] Trigger ${trigger.triggerId} step ${stepIndex + 1}/${totalSteps} POST to event ${step.originalEventDbId} failed: ${err instanceof Error ? err.message : String(err)}`)
+    }
+  }
+
+  // Restore context after rerun execution
+  if (obsCtx) obsCtx.isRerun = false
+}
+

package/src/types/agent.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Agent state type definitions for mid-trace replay support.
+ *
+ * These types enable structured plan/task management for agents, allowing
+ * them to be resumed from any task in the plan without re-executing
+ * already-completed steps.
+ */
+
+export type AgentTaskStatus = 'pending' | 'in-progress' | 'completed' | 'failed'
+
+export interface AgentTask {
+  /** Unique task identifier (e.g. "task-1", "task-2") */
+  id: string
+  /** Current execution status */
+  status: AgentTaskStatus
+  /** Human-readable description of what this task does */
+  description: string
+  /** Tool/service to invoke (e.g. 'apiService', 'queryRefinement') */
+  tool: string
+  /**
+   * Task input parameters. May contain placeholder references to previous
+   * task outputs using the syntax: { $ref: "task-N.output.fieldName" }
+   */
+  input: unknown
+  /** Task result, populated after successful execution */
+  output?: unknown
+  /** Error message if task failed */
+  error?: string
+  /** Unix timestamp when task execution started */
+  startedAt?: number
+  /** Unix timestamp when task execution completed */
+  completedAt?: number
+}
+
+export type AgentPlanStatus = 'planning' | 'executing' | 'completed' | 'failed' | 'paused'
+
+export interface AgentPlan {
+  /** Unique plan identifier */
+  id: string
+  /** Ordered list of tasks to execute */
+  tasks: AgentTask[]
+  /** Overall plan execution status */
+  status: AgentPlanStatus
+  /** Zero-based index of the task currently being executed */
+  currentTaskIndex: number
+  /** Shared data/variables accessible to all tasks */
+  context: Record<string, unknown>
+  /** Additional plan metadata (user query, session ID, etc.) */
+  metadata: Record<string, unknown>
+}
+
+export interface AgentState {
+  /** The full agent plan including completed and pending tasks */
+  plan: AgentPlan
+  /** Partial trace events captured during previous execution */
+  trace: import('../capture/event.js').WorkflowEvent[]
+  /**
+   * Zero-based index of the task to resume from.
+   * Tasks 0..(resumeFromTaskIndex-1) will use cached outputs from plan.tasks.
+   * Tasks resumeFromTaskIndex..end will be executed fresh.
+   */
+  resumeFromTaskIndex: number
+}

package/src/types/expect.d.ts

@@ -0,0 +1,81 @@
+import 'expect';
+import type { TraceHandle, CustomStep, CustomStepKind } from '../trace-adapter/context';
+
+interface LLMStepConfig {
+  model?: string
+  contains?: string        // searches prompt + completion
+  promptContains?: string  // searches only in step.prompt
+  outputContains?: string  // searches only in step.completion
+  provider?: string        // 'openai' | 'claude' | 'gemini' | 'grok'
+  times?: number           // match count must equal exactly this value
+  minTimes?: number        // match count must be >= this value
+  maxTimes?: number        // match count must be <= this value
+}
+
+interface CustomStepConfig {
+  kind?: CustomStepKind
+  name?: string
+  tag?: string
+  contains?: string          // searches payload/result/metadata stringified
+  resultContains?: string    // searches result only
+  payloadContains?: string   // searches payload only
+  metadataContains?: string  // searches metadata only
+  times?: number
+  minTimes?: number
+  maxTimes?: number
+}
+
+interface PromptWhereConfig {
+  filterContains: string           // first filter: prompts that contain this substring
+  requireContains?: string         // then assert: filtered prompts must also contain this
+  requireNotContains?: string      // and must NOT contain this
+  times?: number                   // exact count of filtered prompts
+  minTimes?: number                // min count of filtered prompts
+  maxTimes?: number                // max count of filtered prompts
+  index?: number                   // optional 0-based index into filtered prompts to check specifically
+  nth?: number                     // optional 1-based alias for index
+}
+
+type SupportedProvider = 'openai' | 'claude' | 'gemini' | 'grok'
+
+interface SemanticMatchOptions {
+  provider?: SupportedProvider
+  model?: string
+  sdk?: unknown // optional user-supplied SDK instance
+  apiKey?: string // optional API key override (useful for OpenAI-compatible endpoints)
+  baseURL?: string // optional base URL override for OpenAI-compatible APIs
+}
+
+type EvaluationTarget = 'prompt' | 'result'
+
+interface EvaluationCondition {
+  greaterThan?: number
+  lessThan?: number
+  atLeast?: number
+  atMost?: number
+  equals?: number
+}
+
+interface EvaluateOutputMetricConfig {
+  evaluationPrompt: string
+  target?: EvaluationTarget       // 'prompt' or 'result'; default 'result'
+  index?: number                  // 0-based index into LLM steps
+  nth?: number                    // 1-based alias for index
+  condition?: EvaluationCondition // optional; default atLeast 0.7
+  provider?: SupportedProvider
+  model?: string
+  sdk?: unknown                   // optional SDK instance
+  apiKey?: string                 // optional API key override (useful for OpenAI-compatible endpoints)
+  baseURL?: string                // optional base URL override for OpenAI-compatible APIs
+}
+
+declare module 'expect' {
+  interface Matchers<R> {
+    toHaveLLMStep(config?: LLMStepConfig): R;
+    toCallTool(toolName: string): R;
+    toMatchSemanticOutput(expected: string, options?: SemanticMatchOptions): R;
+    toHaveCustomStep(config?: CustomStepConfig): R;
+    toHavePromptWhere(config: PromptWhereConfig): R;
+    toEvaluateOutputMetric(config: EvaluateOutputMetricConfig): Promise<R>;
+  }
+}

package/src/types/modules.d.ts

@@ -0,0 +1,2 @@
+declare module 'tsx/esm';
+declare module 'tsx/cjs';

package/src/types/portal.ts

@@ -0,0 +1,69 @@
+export interface PortalTask {
+  /** Unique task ID assigned by the backend */
+  taskId: string
+  /** What to rerun */
+  type: 'tool' | 'ai'
+  /** Tool name or model name */
+  name: string
+  /** Tool arguments or LLM prompt/messages */
+  input: unknown
+  /** Model name (for AI tasks) */
+  model?: string
+  /** LLM provider: openai, anthropic, gemini, grok, etc. */
+  provider?: string
+  /** LLM generation parameters */
+  modelParameters?: {
+    temperature?: number
+    max_tokens?: number
+  }
+  /** Passthrough metadata (test group ID, expectation IDs, etc.) */
+  metadata?: Record<string, unknown>
+  /** HTTP/DB events from the same trace to freeze during execution */
+  frozenEvents?: { id: number; type: string; name: string; input: unknown; output: unknown; timestamp: number; durationMs: number | null; streamed?: boolean; streamRaw?: string | null }[]
+}
+
+export interface PortalTaskResult {
+  taskId: string
+  ok: boolean
+  output: unknown
+  error?: string
+  durationMs: number
+  usage?: {
+    inputTokens?: number
+    outputTokens?: number
+    totalTokens?: number
+  }
+  /** Echoed from the original task */
+  metadata?: Record<string, unknown>
+}
+
+export interface PortalServerOptions {
+  /** Port to listen on (default 4574) */
+  port?: number
+  /** Backend URL to POST results to */
+  backendUrl: string
+  /** Auth token for incoming and outgoing requests */
+  apiKey?: string
+  /** Project root directory (default process.cwd()) */
+  cwd?: string
+  /**
+   * Allowed origin domains that may send requests to this portal.
+   * By default only the `backendUrl` domain and localhost are allowed.
+   * Provide additional origins (e.g. 'https://app.elasticdash.com') to extend.
+   */
+  allowedOrigins?: string[]
+}
+
+export interface PortalServerHandle {
+  port: number
+  url: string
+  close: () => Promise<void>
+}
+
+export interface PortalStatus {
+  ok: boolean
+  queueLength: number
+  processing: string | null
+  completed: number
+  failed: number
+}

package/src/utils/debug.ts

@@ -0,0 +1,8 @@
+const DEBUG_KEY = 'ELASTICDASH_DEBUG'
+
+/** Log only when ELASTICDASH_DEBUG=1 is set. Writes to stderr so callers parsing stdout (e.g. `elasticdash run-tool`) get a clean JSON channel. */
+export function debugLog(...args: unknown[]): void {
+  if (typeof process !== 'undefined' && process.env?.[DEBUG_KEY] === '1') {
+    console.error(...args)
+  }
+}

package/src/utils/license-error.ts

@@ -0,0 +1,43 @@
+/**
+ * License-error notifier.
+ *
+ * The ElasticDash backend returns HTTP 402 ("Payment Required") when the
+ * caller's account has no available license / active plan. The SDK can hit
+ * this from several places (telemetry batcher, telemetry push, CI upload),
+ * so we centralise the user-facing message here and print it at most once
+ * per process to avoid spamming stderr on every retry / event.
+ */
+
+const BUY_URL = 'https://app.elasticdash.com'
+
+let alreadyNotified = false
+
+/**
+ * If `status` is 402, print a one-time message directing the user to buy a
+ * plan and return true. Returns false otherwise so callers can keep their
+ * existing branching.
+ *
+ * @param status   HTTP status code from the failed response.
+ * @param context  Optional short label for the failing subsystem (e.g.
+ *                 "telemetry", "ci-upload") — included in the printed
+ *                 prefix to help users locate the source.
+ */
+export function notifyLicenseError(status: number, context?: string): boolean {
+  if (status !== 402) return false
+  if (alreadyNotified) return true
+  alreadyNotified = true
+  const prefix = context ? `[elasticdash:${context}]` : '[elasticdash]'
+  // Write directly to stderr so this message is always visible regardless
+  // of the user's ELASTICDASH_DEBUG setting — a missing license is a hard
+  // blocker, not a debug detail.
+  console.error('')
+  console.error(`${prefix} No available license for your ElasticDash account (HTTP 402).`)
+  console.error(`${prefix} Please go to ${BUY_URL} to buy a plan.`)
+  console.error('')
+  return true
+}
+
+/** Reset the one-time guard. Exposed for tests only. */
+export function resetLicenseErrorNotifier(): void {
+  alreadyNotified = false
+}

package/src/utils/redact.ts

@@ -0,0 +1,25 @@
+/**
+ * Deep-clones a value and replaces any object property whose key matches
+ * one of `keys` (case-insensitive) with "[REDACTED]".
+ * Returns the original value when `keys` is empty.
+ */
+export function redactPayload(value: unknown, keys: string[]): unknown {
+  if (keys.length === 0) return value
+  const lowerKeys = new Set(keys.map((k) => k.toLowerCase()))
+  return redact(value, lowerKeys)
+}
+
+function redact(value: unknown, keys: Set<string>): unknown {
+  if (value === null || value === undefined) return value
+  if (typeof value !== 'object') return value
+
+  if (Array.isArray(value)) {
+    return value.map((item) => redact(item, keys))
+  }
+
+  const out: Record<string, unknown> = {}
+  for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
+    out[k] = keys.has(k.toLowerCase()) ? '[REDACTED]' : redact(v, keys)
+  }
+  return out
+}