elasticdash-test 0.1.18-alpha-21 → 0.1.18-alpha-23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "elasticdash-test",
-  "version": "0.1.18-alpha-21",
+  "version": "0.1.18-alpha-23",
   "description": "AI-native test runner for ElasticDash workflow testing",
   "type": "module",
   "bin": {

package/src/interceptors/ai-interceptor.ts

@@ -14,6 +14,55 @@ function isAIWrapperActive(): boolean {
   return ((globalThis as Record<string, unknown>)[AI_WRAPPER_KEY] as number ?? 0) > 0
 }
 
+/**
+ * When inside a wrapAI call, the ai-interceptor captures the actual HTTP
+ * request payload and stashes it here so wrapAI can attach it to its event.
+ */
+const LLM_REQUEST_KEY = '__elasticdash_last_llm_request__'
+export interface CapturedLLMRequest {
+  url: string
+  provider: string
+  model: string
+  messages?: unknown[]
+  body: Record<string, unknown>
+  promptSnippet?: string
+  usage?: UsageInfo
+}
+
+export function consumeCapturedLLMRequest(): CapturedLLMRequest | undefined {
+  const g = globalThis as Record<string, unknown>
+  const req = g[LLM_REQUEST_KEY] as CapturedLLMRequest | undefined
+  if (req) g[LLM_REQUEST_KEY] = undefined
+  return req
+}
+
+function extractPromptSnippet(body: Record<string, unknown>): string | undefined {
+  let messages: unknown[] | undefined
+  if (Array.isArray(body.messages)) messages = body.messages
+  else if (Array.isArray(body.contents)) messages = body.contents
+
+  if (!messages || messages.length === 0) return undefined
+
+  // Find the last user message
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i] as Record<string, unknown> | undefined
+    if (!msg) continue
+    if (msg.role === 'user') {
+      let content = msg.content
+      if (Array.isArray(content)) {
+        content = content
+          .map((b: unknown) => (b && typeof b === 'object' ? String((b as Record<string, unknown>).text ?? '') : String(b)))
+          .filter(Boolean)
+          .join('')
+      }
+      if (typeof content === 'string') {
+        return content.slice(0, 100)
+      }
+    }
+  }
+  return undefined
+}
+
 type UsageInfo = { inputTokens?: number; outputTokens?: number; totalTokens?: number }
 
 function extractUsage(provider: string, body: Record<string, unknown>): UsageInfo | undefined {
@@ -397,9 +446,45 @@ export function installAIInterceptor(): void {
 
     const provider = detectProvider(url)
 
-    // Skip recording when inside a wrapAI call to avoid duplicate events
+    // Skip recording when inside a wrapAI call to avoid duplicate events,
+    // but capture the actual HTTP request body and response usage so wrapAI
+    // can attach them to its event.
     if (provider && isAIWrapperActive()) {
-      return originalFetch!(input, init)
+      let capturedReq: Record<string, unknown> | undefined
+      let capturedModel = 'unknown'
+      let capturedMessages: unknown[] | undefined
+      let capturedSnippet: string | undefined
+      try {
+        const rawBody = init?.body
+        if (rawBody && typeof rawBody === 'string') {
+          capturedReq = JSON.parse(rawBody) as Record<string, unknown>
+          capturedModel = extractModel(provider, capturedReq, url)
+          capturedMessages = Array.isArray(capturedReq.messages) ? capturedReq.messages : Array.isArray(capturedReq.contents) ? capturedReq.contents : undefined
+          capturedSnippet = extractPromptSnippet(capturedReq)
+        }
+      } catch {
+        // Ignore parse errors
+      }
+
+      const response = await originalFetch!(input, init)
+
+      // Extract usage from the response (clone to avoid consuming the body)
+      if (capturedReq) {
+        const captured: CapturedLLMRequest = {
+          url, provider, model: capturedModel, messages: capturedMessages,
+          body: capturedReq, promptSnippet: capturedSnippet,
+        }
+        try {
+          const cloned = response.clone()
+          const responseBody = await cloned.json() as Record<string, unknown>
+          captured.usage = extractUsage(provider, responseBody)
+        } catch {
+          // Streaming or non-JSON response — usage not available from response
+        }
+        ;(globalThis as Record<string, unknown>)[LLM_REQUEST_KEY] = captured
+      }
+
+      return response
     }
 
     const traceAtCall = getCurrentTrace()

package/src/interceptors/workflow-ai.ts

@@ -2,6 +2,7 @@ import { getCaptureContext } from '../capture/recorder.js'
 import { rawDateNow } from './side-effects.js'
 import { getHttpRunContext, getHttpFrozenEvent, getHttpPromptMock, getHttpUserPromptMock, getHttpAIMock, pushTelemetryEvent, tryAutoInitHttpContext, getObservabilityContext } from './telemetry-push.js'
 import { resolveAIMock, resolvePromptMock, resolveUserPromptMock } from '../internals/mock-resolver.js'
+import { consumeCapturedLLMRequest } from './ai-interceptor.js'
 import type { WorkflowEvent } from '../capture/event.js'
 
 /**
@@ -57,6 +58,23 @@ function extractUsage(output: unknown): UsageInfo | undefined {
   return undefined
 }
 
+/**
+ * After callFn returns, consume any captured LLM request from ai-interceptor
+ * and merge it into the event input so the recorded event contains the actual
+ * HTTP payload sent to the LLM (messages, model, parameters).
+ * Also returns captured usage when the app's output doesn't include it.
+ */
+function enrichFromLLMCapture(input: unknown, appUsage: UsageInfo | undefined): { input: unknown; usage: UsageInfo | undefined } {
+  const captured = consumeCapturedLLMRequest()
+  if (!captured) return { input, usage: appUsage }
+  const enrichedInput = (input && typeof input === 'object')
+    ? { ...(input as Record<string, unknown>), llmRequest: captured.body, promptSnippet: captured.promptSnippet }
+    : { originalInput: input, llmRequest: captured.body, promptSnippet: captured.promptSnippet }
+  // Prefer app-level usage if available, fall back to HTTP-level usage
+  const usage = appUsage ?? captured.usage
+  return { input: enrichedInput, usage }
+}
+
 function isReadableStream(v: unknown): v is ReadableStream<Uint8Array> {
   return (
     typeof v === 'object' &&
@@ -127,10 +145,8 @@ function wrapAsyncIterable<T>(
 export function wrapAI<Args extends unknown[], R>(
   modelName: string,
   callFn: (...args: Args) => Promise<R>,
-  options?: { model?: string; provider?: string },
+  _options?: { model?: string; provider?: string },
 ): (...args: Args) => Promise<R> {
-  const actualModel = options?.model
-  const actualProvider = options?.provider
   return async (...args: Args): Promise<R> => {
     await tryAutoInitHttpContext()
     const ctx = getCaptureContext()
@@ -146,22 +162,19 @@ export function wrapAI<Args extends unknown[], R>(
     // Observability-only mode: record and push, no mocks/replay
     if (!ctx && !httpCtx && obsCtx) {
       const id = obsCtx.nextId()
-      const rawInput = args.length === 1 ? args[0] : args
-      // Attach actual model/provider so reruns use the correct API model name
-      const input = (actualModel || actualProvider)
-        ? { ...(rawInput && typeof rawInput === 'object' ? rawInput as Record<string, unknown> : { prompt: rawInput }), ...(actualModel ? { model: actualModel } : {}), ...(actualProvider ? { provider: actualProvider } : {}) }
-        : rawInput
+      const input = args.length === 1 ? args[0] : args
       try {
         const output = await callFn(...args)
+        const enriched = enrichFromLLMCapture(input, extractUsage(output))
 
         if (isReadableStream(output)) {
           const [streamForCaller, streamForRecorder] = output.tee()
           bufferReadableStream(streamForRecorder).then((rawText) => {
             const durationMs = rawDateNow() - start
-            pushTelemetryEvent({ id, type: 'ai', name: modelName, input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs })
+            pushTelemetryEvent({ id, type: 'ai', name: modelName, input: enriched.input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs, ...(enriched.usage ? { usage: enriched.usage } : {}) })
           }).catch(() => {
             const durationMs = rawDateNow() - start
-            pushTelemetryEvent({ id, type: 'ai', name: modelName, input, output: null, streamed: true, streamRaw: '', timestamp: start, durationMs })
+            pushTelemetryEvent({ id, type: 'ai', name: modelName, input: enriched.input, output: null, streamed: true, streamRaw: '', timestamp: start, durationMs, ...(enriched.usage ? { usage: enriched.usage } : {}) })
           })
           return streamForCaller as unknown as R
         }
@@ -170,23 +183,23 @@ export function wrapAI<Args extends unknown[], R>(
           return wrapAsyncIterable(output, (chunks) => {
             const durationMs = rawDateNow() - start
             const rawText = chunks.map((c) => (typeof c === 'string' ? c : JSON.stringify(c))).join('')
-            pushTelemetryEvent({ id, type: 'ai', name: modelName, input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs })
+            pushTelemetryEvent({ id, type: 'ai', name: modelName, input: enriched.input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs, ...(enriched.usage ? { usage: enriched.usage } : {}) })
           }) as unknown as R
         }
 
         const durationMs = rawDateNow() - start
-        const usage = extractUsage(output)
         const event: WorkflowEvent = {
-          id, type: 'ai', name: modelName, input, output,
+          id, type: 'ai', name: modelName, input: enriched.input, output,
           timestamp: start, durationMs,
-          ...(usage ? { usage } : {}),
+          ...(enriched.usage ? { usage: enriched.usage } : {}),
         }
         pushTelemetryEvent(event)
         return output
       } catch (e) {
+        const enriched = enrichFromLLMCapture(input, undefined)
         const durationMs = rawDateNow() - start
         pushTelemetryEvent({
-          id, type: 'ai', name: modelName, input,
+          id, type: 'ai', name: modelName, input: enriched.input,
           output: { error: String(e) }, timestamp: start, durationMs,
         })
         throw e
@@ -314,15 +327,16 @@ export function wrapAI<Args extends unknown[], R>(
 
     try {
       const output = await callFn(...httpEffectiveArgs)
+      const enriched = enrichFromLLMCapture(input, extractUsage(output))
 
       if (isReadableStream(output)) {
         const [streamForCaller, streamForRecorder] = output.tee()
         bufferReadableStream(streamForRecorder).then((rawText) => {
           const durationMs = rawDateNow() - start
-          pushTelemetryEvent({ id, type: 'ai', name: modelName, input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs })
+          pushTelemetryEvent({ id, type: 'ai', name: modelName, input: enriched.input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs, ...(enriched.usage ? { usage: enriched.usage } : {}) })
         }).catch(() => {
           const durationMs = rawDateNow() - start
-          pushTelemetryEvent({ id, type: 'ai', name: modelName, input, output: null, streamed: true, streamRaw: '', timestamp: start, durationMs })
+          pushTelemetryEvent({ id, type: 'ai', name: modelName, input: enriched.input, output: null, streamed: true, streamRaw: '', timestamp: start, durationMs, ...(enriched.usage ? { usage: enriched.usage } : {}) })
         })
         return streamForCaller as unknown as R
       }
@@ -331,23 +345,23 @@ export function wrapAI<Args extends unknown[], R>(
         return wrapAsyncIterable(output, (chunks) => {
           const durationMs = rawDateNow() - start
           const rawText = chunks.map((c) => (typeof c === 'string' ? c : JSON.stringify(c))).join('')
-          pushTelemetryEvent({ id, type: 'ai', name: modelName, input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs })
+          pushTelemetryEvent({ id, type: 'ai', name: modelName, input: enriched.input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs, ...(enriched.usage ? { usage: enriched.usage } : {}) })
         }) as unknown as R
       }
 
       const durationMs = rawDateNow() - start
-      const usage = extractUsage(output)
       const event: WorkflowEvent = {
-        id, type: 'ai', name: modelName, input, output,
+        id, type: 'ai', name: modelName, input: enriched.input, output,
         timestamp: start, durationMs,
-        ...(usage ? { usage } : {}),
+        ...(enriched.usage ? { usage: enriched.usage } : {}),
       }
       pushTelemetryEvent(event)
       return output
     } catch (e) {
+      const enriched = enrichFromLLMCapture(input, undefined)
       const durationMs = rawDateNow() - start
       pushTelemetryEvent({
-        id, type: 'ai', name: modelName, input,
+        id, type: 'ai', name: modelName, input: enriched.input,
         output: { error: String(e) }, timestamp: start, durationMs,
       })
       throw e