elasticdash-test 0.1.14 → 0.1.16

package/src/html/dashboard.html

@@ -313,6 +313,10 @@
             // ---- Tool Mock Config State ----
             window._toolMockConfig = {}; // { toolName: { mode: 'live'|'mock-all'|'mock-specific', callIndices: [], mockData: {} } }
 
+            // ---- Prompt Mock Config State ----
+            // { [originalSystemPrompt]: newSystemPrompt } — only keys where user enabled the override
+            window._promptMockConfig = {};
+
             function getToolsFromTrace() {
                 // Extract unique tool names and their call details from the uploaded trace observations
                 const toolCalls = {};
@@ -468,6 +472,136 @@
                 return html;
             }
 
+            // ---- Prompt Mock Helpers ----
+
+            /** Extract the system prompt string from an LLM call input object or JSON string. */
+            function extractSystemPromptFromInput(input) {
+                // Input may arrive as a JSON-encoded string (e.g. from Langfuse traces)
+                if (typeof input === 'string') {
+                    try { input = JSON.parse(input); } catch(e) { return null; }
+                }
+                if (!input || typeof input !== 'object') return null;
+                // Anthropic style: { system: "...", messages: [...] }
+                if (typeof input.system === 'string') return input.system;
+                // Custom wrapAI callers: { systemPrompt: "...", messages: [...] }
+                if (typeof input.systemPrompt === 'string' && input.systemPrompt.length > 0) return input.systemPrompt;
+                // OpenAI / plain array: messages with role === "system"
+                var msgs = Array.isArray(input.messages) ? input.messages : (Array.isArray(input) ? input : null);
+                if (msgs) {
+                    for (var i = 0; i < msgs.length; i++) {
+                        var m = msgs[i];
+                        if (m && typeof m === 'object' && m.role === 'system' && typeof m.content === 'string') {
+                            return m.content;
+                        }
+                    }
+                }
+                return null;
+            }
+
+            /**
+             * Returns an array of unique system prompts observed across all GENERATION observations.
+             * Each entry: { systemPrompt, modelName, count, rowIndex }
+             */
+            function getSystemPromptsFromTrace() {
+                var seen = []; // [{ systemPrompt, modelName, count }]
+                var seenMap = {}; // systemPrompt -> index in seen
+                currentObservations.forEach(function(obs) {
+                    if (obs.type !== 'GENERATION') return;
+                    var sp = extractSystemPromptFromInput(obs.input);
+                    if (!sp) return;
+                    if (seenMap[sp] === undefined) {
+                        seenMap[sp] = seen.length;
+                        seen.push({ systemPrompt: sp, modelName: obs.model || obs.name || '(unknown)', count: 0 });
+                    }
+                    seen[seenMap[sp]].count++;
+                });
+                return seen.map(function(e, i) { return Object.assign({}, e, { rowIndex: i }); });
+            }
+
+            function renderPromptMockSection() {
+                const prompts = getSystemPromptsFromTrace();
+                if (prompts.length === 0) {
+                    return '<div style="color:#999;font-size:13px;padding:6px 0;">No system prompts detected in trace. Only AI calls with a system prompt can be mocked here.</div>';
+                }
+
+                let html = '<div style="max-height:360px;overflow-y:auto;border:1px solid #e0e0e0;border-radius:6px;">';
+                html += '<table style="width:100%;border-collapse:collapse;font-size:13px;">';
+                html += '<thead><tr style="background:#f5f5f5;">';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;width:24px;"></th>';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">Model</th>';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">Uses</th>';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">System Prompt (override applies to all calls using this prompt)</th>';
+                html += '</tr></thead><tbody>';
+
+                prompts.forEach(function(row) {
+                    const key = row.systemPrompt;
+                    const isEnabled = window._promptMockConfig[key] !== undefined;
+                    const currentVal = isEnabled ? window._promptMockConfig[key] : row.systemPrompt;
+                    const preview = key.length > 80 ? key.slice(0, 80) + '…' : key;
+
+                    html += '<tr class="prompt-mock-row" data-row-index="' + row.rowIndex + '" style="border-bottom:1px solid #f0f0f0;vertical-align:top;">';
+
+                    // Checkbox column
+                    html += '<td style="padding:8px 10px;white-space:nowrap;">';
+                    html += '<input type="checkbox" class="prompt-mock-enable" title="Override this system prompt"' + (isEnabled ? ' checked' : '') + ' onchange="window.onPromptMockToggle(' + row.rowIndex + ', this.checked)">';
+                    html += '</td>';
+
+                    // Model column
+                    html += '<td style="padding:8px 10px;font-family:Monaco,monospace;font-size:12px;white-space:nowrap;">' + esc(row.modelName) + '</td>';
+
+                    // Uses count column
+                    html += '<td style="padding:8px 10px;color:#555;white-space:nowrap;">' + row.count + 'x</td>';
+
+                    // System prompt column
+                    html += '<td style="padding:8px 10px;width:100%;">';
+                    if (!isEnabled) {
+                        html += '<div style="font-size:11px;color:#888;font-style:italic;font-family:Monaco,monospace;max-width:380px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="' + esc(key) + '">' + esc(preview) + '</div>';
+                    } else {
+                        html += '<textarea class="prompt-mock-input" data-row-index="' + row.rowIndex + '" style="width:100%;box-sizing:border-box;font-size:11px;font-family:Monaco,monospace;padding:4px;border:1px solid #ddd;border-radius:4px;min-height:72px;resize:vertical;" oninput="window.onPromptMockInput(' + row.rowIndex + ', this.value)">' + esc(currentVal) + '</textarea>';
+                    }
+                    html += '</td>';
+
+                    html += '</tr>';
+                });
+
+                html += '</tbody></table></div>';
+                return html;
+            }
+
+            function buildPromptMockConfigFromUI() {
+                // Return _promptMockConfig as-is (string key → string value), filtering out blank values
+                const config = {};
+                Object.keys(window._promptMockConfig).forEach(function(key) {
+                    const val = window._promptMockConfig[key];
+                    if (typeof val === 'string' && val.trim()) config[key] = val;
+                });
+                return config;
+            }
+
+            window.onPromptMockToggle = function(rowIndex, enabled) {
+                const prompts = getSystemPromptsFromTrace();
+                const row = prompts[rowIndex];
+                if (!row) return;
+                const key = row.systemPrompt;
+                if (!enabled) {
+                    delete window._promptMockConfig[key];
+                } else {
+                    // Pre-fill with the original system prompt so user can edit from there
+                    if (window._promptMockConfig[key] === undefined) {
+                        window._promptMockConfig[key] = row.systemPrompt;
+                    }
+                }
+                const container = document.getElementById('promptMockContainer');
+                if (container) container.innerHTML = renderPromptMockSection();
+            };
+
+            window.onPromptMockInput = function(rowIndex, value) {
+                const prompts = getSystemPromptsFromTrace();
+                const row = prompts[rowIndex];
+                if (!row) return;
+                window._promptMockConfig[row.systemPrompt] = value;
+            };
+
             window.onToolMockModeChange = function(toolName, mode) {
                 if (!window._toolMockConfig[toolName]) window._toolMockConfig[toolName] = { mode: 'live' };
                 // Save current mock data before switching
@@ -498,7 +632,8 @@
 
             window.openLiveValidationDialog = function() {
                 if (window.liveValidationDialog) return;
-                window._toolMockConfig = {}; // Reset mock config each time dialog opens
+                window._toolMockConfig = {};    // Reset mock configs each time dialog opens
+                window._promptMockConfig = {};
 
                 const hasTraceTools = currentObservations.some(function(o) { return o.type === 'TOOL'; });
                 const hasRegisteredTools = codeIndex.tools && codeIndex.tools.length > 0;
@@ -524,8 +659,15 @@
                                 Show all registered tools
                             </label>
                         </div>
-                        <div id="toolMockContainer"></div>
+                        <div id="toolMockContainer" style="max-height:160px;overflow-y:auto;"></div>
                     </div>` : ''}
+                    <div style="border-top:1px solid #eee;padding-top:16px;margin-bottom:16px;">
+                        <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:6px;">
+                            <div style="font-size:15px;font-weight:600;">Prompt Mocking</div>
+                            <span style="font-size:12px;color:#888;">Check a row to replace the system prompt for all calls using it</span>
+                        </div>
+                        <div id="promptMockContainer" style="max-height:160px;overflow-y:auto;"></div>
+                    </div>
                     <div style="display:flex;gap:12px;justify-content:space-between;align-items:center;">
                         <span id="liveValidationProgress" style="font-size:14px;color:#555;"></span>
                         <div style="display:flex;gap:12px;">
@@ -536,11 +678,15 @@
                     </div>
                 `;
                 document.body.appendChild(window.liveValidationDialog);
-                // Render the tool mock section after DOM insertion
+                // Render mock sections after DOM insertion
                 const toolMockContainer = document.getElementById('toolMockContainer');
                 if (toolMockContainer) {
                     toolMockContainer.innerHTML = renderToolMockSection(false);
                 }
+                const promptMockContainer = document.getElementById('promptMockContainer');
+                if (promptMockContainer) {
+                    promptMockContainer.innerHTML = renderPromptMockSection();
+                }
                 document.getElementById('cancelLiveValidation').onclick = function() {
                     window.liveValidationDialog.remove();
                     window.liveValidationDialog = null;
@@ -549,9 +695,11 @@
                     const count = parseInt(document.getElementById('liveValidationCount').value, 10);
                     const sequential = document.getElementById('liveValidationSequential').checked;
                     if (count >= 1) {
-                        // Build the tool mock config from UI state and persist for "Run from here"
+                        // Build mock configs from UI state and persist for "Run from here"
                         const toolMockConfig = buildToolMockConfigFromUI();
                         window._toolMockConfig = toolMockConfig;
+                        const promptMockConfig = buildPromptMockConfigFromUI();
+                        window._promptMockConfig = promptMockConfig;
                         const submitBtn = document.getElementById('submitLiveValidation');
                         submitBtn.disabled = true;
                         submitBtn.textContent = 'Validating...';
@@ -588,7 +736,7 @@
                             const collectedTraces = [];
                             let fatalError = null;
                             for (let i = 0; i < count; i++) {
-                                const singlePayload = { workflowName: selectedWorkflow?.name, runCount: 1, sequential: false, observations: currentObservations, toolMockConfig };
+                                const singlePayload = { workflowName: selectedWorkflow?.name, runCount: 1, sequential: false, observations: currentObservations, toolMockConfig, promptMockConfig };
                                 try {
                                     const response = await fetch('/api/validate-workflow', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(singlePayload) });
                                     const data = await response.json();
@@ -607,7 +755,7 @@
                         } else {
                             // Parallel mode: single bulk request
                             if (progressEl) progressEl.textContent = `Running ${count} workflow run${count !== 1 ? 's' : ''} in parallel…`;
-                            const payload = { workflowName: selectedWorkflow?.name, runCount: count, sequential: false, observations: currentObservations, toolMockConfig };
+                            const payload = { workflowName: selectedWorkflow?.name, runCount: count, sequential: false, observations: currentObservations, toolMockConfig, promptMockConfig };
                             try {
                                 const response = await fetch('/api/validate-workflow', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
                                 const data = await response.json();
@@ -1142,7 +1290,7 @@
                     traceLayoutEl.classList.remove("step-4");
                     let headerHtml = '';
                     if (currentStep === 3) {
-                        headerHtml = '<tr><th style="width: 40px;">Check</th><th>Name</th><th>Type</th></tr>';
+                        headerHtml = '<tr><th style="width: 40px;">Check</th><th>Name</th><th>Type</th><th>Duration</th></tr>';
                     }
                     traceLayoutEl.innerHTML = `
                         <div class="trace-left">
@@ -1311,6 +1459,7 @@
                         snapshotId: liveTrace.snapshotId,
                         observations: currentObservations,
                         toolMockConfig: window._toolMockConfig || {},
+                        promptMockConfig: window._promptMockConfig || {},
                     };
                     const response = await fetch('/api/run-from-breakpoint', {
                         method: 'POST',
@@ -1342,7 +1491,7 @@
                 step5RerunInFlight = true;
                 renderObservationTable();
                 try {
-                    const payload = { workflowName: selectedWorkflow?.name, runCount: 1, sequential: false, observations: currentObservations };
+                    const payload = { workflowName: selectedWorkflow?.name, runCount: 1, sequential: false, observations: currentObservations, toolMockConfig: window._toolMockConfig || {}, promptMockConfig: window._promptMockConfig || {} };
                     const response = await fetch('/api/validate-workflow', {
                         method: 'POST',
                         headers: { 'Content-Type': 'application/json' },
@@ -1393,6 +1542,7 @@
                         },
                         snapshotId: liveTrace.snapshotId,
                         toolMockConfig: window._toolMockConfig || {},
+                        promptMockConfig: window._promptMockConfig || {},
                     };
                     const response = await fetch('/api/resume-agent-from-task', {
                         method: 'POST',

package/src/index.ts

@@ -9,7 +9,8 @@ export { installAIInterceptor, uninstallAIInterceptor } from './interceptors/ai-
 export { startLLMProxy, fetchCapturedTrace } from './proxy/llm-capture.js'
 export { recordToolCall } from './tracing.js'
 export { safeRecordToolCall, isWorker } from './internals/conditional-recorder.js'
-export { resolveMock } from './internals/mock-resolver.js'
+export { resolveMock, resolveAIMock, resolvePromptMock } from './internals/mock-resolver.js'
+export type { AIMockEntry, AIMockConfig } from './internals/mock-resolver.js'
 export type { TestResult, FileResult, RunnerOptions } from './runner.js'
 export type { AITestContext, TraceHandle, LLMStep, ToolCall, CustomStep, TraceStep, RunnerHooks } from './trace-adapter/context.js'
 // Workflow capture & replay
@@ -19,7 +20,7 @@ export type { CaptureContext } from './capture/recorder.js'
 export { ReplayController } from './capture/replay.js'
 export { wrapTool } from './interceptors/tool.js'
 export { wrapAI } from './interceptors/workflow-ai.js'
-export { setHttpRunContext, initHttpRunContext, getHttpRunContext } from './interceptors/telemetry-push.js'
+export { setHttpRunContext, initHttpRunContext, getHttpRunContext, getHttpFrozenEvent, getHttpPromptMock, pushTelemetryEvent, tryAutoInitHttpContext, runInHttpContext, runWithInitializedHttpContext } from './interceptors/telemetry-push.js'
 export { wrapDB, wrapPgClient, wrapKnex, wrapMongoCollection, wrapRedisClient } from './interceptors/db.js'
 export { installDBAutoInterceptor, uninstallDBAutoInterceptor } from './interceptors/db-auto.js'
 export { interceptFetch, restoreFetch, readVercelAIStream } from './interceptors/http.js'

package/src/interceptors/telemetry-push.ts

@@ -1,11 +1,15 @@
 import { AsyncLocalStorage } from 'node:async_hooks'
+import { randomUUID } from 'node:crypto'
 import type { WorkflowEvent } from '../capture/event.js'
+import { extractSystemPrompt, replaceSystemPrompt } from '../internals/mock-resolver.js'
 
 interface HttpRunContext {
   runId: string
   dashboardUrl: string
   nextId: () => number
   frozenEvents: Map<number, WorkflowEvent>
+  /** System-prompt-keyed overrides: original system prompt → replacement system prompt */
+  promptMocks: Map<string, string>
 }
 
 const g = globalThis as Record<string, unknown>
@@ -15,16 +19,22 @@ const httpRunAls: AsyncLocalStorage<HttpRunContext | undefined> =
   new AsyncLocalStorage<HttpRunContext | undefined>()
 if (!g[HTTP_RUN_ALS_KEY]) g[HTTP_RUN_ALS_KEY] = httpRunAls
 
-function buildContext(runId: string, dashboardUrl: string, frozenEvents: WorkflowEvent[]): HttpRunContext {
+function buildContext(
+  runId: string,
+  dashboardUrl: string,
+  frozenEvents: WorkflowEvent[],
+  promptMocksRecord: Record<string, string> = {},
+): HttpRunContext {
   let counter = 0
   const frozenMap = new Map<number, WorkflowEvent>()
   for (const e of frozenEvents) frozenMap.set(e.id, e)
-  return { runId, dashboardUrl, nextId: () => ++counter, frozenEvents: frozenMap }
+  const promptMocksMap = new Map<string, string>(Object.entries(promptMocksRecord))
+  return { runId, dashboardUrl, nextId: () => ++counter, frozenEvents: frozenMap, promptMocks: promptMocksMap }
 }
 
 /** Synchronous setup — use when there are no frozen events (live run with no replay). */
 export function setHttpRunContext(runId: string, dashboardUrl: string): void {
-  httpRunAls.enterWith(buildContext(runId, dashboardUrl, []))
+  httpRunAls.enterWith(buildContext(runId, dashboardUrl, [], {}))
 }
 
 /**
@@ -34,16 +44,19 @@ export function setHttpRunContext(runId: string, dashboardUrl: string): void {
  */
 export async function initHttpRunContext(runId: string, dashboardUrl: string): Promise<void> {
   let frozenEvents: WorkflowEvent[] = []
+  let promptMocks: Record<string, string> = {}
   try {
     const res = await fetch(`${dashboardUrl}/api/run-configs/${runId}`)
     if (res.ok) {
-      const data = await res.json() as { frozenEvents?: WorkflowEvent[] }
+      const data = await res.json() as { frozenEvents?: WorkflowEvent[]; promptMocks?: Record<string, string> }
       frozenEvents = Array.isArray(data.frozenEvents) ? data.frozenEvents : []
+      promptMocks = (data.promptMocks && typeof data.promptMocks === 'object' && !Array.isArray(data.promptMocks))
+        ? data.promptMocks : {}
     }
   } catch {
     // Dashboard unreachable or run config not registered — proceed with live execution
   }
-  httpRunAls.enterWith(buildContext(runId, dashboardUrl, frozenEvents))
+  httpRunAls.enterWith(buildContext(runId, dashboardUrl, frozenEvents, promptMocks))
 }
 
 export function getHttpRunContext(): HttpRunContext | undefined {
@@ -55,13 +68,151 @@ export function getHttpFrozenEvent(id: number): WorkflowEvent | undefined {
   return httpRunAls.getStore()?.frozenEvents.get(id)
 }
 
+/**
+ * If a prompt mock is configured for the system prompt found in `input`, returns
+ * a copy of `input` with the system prompt replaced. Otherwise returns `undefined`.
+ */
+export function getHttpPromptMock(input: unknown): unknown | undefined {
+  const ctx = httpRunAls.getStore()
+  if (!ctx || ctx.promptMocks.size === 0) {
+    console.log(`[elasticdash] getHttpPromptMock: skip — promptMocks.size=${ctx?.promptMocks.size ?? 'no ctx'}`)
+    return undefined
+  }
+  const systemPrompt = extractSystemPrompt(input)
+  if (systemPrompt === undefined) {
+    const inputKeys = (input && typeof input === 'object') ? Object.keys(input as object).join(',') : typeof input
+    console.log(`[elasticdash] getHttpPromptMock: no system prompt found in input (keys: ${inputKeys})`)
+    return undefined
+  }
+  const newSystemPrompt = ctx.promptMocks.get(systemPrompt)
+  console.log(`[elasticdash] getHttpPromptMock: extracted system prompt (len=${systemPrompt.length}, first50=${JSON.stringify(systemPrompt.slice(0,50))}) — mock found=${newSystemPrompt !== undefined}`)
+  if (newSystemPrompt !== undefined) {
+    console.log(`[elasticdash] getHttpPromptMock: available mock keys=${JSON.stringify([...ctx.promptMocks.keys()].map(k => k.slice(0,50)))}`)
+  }
+  if (newSystemPrompt === undefined) {
+    console.log(`[elasticdash] getHttpPromptMock: no mock for this prompt. Available mock keys (first 50 chars each): ${JSON.stringify([...ctx.promptMocks.keys()].map(k => k.slice(0,50)))}`)
+    return undefined
+  }
+  return replaceSystemPrompt(input, newSystemPrompt)
+}
+
 export function pushTelemetryEvent(event: WorkflowEvent): void {
   const ctx = httpRunAls.getStore()
-  if (!ctx) return
+  if (!ctx) {
+    console.log(`[elasticdash] pushTelemetryEvent: no HTTP context, dropping event type=${event.type} name=${('name' in event ? event.name : '?')}`)
+    return
+  }
   const { runId, dashboardUrl } = ctx
+  console.log(`[elasticdash] pushTelemetryEvent: posting event type=${event.type} name=${('name' in event ? event.name : '?')} runId=${runId} to ${dashboardUrl}`)
   fetch(`${dashboardUrl}/api/trace-events`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({ runId, event }),
-  }).catch(() => {})
+  }).then(r => {
+    console.log(`[elasticdash] pushTelemetryEvent: response status=${r.status} for type=${event.type} name=${('name' in event ? event.name : '?')}`)
+  }).catch(e => {
+    console.log(`[elasticdash] pushTelemetryEvent: fetch failed: ${e instanceof Error ? e.message : String(e)}`)
+  })
+}
+
+const AUTO_INIT_KEY = '__elasticdash_auto_init_promise__'
+
+/**
+ * Lazily initialises an HTTP run context from environment variables when none
+ * has been set up explicitly. Only activates when `ELASTICDASH_SERVER` is set.
+ *
+ * - If `ELASTICDASH_RUN_ID` is also set, calls `initHttpRunContext` so that
+ *   frozen steps are fetched from the dashboard (enables step freezing).
+ * - Otherwise calls `setHttpRunContext` with a fresh UUID (live/telemetry mode).
+ *
+ * The initialisation runs at most once per process — subsequent calls are
+ * no-ops once the context is established. Errors (e.g. dashboard unreachable)
+ * are swallowed so that live execution always continues unaffected.
+ *
+ * Typical usage: set `ELASTICDASH_SERVER=http://localhost:4573` and optionally
+ * `ELASTICDASH_RUN_ID=<id>` before starting your server or script. Every
+ * `wrapTool` / `wrapAI` call will then auto-connect to the dashboard without
+ * any explicit `initHttpRunContext` call in your code.
+ */
+/**
+ * Runs `callback` inside a fresh HTTP run context scoped to `runId` / `dashboardUrl`.
+ * Uses `als.run()` which guarantees the store is inherited by all async descendants of
+ * `callback`, even when intermediate code (e.g. Langfuse / OTel) spawns its own async
+ * contexts via `als.run()`. Prefer this over `setHttpRunContext` when wrapping a long-lived
+ * async pipeline such as a streaming route handler.
+ */
+export function runInHttpContext<T>(
+  runId: string,
+  dashboardUrl: string,
+  callback: () => Promise<T>,
+): Promise<T> {
+  return httpRunAls.run(buildContext(runId, dashboardUrl, [], {}), callback)
+}
+
+/**
+ * Fetches frozen events and prompt mocks from the dashboard, then runs `callback`
+ * inside an HTTP run context using `als.run()`. This is the preferred function for
+ * streaming route handlers that sit behind Langfuse / OTel instrumentation:
+ *
+ * - `als.run()` guarantees the elasticdash store is inherited through any nested
+ *   `als.run()` calls made by third-party libraries (e.g. `startActiveObservation`).
+ * - Frozen events and prompt mocks are fetched before the callback so step replay
+ *   and prompt mocking work correctly on reruns.
+ *
+ * Falls back to an empty context (live execution, no replay) if the dashboard is
+ * unreachable or the run config is not found.
+ */
+export async function runWithInitializedHttpContext<T>(
+  runId: string,
+  dashboardUrl: string,
+  callback: () => Promise<T>,
+): Promise<T> {
+  let frozenEvents: WorkflowEvent[] = []
+  let promptMocks: Record<string, string> = {}
+  try {
+    const res = await fetch(`${dashboardUrl}/api/run-configs/${runId}`)
+    if (res.ok) {
+      const data = await res.json() as { frozenEvents?: WorkflowEvent[]; promptMocks?: Record<string, string> }
+      frozenEvents = Array.isArray(data.frozenEvents) ? data.frozenEvents : []
+      promptMocks = (data.promptMocks && typeof data.promptMocks === 'object' && !Array.isArray(data.promptMocks))
+        ? data.promptMocks : {}
+      const mockKeys = Object.keys(promptMocks)
+      console.log(`[elasticdash] runWithInitializedHttpContext: fetched ${mockKeys.length} prompt mocks, ${frozenEvents.length} frozen events`)
+      if (mockKeys.length > 0) {
+        console.log(`[elasticdash] runWithInitializedHttpContext: mock keys (first 80 chars each): ${JSON.stringify(mockKeys.map(k => k.slice(0,80)))}`)
+      }
+    } else {
+      console.log(`[elasticdash] runWithInitializedHttpContext: run-configs fetch returned ${res.status}`)
+    }
+  } catch {
+    // Dashboard unreachable or run config not registered — proceed with live execution
+  }
+  return httpRunAls.run(buildContext(runId, dashboardUrl, frozenEvents, promptMocks), callback)
+}
+
+export async function tryAutoInitHttpContext(): Promise<void> {
+  // Fast path: already initialised in this async context
+  if (getHttpRunContext()) return
+
+  const serverUrl = (typeof process !== 'undefined' && process.env?.ELASTICDASH_SERVER) ?? ''
+  if (!serverUrl) return
+
+  // Deduplicate concurrent first calls within the same process
+  const g = globalThis as Record<string, unknown>
+  if (!g[AUTO_INIT_KEY]) {
+    g[AUTO_INIT_KEY] = (async () => {
+      try {
+        const runId = (typeof process !== 'undefined' && process.env?.ELASTICDASH_RUN_ID) ?? ''
+        if (runId) {
+          await initHttpRunContext(runId, serverUrl)
+        } else {
+          setHttpRunContext(randomUUID(), serverUrl)
+        }
+      } catch {
+        // Dashboard unreachable — fall through to live execution
+      }
+    })()
+  }
+
+  await (g[AUTO_INIT_KEY] as Promise<void>)
 }

package/src/interceptors/tool.ts

@@ -1,7 +1,7 @@
 import { getCaptureContext } from '../capture/recorder.js'
 import { getCurrentTrace } from '../trace-adapter/context.js'
 import { rawDateNow } from './side-effects.js'
-import { getHttpRunContext, getHttpFrozenEvent, pushTelemetryEvent } from './telemetry-push.js'
+import { getHttpRunContext, getHttpFrozenEvent, pushTelemetryEvent, tryAutoInitHttpContext } from './telemetry-push.js'
 
 const TOOL_WRAPPER_ACTIVE_KEY = '__elasticdash_tool_wrapper_active__'
 
@@ -85,6 +85,7 @@ export function wrapTool<Args extends unknown[], R>(
   fn: (...args: Args) => Promise<R>,
 ): (...args: Args) => Promise<R> {
   return async (...args: Args): Promise<R> => {
+    await tryAutoInitHttpContext()
     const ctx = getCaptureContext()
     const httpCtx = getHttpRunContext()
     console.log(`[elasticdash] Tool called: ${name}`, { args })

package/src/interceptors/workflow-ai.ts

@@ -1,6 +1,7 @@
 import { getCaptureContext } from '../capture/recorder.js'
 import { rawDateNow } from './side-effects.js'
-import { getHttpRunContext, getHttpFrozenEvent, pushTelemetryEvent } from './telemetry-push.js'
+import { getHttpRunContext, getHttpFrozenEvent, getHttpPromptMock, pushTelemetryEvent, tryAutoInitHttpContext } from './telemetry-push.js'
+import { resolveAIMock, resolvePromptMock } from '../internals/mock-resolver.js'
 import type { WorkflowEvent } from '../capture/event.js'
 
 type UsageInfo = { inputTokens?: number; outputTokens?: number; totalTokens?: number }
@@ -44,13 +45,13 @@ export function wrapAI<Args extends unknown[], R>(
   callFn: (...args: Args) => Promise<R>,
 ): (...args: Args) => Promise<R> {
   return async (...args: Args): Promise<R> => {
+    await tryAutoInitHttpContext()
     const ctx = getCaptureContext()
     const httpCtx = getHttpRunContext()
 
     if (!ctx && !httpCtx) return callFn(...args)
 
     const start = rawDateNow()
-    const input = args.length === 1 ? args[0] : args
 
     if (ctx) {
       const { recorder, replay } = ctx
@@ -60,8 +61,27 @@ export function wrapAI<Args extends unknown[], R>(
         return replay.getRecordedResult(id) as R
       }
 
+      // Check AI mock (output mock — skip real call, return recorded result)
+      const aiMock = resolveAIMock(modelName)
+      if (aiMock.mocked) {
+        const input = args.length === 1 ? args[0] : args
+        const event: WorkflowEvent = {
+          id, type: 'ai', name: modelName, input,
+          output: aiMock.result, timestamp: start, durationMs: 0,
+        }
+        recorder.record(event)
+        if (httpCtx) pushTelemetryEvent(event)
+        return aiMock.result as R
+      }
+
+      // Check prompt mock (system prompt replacement — call real LLM with modified system prompt)
+      const rawInput = args.length === 1 ? args[0] : args
+      const modifiedInput = resolvePromptMock(rawInput)
+      const effectiveArgs: Args = modifiedInput !== undefined ? [modifiedInput] as unknown as Args : args
+      const input = modifiedInput !== undefined ? modifiedInput : rawInput
+
       try {
-        const output = await callFn(...args)
+        const output = await callFn(...effectiveArgs)
         const durationMs = rawDateNow() - start
         const usage = extractUsage(output)
         const event: WorkflowEvent = {
@@ -94,8 +114,14 @@ export function wrapAI<Args extends unknown[], R>(
       return frozen.output as R
     }
 
+    // Check prompt mock (system prompt replacement in HTTP mode)
+    const rawHttpInput = args.length === 1 ? args[0] : args
+    const httpModifiedInput = getHttpPromptMock(rawHttpInput)
+    const httpEffectiveArgs: Args = httpModifiedInput !== undefined ? [httpModifiedInput] as unknown as Args : args
+    const input = httpModifiedInput !== undefined ? httpModifiedInput : rawHttpInput
+
     try {
-      const output = await callFn(...args)
+      const output = await callFn(...httpEffectiveArgs)
       const durationMs = rawDateNow() - start
       const usage = extractUsage(output)
       const event: WorkflowEvent = {