elasticdash-test 0.1.11 → 0.1.12

package/README.md

@@ -38,6 +38,12 @@ npm install elasticdash-test
 
 **Requirements:** Node 20+. For Deno projects, see [Using elasticdash-test in Deno](docs/deno.md).
 
+**Git ignore:** ElasticDash writes temporary runtime artifacts under `.temp/`. Add this to your `.gitignore`:
+
+```gitignore
+.temp/
+```
+
 **Running CLI commands:** Use `npx` to run commands with your locally installed version (recommended to avoid version drift):
 
 ```bash
@@ -92,6 +98,12 @@ Total: 3
 Duration: 3.4s
 ```
 
+**Workflow export requirements:**
+
+- Export plain callable functions from `ed_workflows.ts/js`.
+- Use JSON-serializable inputs/outputs (object or array) so dashboard replay can pass args and read results.
+- Do not export framework-bound handlers directly (for example Next.js `NextRequest`/`NextResponse` route handlers).
+
 ---
 
 ## Documentation
@@ -213,6 +225,45 @@ In manual mode, always isolate tracing in a separate `try/catch` so trace loggin
 
 **→ See [Tool Recording & Replay](docs/tools.md) for checkpoint-based replay and freezing**
 
+### HTTP Streaming Capture and Replay
+
+ElasticDash also captures non-AI `fetch` responses that stream over HTTP (for example SSE and NDJSON endpoints) in the HTTP interceptor.
+
+Currently detected as streaming when response `content-type` includes:
+- `text/event-stream`
+- `application/x-ndjson`
+- `application/stream+json`
+- `application/jsonl`
+
+How it behaves today:
+- During live execution, ElasticDash tees the response stream and returns a real stream to your app code.
+- In parallel, ElasticDash buffers the recorder side of the stream as raw text for trace replay.
+- During replay, ElasticDash reconstructs a stream from that captured raw payload and restores status, status text, and response headers.
+
+Replay fidelity note:
+- Replay preserves stream payload content, but not original chunk boundaries or timing cadence.
+
+Minimal stream consumption example:
+
+```ts
+const res = await fetch('https://example.com/events')
+if (!res.body) throw new Error('Expected a streaming response body')
+
+const reader = res.body.getReader()
+const decoder = new TextDecoder()
+let buffer = ''
+
+for (;;) {
+  const { done, value } = await reader.read()
+  if (done) break
+  buffer += decoder.decode(value, { stream: true })
+}
+
+buffer += decoder.decode()
+```
+
+**→ See [Quick Start Guide](docs/quickstart.md#capture-streaming-flows) for end-to-end setup guidance**
+
 ---
 
 ## Configuration

package/dist/capture/event.d.ts

@@ -11,6 +11,10 @@ export interface WorkflowEvent {
     agentTaskId?: string;
     /** Optional: Zero-based index of the agent task that produced this event */
     agentTaskIndex?: number;
+    /** Set to true when the original response / output was a stream */
+    streamed?: boolean;
+    /** Raw buffered text of a streamed response (used for replay) */
+    streamRaw?: string;
 }
 export interface WorkflowTrace {
     traceId: string;

package/dist/capture/event.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"event.d.ts","sourceRoot":"","sources":["../../src/capture/event.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,aAAa,CAAA;AAE7E,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,iBAAiB,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB"}
+{"version":3,"file":"event.d.ts","sourceRoot":"","sources":["../../src/capture/event.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,iBAAiB,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,GAAG,aAAa,CAAA;AAE7E,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,iBAAiB,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,iEAAiE;IACjE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,aAAa,EAAE,CAAA;CACxB"}

package/dist/capture/recorder.d.ts

@@ -4,7 +4,12 @@ export declare class TraceRecorder {
     events: WorkflowEvent[];
     private _counter;
     private _sideEffectCounter;
+    private _pending;
     record(event: WorkflowEvent): void;
+    /** Register an in-flight async recording promise so flush() can await it. */
+    trackAsync(promise: Promise<void>): void;
+    /** Await all in-flight async recordings. No-op when none are pending. */
+    flush(): Promise<void>;
     nextId(): number;
     /** Separate counter for Date.now / Math.random — never shares IDs with main events. */
     nextSideEffectId(): number;

package/dist/capture/recorder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../src/capture/recorder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAEnD,qBAAa,aAAa;IACxB,MAAM,EAAE,aAAa,EAAE,CAAK;IAC5B,OAAO,CAAC,QAAQ,CAAI;IACpB,OAAO,CAAC,kBAAkB,CAAI;IAE9B,MAAM,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAIlC,MAAM,IAAI,MAAM;IAIhB,uFAAuF;IACvF,gBAAgB,IAAI,MAAM;IAI1B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa;CAMzC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,aAAa,CAAA;IACvB,MAAM,EAAE,gBAAgB,CAAA;CACzB;AASD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,cAAc,GAAG,SAAS,GAAG,IAAI,CAEvE;AAED,wBAAgB,iBAAiB,IAAI,cAAc,GAAG,SAAS,CAE9D"}
+{"version":3,"file":"recorder.d.ts","sourceRoot":"","sources":["../../src/capture/recorder.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC9D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAEnD,qBAAa,aAAa;IACxB,MAAM,EAAE,aAAa,EAAE,CAAK;IAC5B,OAAO,CAAC,QAAQ,CAAI;IACpB,OAAO,CAAC,kBAAkB,CAAI;IAC9B,OAAO,CAAC,QAAQ,CAAgC;IAEhD,MAAM,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAIlC,6EAA6E;IAC7E,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI;IAKxC,yEAAyE;IACnE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,MAAM,IAAI,MAAM;IAIhB,uFAAuF;IACvF,gBAAgB,IAAI,MAAM;IAI1B,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,aAAa;CAMzC;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,aAAa,CAAA;IACvB,MAAM,EAAE,gBAAgB,CAAA;CACzB;AASD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,cAAc,GAAG,SAAS,GAAG,IAAI,CAEvE;AAED,wBAAgB,iBAAiB,IAAI,cAAc,GAAG,SAAS,CAE9D"}

package/dist/capture/recorder.js

@@ -4,9 +4,19 @@ export class TraceRecorder {
     events = [];
     _counter = 0;
     _sideEffectCounter = 0;
+    _pending = new Set();
     record(event) {
         this.events.push(event);
     }
+    /** Register an in-flight async recording promise so flush() can await it. */
+    trackAsync(promise) {
+        this._pending.add(promise);
+        promise.finally(() => { this._pending.delete(promise); });
+    }
+    /** Await all in-flight async recordings. No-op when none are pending. */
+    async flush() {
+        await Promise.allSettled([...this._pending]);
+    }
     nextId() {
         return ++this._counter;
     }

package/dist/capture/recorder.js.map

@@ -1 +1 @@
-{"version":3,"file":"recorder.js","sourceRoot":"","sources":["../../src/capture/recorder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAIxC,MAAM,OAAO,aAAa;IACxB,MAAM,GAAoB,EAAE,CAAA;IACpB,QAAQ,GAAG,CAAC,CAAA;IACZ,kBAAkB,GAAG,CAAC,CAAA;IAE9B,MAAM,CAAC,KAAoB;QACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACzB,CAAC;IAED,MAAM;QACJ,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAA;IACxB,CAAC;IAED,uFAAuF;IACvF,gBAAgB;QACd,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAA;IAClC,CAAC;IAED,OAAO,CAAC,OAAgB;QACtB,OAAO;YACL,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE;YAChC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;SACzB,CAAA;IACH,CAAC;CACF;AAOD,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,eAAe,GAAG,6BAA6B,CAAA;AACrD,MAAM,UAAU,GACb,CAAC,CAAC,eAAe,CAAmD;IACrE,IAAI,iBAAiB,EAA8B,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC;IAAE,CAAC,CAAC,eAAe,CAAC,GAAG,UAAU,CAAA;AAExD,MAAM,UAAU,iBAAiB,CAAC,GAA+B;IAC/D,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,UAAU,CAAC,QAAQ,EAAE,CAAA;AAC9B,CAAC"}
+{"version":3,"file":"recorder.js","sourceRoot":"","sources":["../../src/capture/recorder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAIxC,MAAM,OAAO,aAAa;IACxB,MAAM,GAAoB,EAAE,CAAA;IACpB,QAAQ,GAAG,CAAC,CAAA;IACZ,kBAAkB,GAAG,CAAC,CAAA;IACtB,QAAQ,GAAuB,IAAI,GAAG,EAAE,CAAA;IAEhD,MAAM,CAAC,KAAoB;QACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACzB,CAAC;IAED,6EAA6E;IAC7E,UAAU,CAAC,OAAsB;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAC1B,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA,CAAC,CAAC,CAAC,CAAA;IAC1D,CAAC;IAED,yEAAyE;IACzE,KAAK,CAAC,KAAK;QACT,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM;QACJ,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAA;IACxB,CAAC;IAED,uFAAuF;IACvF,gBAAgB;QACd,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAA;IAClC,CAAC;IAED,OAAO,CAAC,OAAgB;QACtB,OAAO;YACL,OAAO,EAAE,OAAO,IAAI,UAAU,EAAE;YAChC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;SACzB,CAAA;IACH,CAAC;CACF;AAOD,MAAM,CAAC,GAAG,UAAqC,CAAA;AAC/C,MAAM,eAAe,GAAG,6BAA6B,CAAA;AACrD,MAAM,UAAU,GACb,CAAC,CAAC,eAAe,CAAmD;IACrE,IAAI,iBAAiB,EAA8B,CAAA;AACrD,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC;IAAE,CAAC,CAAC,eAAe,CAAC,GAAG,UAAU,CAAA;AAExD,MAAM,UAAU,iBAAiB,CAAC,GAA+B;IAC/D,UAAU,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;AAC3B,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,UAAU,CAAC,QAAQ,EAAE,CAAA;AAC9B,CAAC"}

package/dist/dashboard-server.d.ts

@@ -28,6 +28,18 @@ export interface DashboardServer {
     url: string;
     close(): Promise<void>;
 }
+/** Per-tool mock configuration sent from the dashboard UI */
+export interface ToolMockEntry {
+    /** 'live' = always call real tool, 'mock-all' = mock every call, 'mock-specific' = mock only listed call indices */
+    mode: 'live' | 'mock-all' | 'mock-specific';
+    /** When mode is 'mock-specific', which 1-based call indices to mock */
+    callIndices?: number[];
+    /** Mock data keyed by 1-based call index (or 0 for mock-all default) */
+    mockData?: Record<number, unknown>;
+}
+export interface ToolMockConfig {
+    [toolName: string]: ToolMockEntry;
+}
 /**
  * Start the dashboard server
  */

package/dist/dashboard-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dashboard-server.d.ts","sourceRoot":"","sources":["../src/dashboard-server.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAu9ED;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CA+S1B;AAiFD,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC"}
+{"version":3,"file":"dashboard-server.d.ts","sourceRoot":"","sources":["../src/dashboard-server.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,KAAK,EAAE,QAAQ,EAAE,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB;AAuCD,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,oHAAoH;IACpH,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,eAAe,CAAA;IAC3C,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,cAAc;IAC7B,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,CAAA;CAClC;AAqpFD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,eAAe,CAAC,CA+S1B;AAiFD,eAAO,MAAM,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,CAAC"}

package/dist/dashboard-server.js

@@ -260,6 +260,7 @@ function runWorkflowInSubprocess(workflowsModulePath, toolsModulePath, workflowN
             ...(options?.checkpoint !== undefined ? { checkpoint: options.checkpoint } : {}),
             ...(options?.history !== undefined ? { history: options.history } : {}),
             ...(options?.agentState !== undefined ? { agentState: options.agentState } : {}),
+            ...(options?.toolMockConfig !== undefined ? { toolMockConfig: options.toolMockConfig } : {}),
         });
         child.stdin.write(payload);
         child.stdin.end(); // Always close stdin to avoid subprocess hang
@@ -309,7 +310,9 @@ async function runGenerationObservation(observation) {
 }
 async function rerunObservation(cwd, observation, tools) {
     const type = observation.type?.toUpperCase();
-    if (type === 'TOOL') {
+    const name = observation.name ?? '(unknown)';
+    if (type === 'TOOL' || name.startsWith('tool-')) {
+        observation.name = name.startsWith('tool-') ? name.slice(5) : name; // Support both explicit type and name prefix for tool observations
         return runToolObservation(cwd, observation, tools);
     }
     if (type === 'GENERATION') {
@@ -362,7 +365,9 @@ function resolveWorkflowArgsFromObservations(body, workflowName) {
         return typeof item.name === 'string' && (item.name ?? '').trim() === workflowName;
     });
     if (!matched) {
-        return { error: `No matching observation found for workflow "${workflowName}".` };
+        // No workflow-level observation found (e.g. trace was loaded from an external format that
+        // only contains child observations). Fall back to running the workflow with no arguments.
+        return { args: [], input: null };
     }
     return { args: normalizeWorkflowArgs(matched.input), input: matched.input };
 }
@@ -597,6 +602,10 @@ async function validateWorkflowRuns(cwd, body) {
     }
     const workflowArgs = resolvedInput.args ?? [];
     const workflowInput = resolvedInput.input ?? null;
+    // Parse tool mock config if provided
+    const toolMockConfig = body.toolMockConfig && typeof body.toolMockConfig === 'object' && !Array.isArray(body.toolMockConfig)
+        ? body.toolMockConfig
+        : undefined;
     const workflowsModulePath = resolveWorkflowModule(cwd);
     if (!workflowsModulePath) {
         return {
@@ -612,7 +621,7 @@ async function validateWorkflowRuns(cwd, body) {
     console.log(`[elasticdash] Running workflow "${workflowName}" ${runCount} time(s) in ${mode} mode via subprocess`);
     async function runOne(runNumber) {
         console.log(`[elasticdash] === Run ${runNumber}: Starting workflow "${workflowName}" ===`);
-        const result = await runWorkflowInSubprocess(workflowsModulePath, toolsModulePath, workflowName, workflowArgs, workflowInput)
+        const result = await runWorkflowInSubprocess(workflowsModulePath, toolsModulePath, workflowName, workflowArgs, workflowInput, toolMockConfig ? { toolMockConfig } : undefined)
             .catch(err => {
             throw { ok: false, error: `Workflow subprocess failed: ${formatError(err)}` };
         });
@@ -1186,13 +1195,200 @@ function getDashboardHtml() {
             }
             };
             
+            // ---- Tool Mock Config State ----
+            window._toolMockConfig = {}; // { toolName: { mode: 'live'|'mock-all'|'mock-specific', callIndices: [], mockData: {} } }
+
+            function getToolsFromTrace() {
+                // Extract unique tool names and their call details from the uploaded trace observations
+                const toolCalls = {};
+                currentObservations.forEach(function(obs, i) {
+                    if (obs.type !== 'TOOL') return;
+                    const name = obs.name || '(unknown)';
+                    if (!toolCalls[name]) toolCalls[name] = [];
+                    toolCalls[name].push({ index: toolCalls[name].length + 1, obsIndex: i, input: obs.input, output: obs.output });
+                });
+                return toolCalls;
+            }
+
+            function getAllRegisteredTools() {
+                // From codeIndex.tools (fetched at page load from /api/code-index)
+                return (codeIndex.tools || []).map(function(t) { return t.name; });
+            }
+
+            function buildToolMockConfigFromUI() {
+                const config = {};
+                const rows = document.querySelectorAll('.tool-mock-row');
+                rows.forEach(function(row) {
+                    const toolName = row.dataset.toolName;
+                    const modeSelect = row.querySelector('.tool-mock-mode');
+                    const mode = modeSelect ? modeSelect.value : 'live';
+                    if (mode === 'live') return;
+                    const entry = { mode: mode };
+                    if (mode === 'mock-specific') {
+                        const checkboxes = row.querySelectorAll('.tool-call-checkbox:checked');
+                        entry.callIndices = Array.from(checkboxes).map(function(cb) { return parseInt(cb.value, 10); });
+                        if (entry.callIndices.length === 0) return; // No calls selected, treat as live
+                    }
+                    // Collect mock data
+                    entry.mockData = {};
+                    const dataInputs = row.querySelectorAll('.tool-mock-data-input');
+                    dataInputs.forEach(function(inp) {
+                        const callIdx = parseInt(inp.dataset.callIdx, 10);
+                        if (!inp.value.trim()) return;
+                        try { entry.mockData[callIdx] = JSON.parse(inp.value); }
+                        catch(e) { entry.mockData[callIdx] = inp.value; }
+                    });
+                    config[toolName] = entry;
+                });
+                return config;
+            }
+
+            function cleanValue(value) {
+                if (typeof value === "string") {
+                    value = value.replaceAll('\\\\"', '');
+                    // remove surrounding quotes if they exist
+                    if (value.startsWith('"') && value.endsWith('"')) {
+                        return value.slice(1, -1);
+                    }
+                    return value;
+                }
+
+                if (Array.isArray(value)) {
+                    return value.map(cleanValue);
+                }
+
+                if (typeof value === "object" && value !== null) {
+                    const result = {};
+                    for (const key in value) {
+                    result[key] = cleanValue(value[key]);
+                    }
+                    return result;
+                }
+
+                return value;
+            }
+
+            function convert(input) {
+                const parsed = JSON.parse(input);
+                return cleanValue(parsed);
+            }
+
+            function renderToolMockSection(showAll) {
+                const traceTools = getToolsFromTrace();
+                const allToolNames = getAllRegisteredTools();
+                const traceToolNames = Object.keys(traceTools);
+                const toolNames = showAll
+                    ? Array.from(new Set([...traceToolNames, ...allToolNames]))
+                    : traceToolNames;
+
+                if (toolNames.length === 0) {
+                    return '<div style="color:#999;font-size:13px;padding:6px 0;">No tools detected.</div>';
+                }
+
+                let html = '<div style="max-height:260px;overflow-y:auto;border:1px solid #e0e0e0;border-radius:6px;">';
+                html += '<table style="width:100%;border-collapse:collapse;font-size:13px;">';
+                html += '<thead><tr style="background:#f5f5f5;">';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">Tool</th>';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">Calls in Trace</th>';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">Mock Mode</th>';
+                html += '<th style="padding:6px 10px;text-align:left;border-bottom:1px solid #e0e0e0;">Details</th>';
+                html += '</tr></thead><tbody>';
+
+                toolNames.forEach(function(name) {
+                    const calls = traceTools[name] || [];
+                    const inTrace = traceToolNames.includes(name);
+                    const existing = window._toolMockConfig[name] || { mode: 'live' };
+                    const nameStyle = inTrace ? '' : 'color:#999;';
+
+                    html += '<tr class="tool-mock-row" data-tool-name="' + esc(name) + '" style="border-bottom:1px solid #f0f0f0;">';
+                    html += '<td style="padding:6px 10px;font-family:Monaco,monospace;' + nameStyle + '">' + esc(name) + (inTrace ? '' : ' <span style="font-size:10px;color:#aaa;">(not in trace)</span>') + '</td>';
+                    html += '<td style="padding:6px 10px;">' + calls.length + '</td>';
+                    html += '<td style="padding:6px 10px;">';
+                    html += '<select class="tool-mock-mode" style="font-size:12px;padding:2px 4px;" onchange="window.onToolMockModeChange(\\'' + esc(name) + '\\', this.value)">';
+                    html += '<option value="live"' + (existing.mode === 'live' ? ' selected' : '') + '>Live</option>';
+                    html += '<option value="mock-all"' + (existing.mode === 'mock-all' ? ' selected' : '') + '>Mock All Calls</option>';
+                    if (calls.length > 0) {
+                        html += '<option value="mock-specific"' + (existing.mode === 'mock-specific' ? ' selected' : '') + '>Mock Specific Calls</option>';
+                    }
+                    html += '</select>';
+                    html += '</td>';
+
+                    // Details column: per-call checkboxes + mock data inputs
+                    html += '<td style="padding:6px 10px;">';
+                    if (existing.mode === 'mock-all') {
+                        let defaultData = (existing.mockData && existing.mockData[0] !== undefined) ? JSON.stringify(existing.mockData[0]) : (calls.length > 0 ? JSON.stringify(calls[0].output) : '');
+                        defaultData = convert(defaultData);
+                        html += '<div style="font-size:11px;color:#555;margin-bottom:4px;">Mock data (JSON):</div>';
+                        html += '<textarea class="tool-mock-data-input" data-call-idx="0" style="width:100%;font-size:11px;font-family:Monaco,monospace;padding:4px;border:1px solid #ddd;border-radius:4px;min-height:32px;resize:vertical;" placeholder="Return value for all calls">' + esc(defaultData) + '</textarea>';
+                    } else if (existing.mode === 'mock-specific' && calls.length > 0) {
+                        html += '<div style="font-size:11px;color:#555;margin-bottom:4px;">Select calls to mock:</div>';
+                        calls.forEach(function(call) {
+                            const isChecked = existing.callIndices && existing.callIndices.includes(call.index);
+                            const inputPreview = typeof call.input === 'string' ? call.input.slice(0, 40) : JSON.stringify(call.input || '').slice(0, 40);
+                            let mockVal = (existing.mockData && existing.mockData[call.index] !== undefined) ? JSON.stringify(existing.mockData[call.index]) : JSON.stringify(call.output);
+                            mockVal = convert(mockVal);
+                            html += '<div style="margin-bottom:6px;padding:4px;background:#fafafa;border-radius:4px;border:1px solid #eee;">';
+                            html += '<label style="display:flex;align-items:center;gap:6px;font-size:12px;cursor:pointer;">';
+                            html += '<input type="checkbox" class="tool-call-checkbox" value="' + call.index + '"' + (isChecked ? ' checked' : '') + ' onchange="window.onToolCallCheckChange(\\'' + esc(name) + '\\',' + call.index + ',this.checked)">';
+                            html += '<span>Call #' + call.index + '</span>';
+                            html += '<span style="color:#888;font-size:11px;max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">' + esc(inputPreview) + '</span>';
+                            html += '</label>';
+                            if (isChecked) {
+                                html += '<textarea class="tool-mock-data-input" data-call-idx="' + call.index + '" style="width:100%;font-size:11px;font-family:Monaco,monospace;padding:4px;border:1px solid #ddd;border-radius:4px;min-height:28px;resize:vertical;margin-top:4px;" placeholder="Mock return value (JSON)">' + esc(mockVal) + '</textarea>';
+                            }
+                            html += '</div>';
+                        });
+                    } else {
+                        html += '<span style="color:#aaa;font-size:11px;">—</span>';
+                    }
+                    html += '</td>';
+                    html += '</tr>';
+                });
+
+                html += '</tbody></table></div>';
+                return html;
+            }
+
+            window.onToolMockModeChange = function(toolName, mode) {
+                if (!window._toolMockConfig[toolName]) window._toolMockConfig[toolName] = { mode: 'live' };
+                // Save current mock data before switching
+                window._toolMockConfig[toolName] = { ...window._toolMockConfig[toolName], mode: mode };
+                if (mode === 'mock-specific' && !window._toolMockConfig[toolName].callIndices) {
+                    window._toolMockConfig[toolName].callIndices = [];
+                }
+                // Re-render tool mock section
+                const showAll = document.getElementById('showAllToolsToggle');
+                const container = document.getElementById('toolMockContainer');
+                if (container) container.innerHTML = renderToolMockSection(showAll && showAll.checked);
+            };
+
+            window.onToolCallCheckChange = function(toolName, callIdx, checked) {
+                if (!window._toolMockConfig[toolName]) window._toolMockConfig[toolName] = { mode: 'mock-specific', callIndices: [] };
+                const indices = window._toolMockConfig[toolName].callIndices || [];
+                if (checked && !indices.includes(callIdx)) {
+                    indices.push(callIdx);
+                } else if (!checked) {
+                    const pos = indices.indexOf(callIdx);
+                    if (pos >= 0) indices.splice(pos, 1);
+                }
+                window._toolMockConfig[toolName].callIndices = indices;
+                const showAll = document.getElementById('showAllToolsToggle');
+                const container = document.getElementById('toolMockContainer');
+                if (container) container.innerHTML = renderToolMockSection(showAll && showAll.checked);
+            };
+
             window.openLiveValidationDialog = function() {
                 if (window.liveValidationDialog) return;
+                window._toolMockConfig = {}; // Reset mock config each time dialog opens
+
+                const hasTraceTools = currentObservations.some(function(o) { return o.type === 'TOOL'; });
+                const hasRegisteredTools = codeIndex.tools && codeIndex.tools.length > 0;
+
                 window.liveValidationDialog = document.createElement('div');
                 window.liveValidationDialog.id = 'liveValidationDialog';
                 window.liveValidationDialog.style = 'position:fixed;top:0;left:0;width:100vw;height:100vh;background:rgba(0,0,0,0.25);display:flex;align-items:center;justify-content:center;z-index:9999;';
                 window.liveValidationDialog.innerHTML = \`
-                    <div style="background:white;padding:32px 28px;border-radius:12px;box-shadow:0 2px 24px #0002;min-width:600px;max-width:90vw;">
+                    <div style="background:white;padding:32px 28px;border-radius:12px;box-shadow:0 2px 24px #0002;min-width:680px;max-width:90vw;max-height:90vh;overflow-y:auto;">
                     <h3 style="margin-top:0;margin-bottom:18px;font-size:20px;">Validate Updated Flow with Live Data</h3>
                     <label style="font-size:15px;display:block;margin-bottom:8px;">How many times do you want to run the flow with live data?</label>
                     <input id="liveValidationCount" type="number" min="1" value="1" style="width:100%;font-size:16px;padding:6px 10px;margin-bottom:18px;" />
@@ -1200,6 +1396,17 @@ function getDashboardHtml() {
                         <input id="liveValidationSequential" type="checkbox" />
                         Run in sequence instead of parallel
                     </label>
+                    \${(hasTraceTools || hasRegisteredTools) ? \`
+                    <div style="border-top:1px solid #eee;padding-top:16px;margin-bottom:16px;">
+                        <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:10px;">
+                            <div style="font-size:15px;font-weight:600;">Tool Mocking</div>
+                            <label style="display:flex;align-items:center;gap:6px;font-size:13px;cursor:pointer;">
+                                <input id="showAllToolsToggle" type="checkbox" onchange="document.getElementById('toolMockContainer').innerHTML = renderToolMockSection(this.checked);" />
+                                Show all registered tools
+                            </label>
+                        </div>
+                        <div id="toolMockContainer"></div>
+                    </div>\` : ''}
                     <div style="display:flex;gap:12px;justify-content:space-between;align-items:center;">
                         <span id="liveValidationProgress" style="font-size:14px;color:#555;"></span>
                         <div style="display:flex;gap:12px;">
@@ -1210,6 +1417,11 @@ function getDashboardHtml() {
                     </div>
                 \`;
                 document.body.appendChild(window.liveValidationDialog);
+                // Render the tool mock section after DOM insertion
+                const toolMockContainer = document.getElementById('toolMockContainer');
+                if (toolMockContainer) {
+                    toolMockContainer.innerHTML = renderToolMockSection(false);
+                }
                 document.getElementById('cancelLiveValidation').onclick = function() {
                     window.liveValidationDialog.remove();
                     window.liveValidationDialog = null;
@@ -1218,65 +1430,76 @@ function getDashboardHtml() {
                     const count = parseInt(document.getElementById('liveValidationCount').value, 10);
                     const sequential = document.getElementById('liveValidationSequential').checked;
                     if (count >= 1) {
+                        // Build the tool mock config from UI state
+                        const toolMockConfig = buildToolMockConfigFromUI();
                         const submitBtn = document.getElementById('submitLiveValidation');
                         submitBtn.disabled = true;
                         submitBtn.textContent = 'Validating...';
                         const progressEl = document.getElementById('liveValidationProgress');
-                        if (progressEl) progressEl.textContent = \`0 of \${count} workflow runs\`;
-                        let completed = 0;
-                        // Wrap the fetch in a progress simulation for user feedback
-                        const payload = { workflowName: selectedWorkflow?.name, runCount: count, sequential, observations: currentObservations };
-                        // Start the fetch but also update progress as we go
-                        const progressInterval = setInterval(() => {
-                            if (progressEl && completed < count) {
-                                progressEl.textContent = \`\${completed} of \${count} workflow runs\`;
-                            }
-                        }, 200);
-                        try {
-                            const responsePromise = fetch('/api/validate-workflow', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
-                            // Simulate progress incrementing as if runs are happening (for demo, increment every 500ms)
-                            while (completed < count) {
-                                await new Promise(r => setTimeout(r, 500));
-                                completed++;
-                                if (progressEl) progressEl.textContent = \`\${completed} of \${count} workflow runs\`;
-                            }
-                            const response = await responsePromise;
-                            const data = await response.json();
-                            clearInterval(progressInterval);
+
+                        function finishValidation(collectedTraces, errorMsg, usedSequential) {
                             if (progressEl) progressEl.textContent = '';
                             window.liveValidationDialog.remove();
                             window.liveValidationDialog = null;
                             window.liveValidationCount = count;
-                            window.liveValidationSequential = sequential;
+                            window.liveValidationSequential = usedSequential;
                             window.step5SelectedTrace = 0;
                             window.step5SelectedObservation = 0;
                             currentStep = 5;
                             updateModalTitle();
                             updateFooterButtons();
-                            if (response.ok && data.ok) {
-                                step5RunTraces = Array.isArray(data.traces) ? data.traces : [];
-                                persistTraces();
-                                step5RunMeta = { loading: false, error: '', runCount: typeof data.runCount === 'number' ? data.runCount : count, sequential: data.mode === 'sequential' };
-                            } else {
+                            if (errorMsg && collectedTraces.length === 0) {
                                 step5RunTraces = [];
                                 localStorage.removeItem('ed_step5RunTraces');
-                                step5RunMeta = { loading: false, error: data.error || 'Workflow validation failed.', runCount: count, sequential };
+                                step5RunMeta = { loading: false, error: errorMsg, runCount: count, sequential: usedSequential };
+                            } else {
+                                step5RunTraces = collectedTraces;
+                                persistTraces();
+                                step5RunMeta = { loading: false, error: '', runCount: collectedTraces.length, sequential: usedSequential };
+                            }
+                            if (window.step5SelectedTrace > step5RunTraces.length) window.step5SelectedTrace = 0;
+                            window.step5SelectedObservation = 0;
+                            renderObservationTable();
+                        }
+
+                        if (sequential) {
+                            // Sequential mode: fire one request per run so progress reflects real completion
+                            if (progressEl) progressEl.textContent = \`0 of \${count} workflow runs completed\`;
+                            const collectedTraces = [];
+                            let fatalError = null;
+                            for (let i = 0; i < count; i++) {
+                                const singlePayload = { workflowName: selectedWorkflow?.name, runCount: 1, sequential: false, observations: currentObservations, toolMockConfig };
+                                try {
+                                    const response = await fetch('/api/validate-workflow', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(singlePayload) });
+                                    const data = await response.json();
+                                    if (response.ok && data.ok && Array.isArray(data.traces) && data.traces.length > 0) {
+                                        collectedTraces.push({ ...data.traces[0], runNumber: i + 1 });
+                                    } else {
+                                        // Push an error trace so the run is still visible in Step 5
+                                        collectedTraces.push({ runNumber: i + 1, ok: false, error: data.error || 'Workflow validation failed.', observations: [], workflowTrace: null });
+                                    }
+                                } catch (err) {
+                                    collectedTraces.push({ runNumber: i + 1, ok: false, error: err && err.message ? err.message : String(err), observations: [], workflowTrace: null });
+                                }
+                                if (progressEl) progressEl.textContent = \`\${i + 1} of \${count} workflow runs completed\`;
+                            }
+                            finishValidation(collectedTraces, fatalError, true);
+                        } else {
+                            // Parallel mode: single bulk request
+                            if (progressEl) progressEl.textContent = \`Running \${count} workflow run\${count !== 1 ? 's' : ''} in parallel…\`;
+                            const payload = { workflowName: selectedWorkflow?.name, runCount: count, sequential: false, observations: currentObservations, toolMockConfig };
+                            try {
+                                const response = await fetch('/api/validate-workflow', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
+                                const data = await response.json();
+                                if (response.ok && data.ok) {
+                                    finishValidation(Array.isArray(data.traces) ? data.traces : [], null, false);
+                                } else {
+                                    finishValidation([], data.error || 'Workflow validation failed.', false);
+                                }
+                            } catch (err) {
+                                finishValidation([], err && err.message ? err.message : String(err), false);
                             }
-                        } catch (err) {
-                            clearInterval(progressInterval);
-                            if (progressEl) progressEl.textContent = '';
-                            window.liveValidationDialog.remove();
-                            window.liveValidationDialog = null;
-                            currentStep = 5;
-                            updateModalTitle();
-                            updateFooterButtons();
-                            step5RunTraces = [];
-                            localStorage.removeItem('ed_step5RunTraces');
-                            step5RunMeta = { loading: false, error: err && err.message ? err.message : String(err), runCount: count, sequential };
                         }
-                        if (window.step5SelectedTrace > step5RunTraces.length) window.step5SelectedTrace = 0;
-                        window.step5SelectedObservation = 0;
-                        renderObservationTable();
                     } else {
                         document.getElementById('liveValidationCount').style.borderColor = 'red';
                     }