@botpress/evals 1.19.0-beta.9 → 2.0.0-alpha.0

package/dist/runner.js

@@ -1,6 +1,3 @@
-// src/runner.ts
-import { Client as BpClient2 } from "@botpress/client";
-
 // src/definition.ts
 var Eval = class {
   name;
@@ -26,83 +23,33 @@ var Eval = class {
 // src/types.ts
 var defaultLogger = console;
 
-// src/client.ts
-import { Client as BpClient } from "@botpress/client";
-import { createRequire } from "module";
-
-// ../analytics/src/errors.ts
-var AdkError = class extends Error {
-  /**
-   * Marker for cross-module-instance checks — `instanceof` breaks when a
-   * package is duplicated in the dependency graph.
-   * @internal
-   */
-  static __IS_ADK_BASE_ERROR = true;
-  /**
-   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
-   * pin `Code` to a literal union (`class EvalRunnerError extends
-   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
-   * in switches.
-   */
-  code;
-  /**
-   * True for user/environment conditions the developer can fix themselves
-   * (validation, auth, connectivity). False for internal bugs. Defaults to
-   * false so unclassified errors surface as bugs until someone triages them.
-   */
-  expected;
-  /** Structured context. Sanitized before any telemetry leaves the machine. */
-  details;
-  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
-  suggestion;
-  constructor(opts) {
-    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
-    this.name = this.constructor.name;
-    this.code = opts.code;
-    this.expected = opts.expected ?? false;
-    if (opts.details !== void 0) {
-      this.details = opts.details;
-    }
-    if (opts.suggestion !== void 0) {
-      this.suggestion = opts.suggestion;
-    }
-  }
-};
-function isAdkError(error) {
-  if (error instanceof AdkError) return true;
-  if (!(error instanceof Error)) return false;
-  return error.constructor?.__IS_ADK_BASE_ERROR === true;
-}
-
 // src/errors.ts
+import { AdkError } from "@botpress/analytics";
 var EvalRunnerError = class extends AdkError {
 };
 
 // src/client.ts
-var defaultChatClient = null;
-function getDefaultChatClient() {
-  if (defaultChatClient) return defaultChatClient;
-  const require2 = createRequire(import.meta.url);
-  const mod = require2("@botpress/chat");
-  const ChatClient = mod.default?.Client ?? mod.Client;
-  if (!ChatClient) {
-    throw new EvalRunnerError({
-      code: "CHAT_CLIENT_MISSING",
-      message: "@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly."
-    });
-  }
-  return ChatClient;
-}
+var TRACE_STREAM_PREFLIGHT_TIMEOUT_MS = 1e4;
 var ChatSession = class {
-  constructor(webhookId, chatClient) {
-    this.webhookId = webhookId;
-    this.chatClient = chatClient;
+  constructor(bpClient, botId, _chatWebhookId, _chatBaseUrl, _chatClient) {
+    this.bpClient = bpClient;
+    this.botId = botId;
+    this._chatWebhookId = _chatWebhookId;
+    this._chatBaseUrl = _chatBaseUrl;
+    this._chatClient = _chatClient;
   }
   client = null;
   conversationId = null;
   async connect() {
-    const ChatClient = this.chatClient ?? getDefaultChatClient();
-    this.client = await ChatClient.connect({ webhookId: this.webhookId });
+    const webhookId = this._chatWebhookId ?? await discoverWebhookId(this.bpClient, this.botId);
+    const ChatClientCtor = this._chatClient ?? (await import(
+      /* webpackIgnore: true */
+      "@botpress/chat"
+    )).Client;
+    this.client = await ChatClientCtor.connect({
+      webhookId,
+      ...this._chatBaseUrl ? { baseApiUrl: this._chatBaseUrl } : {}
+    });
   }
   /** Invariant guard — calling any session method before connect() is a runner bug. */
   assertConnected() {
@@ -126,6 +73,16 @@ var ChatSession = class {
     }
     return this.conversationId;
   }
+  /**
+   * Open a fresh conversation under the same user (the user is fixed at
+   * connect()) and make it active. Returns the new conversation id.
+   */
+  async newConversation() {
+    const client = this.assertConnected();
+    const conv = await client.createConversation({});
+    this.conversationId = conv.conversation.id;
+    return conv.conversation.id;
+  }
   /**
    * Send a user message. Fire-and-forget — observation is via SSECollector.
    */
@@ -134,17 +91,13 @@ var ChatSession = class {
     const conversationId = await this.ensureConversation();
     await client.createMessage({ conversationId, payload: { type: "text", text: message } });
   }
-  /**
-   * Send an event. Fire-and-forget — observation is via SSECollector.
-   */
-  async sendEvent(type, payload) {
+  async sendEvent(payload) {
     const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await client.createEvent({ type, payload, conversationId });
+    await client.createEvent({ payload, conversationId });
   }
 };
-async function discoverWebhookId(botId, token, apiUrl) {
-  const client = new BpClient({ token, botId, apiUrl });
+async function discoverWebhookId(client, botId) {
   const { bot } = await client.getBot({ id: botId });
   const integrations = bot.integrations || {};
   const chat = Object.values(integrations).find((int) => int.name === "chat");
@@ -156,7 +109,7 @@ async function discoverWebhookId(botId, token, apiUrl) {
         "The `chat` integration is not installed on this bot \u2014 evals require it.",
         "",
         "To fix:",
-        "  1. Install it:    adk add chat",
+        "  1. Install it:    adk integrations add chat",
         "  2. Redeploy:      adk dev   (or: adk deploy)"
       ].join("\n"),
       expected: true
@@ -164,6 +117,77 @@ async function discoverWebhookId(botId, token, apiUrl) {
   }
   return webhookId;
 }
+async function assertTraceStreamReadable(devServerUrl, headers = {}, timeoutMs = TRACE_STREAM_PREFLIGHT_TIMEOUT_MS) {
+  const url = `${devServerUrl.replace(/\/$/, "")}/api/traces/stream?count=1`;
+  const abortController = new AbortController();
+  let timedOut = false;
+  let reader = null;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    abortController.abort();
+  }, timeoutMs);
+  try {
+    const res = await fetch(url, {
+      headers: { ...headers, Accept: "text/event-stream" },
+      signal: abortController.signal
+    });
+    if (!res.ok) {
+      throw traceStreamPreflightError(`Trace stream pre-flight failed: ${res.status} ${res.statusText}`);
+    }
+    if (!res.body) {
+      throw traceStreamPreflightError("Trace stream pre-flight failed: response had no body.");
+    }
+    reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        throw traceStreamPreflightError("Trace stream pre-flight failed: stream closed before snapshot.");
+      }
+      buffer += decoder.decode(value, { stream: true });
+      const parts = buffer.split("\n\n");
+      buffer = parts.pop() ?? "";
+      for (const part of parts) {
+        if (parseSSEEventName(part) === "snapshot") {
+          return;
+        }
+      }
+    }
+  } catch (err) {
+    if (err instanceof EvalRunnerError) {
+      throw err;
+    }
+    if (timedOut) {
+      throw traceStreamPreflightError(
+        `Trace stream pre-flight timed out after ${timeoutMs}ms before receiving a snapshot.`
+      );
+    }
+    throw traceStreamPreflightError(
+      `Trace stream pre-flight failed: ${err instanceof Error ? err.message : String(err)}`
+    );
+  } finally {
+    clearTimeout(timer);
+    reader?.cancel().catch(() => {
+    });
+  }
+}
+function parseSSEEventName(block) {
+  for (const line of block.split("\n")) {
+    if (line.startsWith("event:")) {
+      return line.slice("event:".length).trim();
+    }
+  }
+  return null;
+}
+function traceStreamPreflightError(message) {
+  return new EvalRunnerError({
+    code: "SSE_CONNECT_FAILED",
+    message,
+    expected: true,
+    suggestion: "Make sure the dev server is running (`adk dev`) and trace streaming is available."
+  });
+}
 async function assertChatChannelBound(devServerUrl, headers = {}, logger = defaultLogger) {
   let conversations = [];
   try {
@@ -201,13 +225,17 @@ async function assertChatChannelBound(devServerUrl, headers = {}, logger = defau
   });
 }
 
-// src/sse-collector.ts
+// src/spans/sse-collector.ts
 function spanKey(span) {
   return `${span.id.trace}:${span.id.span}`;
 }
 function spanData(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
 }
+var TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
+function isTurnHandlerSpan(span) {
+  return TURN_HANDLER_SPAN_NAMES.has(span.name);
+}
 var SSECollector = class _SSECollector {
   devServerUrl;
   headers;
@@ -225,6 +253,10 @@ var SSECollector = class _SSECollector {
   connected = false;
   /** Error from the SSE stream if it died unexpectedly. */
   streamError = null;
+  /** Passive counters used only to explain timeout failures. */
+  updateEventCount = 0;
+  keepaliveEventCount = 0;
+  lastEvent = null;
   /** Set while a settle phase is in flight; called by disconnect() to short-circuit. */
   _cancelSettle = null;
   constructor(devServerUrl, headers = {}) {
@@ -240,9 +272,10 @@ var SSECollector = class _SSECollector {
     const attributeValue = "conversationId" in filter ? filter.conversationId : filter.workflowId;
     const url = `${this.devServerUrl}/api/traces/stream?attributeName=${encodeURIComponent(attributeName)}&attributeValue=${encodeURIComponent(attributeValue)}`;
     this.abortController = new AbortController();
+    const myController = this.abortController;
     const response = await fetch(url, {
       headers: { ...this.headers, Accept: "text/event-stream" },
-      signal: this.abortController.signal
+      signal: myController.signal
     });
     if (!response.ok) {
       throw new EvalRunnerError({
@@ -257,28 +290,48 @@ var SSECollector = class _SSECollector {
     }
     this.connected = true;
     this._consumeStream(response.body).catch((err) => {
-      if (this.connected) {
+      if (this.connected && this.abortController === myController) {
         this.streamError = err instanceof Error ? err : new Error(String(err));
       }
     });
     await this._waitForSnapshot();
   }
+  /**
+   * Re-point at a new conversation, keeping `spanMap` and `seenHandlerKeys` (so
+   * the earlier conversation stays visible and its completed handler can't
+   * satisfy the next turn). Resets the snapshot latch so connect() awaits the
+   * new conversation's snapshot.
+   */
+  async repoint(filter) {
+    this.connected = false;
+    this.abortController?.abort();
+    this.abortController = null;
+    this._cancelSettle?.();
+    this._cancelSettle = null;
+    this.waiters = [];
+    this.snapshotReceived = false;
+    this.snapshotResolvers = [];
+    this.streamError = null;
+    await this.connect(filter);
+  }
   /**
    * Mark the beginning of a new turn.
-   * Records which handler.conversation span IDs have been seen so far,
-   * so waitForTurnComplete() only resolves on NEW handler completions.
+   * Records which turn-handler span IDs (handler.conversation / handler.event)
+   * have been seen so far, so waitForTurnComplete() only resolves on NEW
+   * handler completions.
    */
   startTurn() {
     this.turnStartKeys = new Set(this.spanMap.keys());
     for (const [key, span] of this.spanMap) {
-      if (span.name === "handler.conversation") {
+      if (isTurnHandlerSpan(span)) {
         this.seenHandlerKeys.add(key);
       }
     }
   }
   static SETTLE_MS = 100;
   /**
-   * Wait for a handler.conversation span to reach terminal status (ok or error).
+   * Wait for a turn-handler span (handler.conversation or handler.event) to
+   * reach terminal status (ok or error).
    * Only considers spans NOT seen before the last startTurn() call.
    *
    * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)
@@ -287,8 +340,10 @@ var SSECollector = class _SSECollector {
    * spans may arrive slightly after the handler close event.
    */
   async waitForTurnComplete(opts) {
-    if (this._hasNewCompletedHandler()) {
-      return this._settleAfterHandler();
+    const accept = opts.acceptSpanNames;
+    const settle = { quietMs: opts.settleQuietMs, maxMs: opts.settleMaxMs };
+    if (this._hasNewCompletedHandler(accept)) {
+      return this._settleAfterHandler(settle);
     }
     if (opts.abortSignal?.aborted) {
       throw new Error("waitForTurnComplete aborted");
@@ -297,8 +352,13 @@ var SSECollector = class _SSECollector {
       const timer = setTimeout(() => {
         this._removeWaiter(waiter);
         opts.abortSignal?.removeEventListener("abort", onAbort);
-        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation span completed";
-        reject(new Error(`waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}`));
+        const expected = accept ? [...accept].join(" or ") : "handler.conversation or handler.event";
+        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : `no ${expected} span completed`;
+        reject(
+          new Error(
+            `waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}; diagnostics: ${this._formatTurnDiagnostics()}`
+          )
+        );
       }, opts.timeout);
       const onAbort = () => {
         clearTimeout(timer);
@@ -307,11 +367,11 @@ var SSECollector = class _SSECollector {
       };
       opts.abortSignal?.addEventListener("abort", onAbort, { once: true });
       const waiter = (_span) => {
-        if (this._hasNewCompletedHandler()) {
+        if (this._hasNewCompletedHandler(accept)) {
           clearTimeout(timer);
           this._removeWaiter(waiter);
           opts.abortSignal?.removeEventListener("abort", onAbort);
-          this._settleAfterHandler().then(resolve2, reject);
+          this._settleAfterHandler(settle).then(resolve2, reject);
         }
       };
       this.waiters.push(waiter);
@@ -325,7 +385,9 @@ var SSECollector = class _SSECollector {
    * Exposes a cancellation hook via _cancelSettle so disconnect() can short-
    * circuit an in-flight settle phase and avoid leaking timers for up to 2s.
    */
-  _settleAfterHandler() {
+  _settleAfterHandler(settle) {
+    const quietMs = settle?.quietMs ?? _SSECollector.SETTLE_MS;
+    const maxMs = settle?.maxMs ?? _SSECollector.MAX_SETTLE_MS;
     return new Promise((resolve2) => {
       let resolved = false;
       const done = () => {
@@ -337,12 +399,12 @@ var SSECollector = class _SSECollector {
         this._cancelSettle = null;
         resolve2();
       };
-      const hardCap = setTimeout(done, _SSECollector.MAX_SETTLE_MS);
-      let settleTimer = setTimeout(done, _SSECollector.SETTLE_MS);
+      const hardCap = setTimeout(done, maxMs);
+      let settleTimer = setTimeout(done, quietMs);
       const settleWaiter = () => {
         if (resolved) return;
         clearTimeout(settleTimer);
-        settleTimer = setTimeout(done, _SSECollector.SETTLE_MS);
+        settleTimer = setTimeout(done, quietMs);
       };
       this.waiters.push(settleWaiter);
       this._cancelSettle = done;
@@ -507,6 +569,7 @@ ${value}` : value;
    * Dispatch a parsed SSE event to the appropriate handler.
    */
   _handleEvent(eventName, payload) {
+    this._recordEvent(eventName);
     if (eventName === "snapshot") {
       const data2 = payload;
       const spans = Array.isArray(data2.spans) ? data2.spans : [];
@@ -526,6 +589,44 @@ ${value}` : value;
       }
     }
   }
+  _recordEvent(eventName) {
+    this.lastEvent = { name: eventName, at: Date.now() };
+    if (eventName === "update") {
+      this.updateEventCount++;
+    } else if (eventName === "keepalive") {
+      this.keepaliveEventCount++;
+    }
+  }
+  _formatTurnDiagnostics() {
+    const turnSpans = this.getTurnSpans();
+    const spanNames = /* @__PURE__ */ new Set();
+    const handlerStatuses = /* @__PURE__ */ new Set();
+    for (const span of turnSpans) {
+      spanNames.add(span.name);
+      if (isTurnHandlerSpan(span)) {
+        handlerStatuses.add(span.status);
+      }
+    }
+    const lastEvent = this.lastEvent ? `${this.lastEvent.name} ${Date.now() - this.lastEvent.at}ms ago` : "none";
+    return [
+      `snapshotReceived=${this.snapshotReceived}`,
+      `totalUpdates=${this.updateEventCount}`,
+      `totalKeepalives=${this.keepaliveEventCount}`,
+      `lastEvent=${lastEvent}`,
+      `totalSpans=${this.spanMap.size}`,
+      `turnSpans=${turnSpans.length}`,
+      `turnSpanNames=${this._formatSet(spanNames)}`,
+      `turnHandlerStatuses=${this._formatSet(handlerStatuses)}`
+    ].join(", ");
+  }
+  _formatSet(values, limit = 12) {
+    if (values.size === 0) {
+      return "none";
+    }
+    const items = Array.from(values);
+    const suffix = items.length > limit ? `|+${items.length - limit} more` : "";
+    return `${items.slice(0, limit).join("|")}${suffix}`;
+  }
   /**
    * Insert or replace a span in the map (dedup by trace+span ID).
    */
@@ -551,7 +652,7 @@ ${value}` : value;
     }
   }
   /**
-   * Check if there is a handler.conversation span that:
+   * Check if there is a turn-handler span (handler.conversation / handler.event) that:
    * 1. Was NOT seen before startTurn()
    * 2. Has terminal status (ok or error)
    *
@@ -560,9 +661,9 @@ ${value}` : value;
    * waitForTurnComplete check (which would otherwise resolve instantly
    * against stale data instead of waiting for the new turn's handler).
    */
-  _hasNewCompletedHandler() {
+  _hasNewCompletedHandler(acceptSpanNames) {
     for (const [key, span] of this.spanMap) {
-      if (span.name === "handler.conversation" && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
+      if (isTurnHandlerSpan(span) && (!acceptSpanNames || acceptSpanNames.has(span.name)) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
         this.seenHandlerKeys.add(key);
         return true;
       }
@@ -589,6 +690,38 @@ ${value}` : value;
   }
 };
 
+// src/spans/local-span-source.ts
+var LocalSpanSource = class {
+  collector;
+  constructor(devServerUrl, headers = {}) {
+    this.collector = new SSECollector(devServerUrl, headers);
+  }
+  async connect(filter) {
+    await this.collector.connect(filter);
+  }
+  async repoint(filter) {
+    await this.collector.repoint(filter);
+  }
+  startTurn() {
+    this.collector.startTurn();
+  }
+  async waitForTurnComplete(opts) {
+    await this.collector.waitForTurnComplete(opts);
+  }
+  async waitForWorkflow(name, opts) {
+    await this.collector.waitForWorkflow(name, opts);
+  }
+  getTurnSpans() {
+    return this.collector.getTurnSpans();
+  }
+  getAllSpans() {
+    return this.collector.getAllSpans();
+  }
+  disconnect() {
+    this.collector.disconnect();
+  }
+};
+
 // src/transformer.ts
 function data(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
@@ -676,7 +809,7 @@ function extractWorkflowSpans(spans) {
 }
 function extractHandler(spans) {
   const handler = spans.find(
-    (s) => (s.name === "handler.conversation" || s.name === "handler.workflow") && s.status !== "running"
+    (s) => (s.name === "handler.conversation" || s.name === "handler.event" || s.name === "handler.workflow") && s.status !== "running"
   );
   if (!handler) {
     return { handlerDuration: 0, handlerStatus: "ok" };
@@ -688,55 +821,26 @@ function extractHandler(spans) {
 }
 
 // src/graders/llm.ts
-import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
-var DEFAULT_PASS_THRESHOLD = 3;
-function buildJudgeSystemPrompt(passThreshold) {
-  return `You are an evaluation judge for a chatbot. You will be given:
-- The user's message
-- The bot's response
-- Grading criteria
-
-Your job is to determine if the bot's response meets the criteria.
-
-Respond with a JSON object:
-{
-  "pass": true/false,
-  "score": 1-5,
-  "reason": "brief explanation"
-}
-
-Scoring guide:
-- 5: Fully meets criteria, excellent response
-- 4: Mostly meets criteria, minor issues
-- 3: Partially meets criteria, notable gaps
-- 2: Barely meets criteria, significant issues
-- 1: Does not meet criteria
-
-A score of ${passThreshold} or above is a pass.`;
-}
+import { Zai } from "@botpress/zai";
 var MODEL_ALIASES = ["fast", "best"];
-var _cognitive = null;
+var _zai = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _failClosed = true;
 var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
-async function initLLMJudge(credentials, options) {
+async function initLLMJudge(client, options) {
   const log = options?.logger ?? defaultLogger;
   _log = log;
-  const client = new Client({
-    token: credentials.token,
-    apiUrl: credentials.apiUrl,
-    botId: credentials.botId,
-    ...credentials.workspaceId !== void 0 ? { workspaceId: credentials.workspaceId } : {}
-  });
-  _cognitive = new Cognitive({ client, __experimental_beta: true });
+  _failClosed = options?.failClosed ?? true;
+  const cognitive = new Cognitive({ client, __experimental_beta: true });
   try {
-    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys());
+    _availableModels = Array.from((await cognitive.fetchRemoteModels()).keys());
   } catch (err) {
     log.warn(`Failed to fetch available models: ${err.message}. LLM judge will be unavailable.`);
     _availableModels = [];
@@ -747,66 +851,42 @@ async function initLLMJudge(credentials, options) {
   } else if (!_isModelValid(_judgeModel)) {
     log.warn(`Configured LLM judge model "${_judgeModel}" is invalid. Run "adk models" to list available models.`);
   }
+  _zai = new Zai({ client: cognitive, modelId: _judgeModel });
 }
 async function gradeLLMJudge(botResponse, criteria, context) {
-  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
+  const assertion = `llm_judge: "${criteria}"`;
   try {
-    if (!_cognitive) {
+    if (!_zai) {
       _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
-        assertion: `llm_judge: "${criteria}"`,
-        pass: false,
+        assertion,
+        pass: !_failClosed,
+        skipped: true,
         expected: criteria,
         actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
       return {
-        assertion: `llm_judge: "${criteria}"`,
+        assertion,
         pass: false,
         expected: criteria,
         actual: "FAILED \u2014 invalid LLM judge model"
       };
     }
-    const { output } = await _cognitive.generateContent({
-      model: _judgeModel,
-      temperature: 0,
-      responseFormat: "json_object",
-      systemPrompt: buildJudgeSystemPrompt(threshold),
-      messages: [
-        {
-          role: "user",
-          content: `User message: ${context.userMessage}
-
-Bot response: ${botResponse}
-
-Criteria: ${criteria}`
-        }
-      ]
-    });
-    const rawContent = output?.choices?.[0]?.content;
-    const content = typeof rawContent === "string" ? rawContent : null;
-    if (!content) {
-      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
-      return {
-        assertion: `llm_judge: "${criteria}"`,
-        pass: false,
-        expected: criteria,
-        actual: "FAILED \u2014 LLM judge returned empty response"
-      };
-    }
-    const verdict = JSON.parse(content);
+    const { output } = await _zai.check({ userMessage: context.userMessage, botResponse }, criteria).result();
     return {
-      assertion: `llm_judge: "${criteria}"`,
-      pass: verdict.score >= threshold,
+      assertion,
+      pass: output.value,
       expected: criteria,
-      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
+      actual: `${output.value ? "Pass" : "Fail"} \u2014 ${output.explanation}`
     };
   } catch (err) {
     _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
-      assertion: `llm_judge: "${criteria}"`,
-      pass: false,
+      assertion,
+      pass: !_failClosed,
+      skipped: true,
       expected: criteria,
       actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
@@ -1229,6 +1309,7 @@ function filterEvals(evals, filter) {
 }
 
 // src/runner.ts
+import { isAdkError } from "@botpress/analytics";
 import { randomUUID } from "crypto";
 var DEFAULT_IDLE_TIMEOUT = 3e4;
 var DEFAULT_DEV_SERVER_URL = "http://localhost:3001";
@@ -1315,27 +1396,22 @@ async function runEval(evalDef, connection, options = {}) {
   const start = Date.now();
   const turns = [];
   let outcomeAssertions = [];
-  const collector = new SSECollector(devServerUrl, devServerHeaders);
+  const collector = options.spanSource ?? new LocalSpanSource(devServerUrl, devServerHeaders);
   try {
-    const session = new ChatSession(connection.webhookId, options.chatClient);
+    const session = new ChatSession(
+      connection.client,
+      connection.botId,
+      options.chatWebhookId,
+      options.chatBaseUrl,
+      options.chatClient
+    );
     await session.connect();
-    let bpClient = null;
-    const getBpClient = () => {
-      if (!bpClient) {
-        bpClient = new BpClient2({
-          token: connection.token,
-          botId: connection.botId,
-          apiUrl: connection.apiUrl
-        });
-      }
-      return bpClient;
-    };
     let conversationId = "";
     if (evalDef.setup?.state?.conversation || evalDef.setup?.workflow) {
       conversationId = await session.ensureConversation();
     }
     await seedEvalState(
-      getBpClient(),
+      connection.client,
       {
         botId: connection.botId,
         userId: session.userId,
@@ -1344,7 +1420,7 @@ async function runEval(evalDef, connection, options = {}) {
       evalDef.setup
     );
     await triggerEvalWorkflow(
-      getBpClient(),
+      connection.client,
       {
         userId: session.userId,
         ...conversationId ? { conversationId } : {}
@@ -1380,24 +1456,34 @@ async function runEval(evalDef, connection, options = {}) {
       if (!turn.user && !turn.event) {
         throw turnConfigError(`must have either 'user' or 'event'.`);
       }
-      const turnLabelForEvent = turn.event ? `[event: ${turn.event.type}]` : turn.user;
-      options.onProgress?.({
+      const turnLabel = turn.event ? "[event]" : turn.user;
+      await options.onProgress?.({
         type: "turn_start",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnNumber: i + 1,
+        turnIndex: i,
         totalTurns: evalDef.conversation.length,
-        userMessage: turnLabelForEvent
+        userMessage: turnLabel
       });
+      if (turn.newConversation && i > 0) {
+        const newConversationId = await session.newConversation();
+        await collector.repoint({ conversationId: newConversationId });
+      }
       collector.startTurn();
       if (turn.event) {
-        await session.sendEvent(turn.event.type, turn.event.payload ?? {});
+        await session.sendEvent(turn.event.payload);
       } else {
         await session.sendMessage(turn.user);
       }
       try {
         await collector.waitForTurnComplete({
           timeout: idleTimeout,
+          acceptSpanNames: turn.event ? /* @__PURE__ */ new Set(["handler.event"]) : /* @__PURE__ */ new Set(["handler.conversation"]),
+          // Long quiet window: a tool-started workflow and its callback only run
+          // AFTER the handler closes; sending the next message into that window
+          // can get it absorbed by their transcript saves and silently skipped.
+          settleQuietMs: 1500,
+          settleMaxMs: 15e3,
           ...options.signal !== void 0 ? { abortSignal: options.signal } : {}
         });
       } catch (err) {
@@ -1421,7 +1507,6 @@ async function runEval(evalDef, connection, options = {}) {
       const turnData = transformSpans(turnSpans);
       const botResponse = turnData.messages.join("\n");
       const botDuration = turnData.handlerDuration;
-      const turnLabel = turn.event ? `[event: ${turn.event.type}]` : turn.user;
       const evalStart = Date.now();
       let assertions = [];
       const noResponse = !turn.expectSilence && turnData.messages.length === 0;
@@ -1476,7 +1561,7 @@ async function runEval(evalDef, connection, options = {}) {
       const turnPass = assertions.every((a) => a.pass);
       const evalDuration = Date.now() - evalStart;
       const turnReport = {
-        turnNumber: i + 1,
+        turnIndex: i,
         userMessage: turnLabel,
         botResponse,
         assertions,
@@ -1485,11 +1570,11 @@ async function runEval(evalDef, connection, options = {}) {
         evalDuration
       };
       turns.push(turnReport);
-      options.onProgress?.({
+      await options.onProgress?.({
         type: "turn_complete",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnNumber: i + 1,
+        turnIndex: i,
         totalTurns: evalDef.conversation.length,
         turnReport
       });
@@ -1536,7 +1621,15 @@ async function runEval(evalDef, connection, options = {}) {
   } catch (err) {
     const adkErr = isAdkError(err) ? err : void 0;
     if (!adkErr || !adkErr.expected) {
-      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${err.message}`);
+      const errMsg = err.message ?? String(err);
+      const errStack = err.stack ?? "";
+      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${errMsg}`);
+      logger.error(`  stack: ${errStack}`);
+      logger.error(`  eval: ${evalDef.name}, turn: ${turns.length}/${evalDef.conversation.length}`);
+      logger.error(`  connection: botId=${connection.botId}`);
+      logger.error(
+        `  spanSource: ${options.spanSource?.constructor?.name ?? "default"}, devServerUrl: ${options.devServerUrl ?? "n/a"}`
+      );
       options.onException?.(err, { source: "evals", eval_name: evalDef.name });
     }
     return {
@@ -1554,9 +1647,15 @@ async function runEval(evalDef, connection, options = {}) {
 }
 async function runEvalSuite(config, filter) {
   const start = Date.now();
-  const runId = randomUUID().replace(/-/g, "").slice(0, 26);
-  const evalsDir = `${config.agentPath}/evals`;
-  const allEvals = await loadEvalsFromDir(evalsDir);
+  const runId = config.runId ?? randomUUID().replace(/-/g, "").slice(0, 26);
+  let allEvals;
+  if (config.definitions) {
+    allEvals = config.definitions;
+  } else if (config.agentPath) {
+    allEvals = await loadEvalsFromDir(`${config.agentPath}/evals`);
+  } else {
+    throw new Error("Either `definitions` or `agentPath` must be provided in EvalRunnerConfig");
+  }
   const evals = filterEvals(allEvals, filter);
   if (evals.length === 0) {
     return {
@@ -1570,37 +1669,26 @@ async function runEvalSuite(config, filter) {
       ...filter !== void 0 ? { filter } : {}
     };
   }
-  await initLLMJudge(
-    {
-      token: config.credentials.token,
-      apiUrl: config.credentials.apiUrl,
-      botId: config.credentials.botId,
-      ...config.credentials.workspaceId !== void 0 ? { workspaceId: config.credentials.workspaceId } : {}
-    },
-    {
-      ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
-      ...config.logger !== void 0 ? { logger: config.logger } : {}
-    }
-  );
-  let webhookId = config.credentials.webhookId;
-  if (!webhookId) {
-    webhookId = await discoverWebhookId(config.credentials.botId, config.credentials.token, config.credentials.apiUrl);
-  }
+  await initLLMJudge(config.client, {
+    ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
+    ...config.logger !== void 0 ? { logger: config.logger } : {}
+  });
   const connection = {
-    webhookId,
-    botId: config.credentials.botId,
-    token: config.credentials.token,
-    apiUrl: config.credentials.apiUrl
+    client: config.client,
+    botId: config.botId
   };
   const devServerUrl = config.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = config.devServerHeaders ?? {};
-  await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
+  if (!config.createSpanSource) {
+    await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
+    await assertTraceStreamReadable(devServerUrl, devServerHeaders);
+  }
   const reports = [];
-  config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
+  await config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
   for (let i = 0; i < evals.length; i++) {
     if (config.signal?.aborted) break;
     const evalDef = evals[i];
-    config.onProgress?.({
+    await config.onProgress?.({
       type: "eval_start",
       evalName: evalDef.name,
       index: i,
@@ -1616,10 +1704,13 @@ async function runEvalSuite(config, filter) {
       ...config.signal !== void 0 ? { signal: config.signal } : {},
       ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {},
       ...config.logger !== void 0 ? { logger: config.logger } : {},
-      ...config.onException !== void 0 ? { onException: config.onException } : {}
+      ...config.onException !== void 0 ? { onException: config.onException } : {},
+      ...config.createSpanSource !== void 0 ? { spanSource: config.createSpanSource() } : {},
+      ...config.chatWebhookId !== void 0 ? { chatWebhookId: config.chatWebhookId } : {},
+      ...config.chatBaseUrl !== void 0 ? { chatBaseUrl: config.chatBaseUrl } : {}
     });
     reports.push(report);
-    config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
+    await config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
   }
   const runReport = {
     id: runId,
@@ -1634,7 +1725,7 @@ async function runEvalSuite(config, filter) {
   if (config.signal?.aborted) {
     runReport.aborted = true;
   }
-  config.onProgress?.({ type: "suite_complete", report: runReport });
+  await config.onProgress?.({ type: "suite_complete", report: runReport });
   return runReport;
 }
 export {