@botpress/evals 1.19.0-beta.7 → 2.0.0-alpha.0

package/dist/runner.js

@@ -1,96 +1,204 @@
-// src/runner.ts
-import { Client as BpClient2 } from "@botpress/client";
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// src/types.ts
+var defaultLogger = console;
+
+// src/errors.ts
+import { AdkError } from "@botpress/analytics";
+var EvalRunnerError = class extends AdkError {
+};
 
 // src/client.ts
-import { Client as BpClient } from "@botpress/client";
-import { createRequire } from "module";
-var defaultChatClient = null;
-function getDefaultChatClient() {
-  if (defaultChatClient) return defaultChatClient;
-  const require2 = createRequire(import.meta.url);
-  const mod = require2("@botpress/chat");
-  const ChatClient = mod.default?.Client ?? mod.Client;
-  if (!ChatClient) {
-    throw new Error("@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.");
-  }
-  return ChatClient;
-}
+var TRACE_STREAM_PREFLIGHT_TIMEOUT_MS = 1e4;
 var ChatSession = class {
-  constructor(webhookId, chatClient) {
-    this.webhookId = webhookId;
-    this.chatClient = chatClient;
+  constructor(bpClient, botId, _chatWebhookId, _chatBaseUrl, _chatClient) {
+    this.bpClient = bpClient;
+    this.botId = botId;
+    this._chatWebhookId = _chatWebhookId;
+    this._chatBaseUrl = _chatBaseUrl;
+    this._chatClient = _chatClient;
   }
   client = null;
   conversationId = null;
   async connect() {
-    const ChatClient = this.chatClient ?? getDefaultChatClient();
-    this.client = await ChatClient.connect({ webhookId: this.webhookId });
+    const webhookId = this._chatWebhookId ?? await discoverWebhookId(this.bpClient, this.botId);
+    const ChatClientCtor = this._chatClient ?? (await import(
+      /* webpackIgnore: true */
+      "@botpress/chat"
+    )).Client;
+    this.client = await ChatClientCtor.connect({
+      webhookId,
+      ...this._chatBaseUrl ? { baseApiUrl: this._chatBaseUrl } : {}
+    });
   }
-  get userId() {
+  /** Invariant guard — calling any session method before connect() is a runner bug. */
+  assertConnected() {
     if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
+      throw new EvalRunnerError({
+        code: "CHAT_NOT_CONNECTED",
+        message: "ChatSession not connected. Call connect() first."
+      });
     }
-    return this.client.user.id;
+    return this.client;
+  }
+  get userId() {
+    return this.assertConnected().user.id;
   }
   async ensureConversation() {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     if (!this.conversationId) {
-      const conv = await this.client.createConversation({});
+      const conv = await client.createConversation({});
       this.conversationId = conv.conversation.id;
       return conv.conversation.id;
     }
     return this.conversationId;
   }
+  /**
+   * Open a fresh conversation under the same user (the user is fixed at
+   * connect()) and make it active. Returns the new conversation id.
+   */
+  async newConversation() {
+    const client = this.assertConnected();
+    const conv = await client.createConversation({});
+    this.conversationId = conv.conversation.id;
+    return conv.conversation.id;
+  }
   /**
    * Send a user message. Fire-and-forget — observation is via SSECollector.
    */
   async sendMessage(message) {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await this.client.createMessage({ conversationId, payload: { type: "text", text: message } });
+    await client.createMessage({ conversationId, payload: { type: "text", text: message } });
   }
-  /**
-   * Send an event. Fire-and-forget — observation is via SSECollector.
-   */
-  async sendEvent(type, payload) {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+  async sendEvent(payload) {
+    const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await this.client.createEvent({ type, payload, conversationId });
+    await client.createEvent({ payload, conversationId });
   }
 };
-async function discoverWebhookId(botId, token, apiUrl) {
-  const client = new BpClient({ token, botId, apiUrl });
+async function discoverWebhookId(client, botId) {
   const { bot } = await client.getBot({ id: botId });
   const integrations = bot.integrations || {};
   const chat = Object.values(integrations).find((int) => int.name === "chat");
   const webhookId = chat?.webhookId;
   if (!webhookId || typeof webhookId !== "string") {
-    throw new Error(
-      [
+    throw new EvalRunnerError({
+      code: "CHAT_INTEGRATION_MISSING",
+      message: [
         "The `chat` integration is not installed on this bot \u2014 evals require it.",
         "",
         "To fix:",
-        "  1. Install it:    adk add chat",
+        "  1. Install it:    adk integrations add chat",
         "  2. Redeploy:      adk dev   (or: adk deploy)"
-      ].join("\n")
-    );
+      ].join("\n"),
+      expected: true
+    });
   }
   return webhookId;
 }
-async function assertChatChannelBound(devServerUrl, headers = {}) {
+async function assertTraceStreamReadable(devServerUrl, headers = {}, timeoutMs = TRACE_STREAM_PREFLIGHT_TIMEOUT_MS) {
+  const url = `${devServerUrl.replace(/\/$/, "")}/api/traces/stream?count=1`;
+  const abortController = new AbortController();
+  let timedOut = false;
+  let reader = null;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    abortController.abort();
+  }, timeoutMs);
+  try {
+    const res = await fetch(url, {
+      headers: { ...headers, Accept: "text/event-stream" },
+      signal: abortController.signal
+    });
+    if (!res.ok) {
+      throw traceStreamPreflightError(`Trace stream pre-flight failed: ${res.status} ${res.statusText}`);
+    }
+    if (!res.body) {
+      throw traceStreamPreflightError("Trace stream pre-flight failed: response had no body.");
+    }
+    reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        throw traceStreamPreflightError("Trace stream pre-flight failed: stream closed before snapshot.");
+      }
+      buffer += decoder.decode(value, { stream: true });
+      const parts = buffer.split("\n\n");
+      buffer = parts.pop() ?? "";
+      for (const part of parts) {
+        if (parseSSEEventName(part) === "snapshot") {
+          return;
+        }
+      }
+    }
+  } catch (err) {
+    if (err instanceof EvalRunnerError) {
+      throw err;
+    }
+    if (timedOut) {
+      throw traceStreamPreflightError(
+        `Trace stream pre-flight timed out after ${timeoutMs}ms before receiving a snapshot.`
+      );
+    }
+    throw traceStreamPreflightError(
+      `Trace stream pre-flight failed: ${err instanceof Error ? err.message : String(err)}`
+    );
+  } finally {
+    clearTimeout(timer);
+    reader?.cancel().catch(() => {
+    });
+  }
+}
+function parseSSEEventName(block) {
+  for (const line of block.split("\n")) {
+    if (line.startsWith("event:")) {
+      return line.slice("event:".length).trim();
+    }
+  }
+  return null;
+}
+function traceStreamPreflightError(message) {
+  return new EvalRunnerError({
+    code: "SSE_CONNECT_FAILED",
+    message,
+    expected: true,
+    suggestion: "Make sure the dev server is running (`adk dev`) and trace streaming is available."
+  });
+}
+async function assertChatChannelBound(devServerUrl, headers = {}, logger = defaultLogger) {
   let conversations = [];
   try {
     const res = await fetch(`${devServerUrl}/api/agent`, { headers });
     if (!res.ok) return;
     const agent = await res.json();
     conversations = agent.conversations ?? [];
-  } catch {
+  } catch (err) {
+    logger?.warn(
+      `Skipping chat.channel binding pre-flight \u2014 could not query ${devServerUrl}/api/agent: ${err instanceof Error ? err.message : String(err)}`
+    );
     return;
   }
   if (conversations.length === 0) return;
@@ -101,8 +209,9 @@ async function assertChatChannelBound(devServerUrl, headers = {}) {
   };
   const bound = conversations.some((c) => matchesChatChannel(c.channel));
   if (bound) return;
-  throw new Error(
-    [
+  throw new EvalRunnerError({
+    code: "CHAT_CHANNEL_UNBOUND",
+    message: [
       "No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.",
       "",
       "To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):",
@@ -111,17 +220,22 @@ async function assertChatChannelBound(devServerUrl, headers = {}) {
       "  channel: ['chat.channel', ...],    // array form",
       "",
       "The `chat` integration is how evals and `adk chat --single` send messages programmatically."
-    ].join("\n")
-  );
+    ].join("\n"),
+    expected: true
+  });
 }
 
-// src/sse-collector.ts
+// src/spans/sse-collector.ts
 function spanKey(span) {
   return `${span.id.trace}:${span.id.span}`;
 }
 function spanData(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
 }
+var TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
+function isTurnHandlerSpan(span) {
+  return TURN_HANDLER_SPAN_NAMES.has(span.name);
+}
 var SSECollector = class _SSECollector {
   devServerUrl;
   headers;
@@ -139,6 +253,10 @@ var SSECollector = class _SSECollector {
   connected = false;
   /** Error from the SSE stream if it died unexpectedly. */
   streamError = null;
+  /** Passive counters used only to explain timeout failures. */
+  updateEventCount = 0;
+  keepaliveEventCount = 0;
+  lastEvent = null;
   /** Set while a settle phase is in flight; called by disconnect() to short-circuit. */
   _cancelSettle = null;
   constructor(devServerUrl, headers = {}) {
@@ -154,40 +272,66 @@ var SSECollector = class _SSECollector {
     const attributeValue = "conversationId" in filter ? filter.conversationId : filter.workflowId;
     const url = `${this.devServerUrl}/api/traces/stream?attributeName=${encodeURIComponent(attributeName)}&attributeValue=${encodeURIComponent(attributeValue)}`;
     this.abortController = new AbortController();
+    const myController = this.abortController;
     const response = await fetch(url, {
       headers: { ...this.headers, Accept: "text/event-stream" },
-      signal: this.abortController.signal
+      signal: myController.signal
     });
     if (!response.ok) {
-      throw new Error(`SSE connection failed: ${response.status} ${response.statusText}`);
+      throw new EvalRunnerError({
+        code: "SSE_CONNECT_FAILED",
+        message: `SSE connection failed: ${response.status} ${response.statusText}`,
+        expected: true,
+        suggestion: "Make sure the dev server is running (`adk dev`) and reachable."
+      });
     }
     if (!response.body) {
-      throw new Error("SSE response has no body");
+      throw new EvalRunnerError({ code: "SSE_NO_BODY", message: "SSE response has no body" });
     }
     this.connected = true;
     this._consumeStream(response.body).catch((err) => {
-      if (this.connected) {
+      if (this.connected && this.abortController === myController) {
         this.streamError = err instanceof Error ? err : new Error(String(err));
       }
     });
     await this._waitForSnapshot();
   }
+  /**
+   * Re-point at a new conversation, keeping `spanMap` and `seenHandlerKeys` (so
+   * the earlier conversation stays visible and its completed handler can't
+   * satisfy the next turn). Resets the snapshot latch so connect() awaits the
+   * new conversation's snapshot.
+   */
+  async repoint(filter) {
+    this.connected = false;
+    this.abortController?.abort();
+    this.abortController = null;
+    this._cancelSettle?.();
+    this._cancelSettle = null;
+    this.waiters = [];
+    this.snapshotReceived = false;
+    this.snapshotResolvers = [];
+    this.streamError = null;
+    await this.connect(filter);
+  }
   /**
    * Mark the beginning of a new turn.
-   * Records which handler.conversation span IDs have been seen so far,
-   * so waitForTurnComplete() only resolves on NEW handler completions.
+   * Records which turn-handler span IDs (handler.conversation / handler.event)
+   * have been seen so far, so waitForTurnComplete() only resolves on NEW
+   * handler completions.
    */
   startTurn() {
     this.turnStartKeys = new Set(this.spanMap.keys());
     for (const [key, span] of this.spanMap) {
-      if (span.name === "handler.conversation") {
+      if (isTurnHandlerSpan(span)) {
         this.seenHandlerKeys.add(key);
       }
     }
   }
   static SETTLE_MS = 100;
   /**
-   * Wait for a handler.conversation span to reach terminal status (ok or error).
+   * Wait for a turn-handler span (handler.conversation or handler.event) to
+   * reach terminal status (ok or error).
    * Only considers spans NOT seen before the last startTurn() call.
    *
    * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)
@@ -196,8 +340,10 @@ var SSECollector = class _SSECollector {
    * spans may arrive slightly after the handler close event.
    */
   async waitForTurnComplete(opts) {
-    if (this._hasNewCompletedHandler()) {
-      return this._settleAfterHandler();
+    const accept = opts.acceptSpanNames;
+    const settle = { quietMs: opts.settleQuietMs, maxMs: opts.settleMaxMs };
+    if (this._hasNewCompletedHandler(accept)) {
+      return this._settleAfterHandler(settle);
     }
     if (opts.abortSignal?.aborted) {
       throw new Error("waitForTurnComplete aborted");
@@ -206,8 +352,13 @@ var SSECollector = class _SSECollector {
       const timer = setTimeout(() => {
         this._removeWaiter(waiter);
         opts.abortSignal?.removeEventListener("abort", onAbort);
-        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation span completed";
-        reject(new Error(`waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}`));
+        const expected = accept ? [...accept].join(" or ") : "handler.conversation or handler.event";
+        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : `no ${expected} span completed`;
+        reject(
+          new Error(
+            `waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}; diagnostics: ${this._formatTurnDiagnostics()}`
+          )
+        );
       }, opts.timeout);
       const onAbort = () => {
         clearTimeout(timer);
@@ -216,11 +367,11 @@ var SSECollector = class _SSECollector {
       };
       opts.abortSignal?.addEventListener("abort", onAbort, { once: true });
       const waiter = (_span) => {
-        if (this._hasNewCompletedHandler()) {
+        if (this._hasNewCompletedHandler(accept)) {
           clearTimeout(timer);
           this._removeWaiter(waiter);
           opts.abortSignal?.removeEventListener("abort", onAbort);
-          this._settleAfterHandler().then(resolve2, reject);
+          this._settleAfterHandler(settle).then(resolve2, reject);
         }
       };
       this.waiters.push(waiter);
@@ -234,7 +385,9 @@ var SSECollector = class _SSECollector {
    * Exposes a cancellation hook via _cancelSettle so disconnect() can short-
    * circuit an in-flight settle phase and avoid leaking timers for up to 2s.
    */
-  _settleAfterHandler() {
+  _settleAfterHandler(settle) {
+    const quietMs = settle?.quietMs ?? _SSECollector.SETTLE_MS;
+    const maxMs = settle?.maxMs ?? _SSECollector.MAX_SETTLE_MS;
     return new Promise((resolve2) => {
       let resolved = false;
       const done = () => {
@@ -246,12 +399,12 @@ var SSECollector = class _SSECollector {
         this._cancelSettle = null;
         resolve2();
       };
-      const hardCap = setTimeout(done, _SSECollector.MAX_SETTLE_MS);
-      let settleTimer = setTimeout(done, _SSECollector.SETTLE_MS);
+      const hardCap = setTimeout(done, maxMs);
+      let settleTimer = setTimeout(done, quietMs);
       const settleWaiter = () => {
         if (resolved) return;
         clearTimeout(settleTimer);
-        settleTimer = setTimeout(done, _SSECollector.SETTLE_MS);
+        settleTimer = setTimeout(done, quietMs);
       };
       this.waiters.push(settleWaiter);
       this._cancelSettle = done;
@@ -416,6 +569,7 @@ ${value}` : value;
    * Dispatch a parsed SSE event to the appropriate handler.
    */
   _handleEvent(eventName, payload) {
+    this._recordEvent(eventName);
     if (eventName === "snapshot") {
       const data2 = payload;
       const spans = Array.isArray(data2.spans) ? data2.spans : [];
@@ -435,6 +589,44 @@ ${value}` : value;
       }
     }
   }
+  _recordEvent(eventName) {
+    this.lastEvent = { name: eventName, at: Date.now() };
+    if (eventName === "update") {
+      this.updateEventCount++;
+    } else if (eventName === "keepalive") {
+      this.keepaliveEventCount++;
+    }
+  }
+  _formatTurnDiagnostics() {
+    const turnSpans = this.getTurnSpans();
+    const spanNames = /* @__PURE__ */ new Set();
+    const handlerStatuses = /* @__PURE__ */ new Set();
+    for (const span of turnSpans) {
+      spanNames.add(span.name);
+      if (isTurnHandlerSpan(span)) {
+        handlerStatuses.add(span.status);
+      }
+    }
+    const lastEvent = this.lastEvent ? `${this.lastEvent.name} ${Date.now() - this.lastEvent.at}ms ago` : "none";
+    return [
+      `snapshotReceived=${this.snapshotReceived}`,
+      `totalUpdates=${this.updateEventCount}`,
+      `totalKeepalives=${this.keepaliveEventCount}`,
+      `lastEvent=${lastEvent}`,
+      `totalSpans=${this.spanMap.size}`,
+      `turnSpans=${turnSpans.length}`,
+      `turnSpanNames=${this._formatSet(spanNames)}`,
+      `turnHandlerStatuses=${this._formatSet(handlerStatuses)}`
+    ].join(", ");
+  }
+  _formatSet(values, limit = 12) {
+    if (values.size === 0) {
+      return "none";
+    }
+    const items = Array.from(values);
+    const suffix = items.length > limit ? `|+${items.length - limit} more` : "";
+    return `${items.slice(0, limit).join("|")}${suffix}`;
+  }
   /**
    * Insert or replace a span in the map (dedup by trace+span ID).
    */
@@ -460,7 +652,7 @@ ${value}` : value;
     }
   }
   /**
-   * Check if there is a handler.conversation span that:
+   * Check if there is a turn-handler span (handler.conversation / handler.event) that:
    * 1. Was NOT seen before startTurn()
    * 2. Has terminal status (ok or error)
    *
@@ -469,9 +661,9 @@ ${value}` : value;
    * waitForTurnComplete check (which would otherwise resolve instantly
    * against stale data instead of waiting for the new turn's handler).
    */
-  _hasNewCompletedHandler() {
+  _hasNewCompletedHandler(acceptSpanNames) {
     for (const [key, span] of this.spanMap) {
-      if (span.name === "handler.conversation" && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
+      if (isTurnHandlerSpan(span) && (!acceptSpanNames || acceptSpanNames.has(span.name)) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
         this.seenHandlerKeys.add(key);
         return true;
       }
@@ -498,6 +690,38 @@ ${value}` : value;
   }
 };
 
+// src/spans/local-span-source.ts
+var LocalSpanSource = class {
+  collector;
+  constructor(devServerUrl, headers = {}) {
+    this.collector = new SSECollector(devServerUrl, headers);
+  }
+  async connect(filter) {
+    await this.collector.connect(filter);
+  }
+  async repoint(filter) {
+    await this.collector.repoint(filter);
+  }
+  startTurn() {
+    this.collector.startTurn();
+  }
+  async waitForTurnComplete(opts) {
+    await this.collector.waitForTurnComplete(opts);
+  }
+  async waitForWorkflow(name, opts) {
+    await this.collector.waitForWorkflow(name, opts);
+  }
+  getTurnSpans() {
+    return this.collector.getTurnSpans();
+  }
+  getAllSpans() {
+    return this.collector.getAllSpans();
+  }
+  disconnect() {
+    this.collector.disconnect();
+  }
+};
+
 // src/transformer.ts
 function data(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
@@ -585,7 +809,7 @@ function extractWorkflowSpans(spans) {
 }
 function extractHandler(spans) {
   const handler = spans.find(
-    (s) => (s.name === "handler.conversation" || s.name === "handler.workflow") && s.status !== "running"
+    (s) => (s.name === "handler.conversation" || s.name === "handler.event" || s.name === "handler.workflow") && s.status !== "running"
   );
   if (!handler) {
     return { handlerDuration: 0, handlerStatus: "ok" };
@@ -597,53 +821,26 @@ function extractHandler(spans) {
 }
 
 // src/graders/llm.ts
-import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
-var DEFAULT_PASS_THRESHOLD = 3;
-function buildJudgeSystemPrompt(passThreshold) {
-  return `You are an evaluation judge for a chatbot. You will be given:
-- The user's message
-- The bot's response
-- Grading criteria
-
-Your job is to determine if the bot's response meets the criteria.
-
-Respond with a JSON object:
-{
-  "pass": true/false,
-  "score": 1-5,
-  "reason": "brief explanation"
-}
-
-Scoring guide:
-- 5: Fully meets criteria, excellent response
-- 4: Mostly meets criteria, minor issues
-- 3: Partially meets criteria, notable gaps
-- 2: Barely meets criteria, significant issues
-- 1: Does not meet criteria
-
-A score of ${passThreshold} or above is a pass.`;
-}
+import { Zai } from "@botpress/zai";
 var MODEL_ALIASES = ["fast", "best"];
-var _cognitive = null;
+var _zai = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _failClosed = true;
+var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
-async function initLLMJudge(credentials, options) {
-  const log = options?.logger ?? console;
-  const client = new Client({
-    token: credentials.token,
-    apiUrl: credentials.apiUrl,
-    botId: credentials.botId,
-    ...credentials.workspaceId !== void 0 ? { workspaceId: credentials.workspaceId } : {}
-  });
-  _cognitive = new Cognitive({ client, __experimental_beta: true });
+async function initLLMJudge(client, options) {
+  const log = options?.logger ?? defaultLogger;
+  _log = log;
+  _failClosed = options?.failClosed ?? true;
+  const cognitive = new Cognitive({ client, __experimental_beta: true });
   try {
-    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys());
+    _availableModels = Array.from((await cognitive.fetchRemoteModels()).keys());
   } catch (err) {
     log.warn(`Failed to fetch available models: ${err.message}. LLM judge will be unavailable.`);
     _availableModels = [];
@@ -654,65 +851,44 @@ async function initLLMJudge(credentials, options) {
   } else if (!_isModelValid(_judgeModel)) {
     log.warn(`Configured LLM judge model "${_judgeModel}" is invalid. Run "adk models" to list available models.`);
   }
+  _zai = new Zai({ client: cognitive, modelId: _judgeModel });
 }
 async function gradeLLMJudge(botResponse, criteria, context) {
-  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
+  const assertion = `llm_judge: "${criteria}"`;
   try {
-    if (!_cognitive) {
+    if (!_zai) {
+      _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
-        assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        assertion,
+        pass: !_failClosed,
+        skipped: true,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+        actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
       return {
-        assertion: `llm_judge: "${criteria}"`,
+        assertion,
         pass: false,
         expected: criteria,
         actual: "FAILED \u2014 invalid LLM judge model"
       };
     }
-    const { output } = await _cognitive.generateContent({
-      model: _judgeModel,
-      temperature: 0,
-      responseFormat: "json_object",
-      systemPrompt: buildJudgeSystemPrompt(threshold),
-      messages: [
-        {
-          role: "user",
-          content: `User message: ${context.userMessage}
-
-Bot response: ${botResponse}
-
-Criteria: ${criteria}`
-        }
-      ]
-    });
-    const rawContent = output?.choices?.[0]?.content;
-    const content = typeof rawContent === "string" ? rawContent : null;
-    if (!content) {
-      return {
-        assertion: `llm_judge: "${criteria}"`,
-        pass: true,
-        expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge returned empty response"
-      };
-    }
-    const verdict = JSON.parse(content);
+    const { output } = await _zai.check({ userMessage: context.userMessage, botResponse }, criteria).result();
     return {
-      assertion: `llm_judge: "${criteria}"`,
-      pass: verdict.score >= threshold,
+      assertion,
+      pass: output.value,
       expected: criteria,
-      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
+      actual: `${output.value ? "Pass" : "Fail"} \u2014 ${output.explanation}`
     };
   } catch (err) {
+    _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
-      assertion: `llm_judge: "${criteria}"`,
-      pass: true,
+      assertion,
+      pass: !_failClosed,
+      skipped: true,
       expected: criteria,
-      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+      actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
   }
 }
@@ -1053,30 +1229,6 @@ function gradeOutcome(mutations, evalDef, workflowSpans) {
 // src/loader.ts
 import { readdirSync, existsSync } from "fs";
 import { resolve } from "path";
-
-// src/definition.ts
-var Eval = class {
-  name;
-  description;
-  tags;
-  type;
-  setup;
-  conversation;
-  outcome;
-  options;
-  constructor(def) {
-    this.name = def.name;
-    this.conversation = def.conversation;
-    if (def.description !== void 0) this.description = def.description;
-    if (def.tags !== void 0) this.tags = def.tags;
-    if (def.type !== void 0) this.type = def.type;
-    if (def.setup !== void 0) this.setup = def.setup;
-    if (def.outcome !== void 0) this.outcome = def.outcome;
-    if (def.options !== void 0) this.options = def.options;
-  }
-};
-
-// src/loader.ts
 function isEvalDefinition(value) {
   return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
 }
@@ -1087,15 +1239,17 @@ async function loadEvalFile(filePath) {
     mod = await import(absPath);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
-    throw new Error(
-      `Failed to load eval file ${filePath}: ${msg}
+    throw new EvalRunnerError({
+      code: "EVAL_LOAD_FAILED",
+      message: `Failed to load eval file ${filePath}: ${msg}
 
 Make sure your eval file:
   - Has no syntax or type errors
   - Exports one or more \`new Eval({...})\` instances
   - Has all dependencies installed (\`bun install\`)`,
-      { cause: err }
-    );
+      expected: true,
+      cause: err
+    });
   }
   const results = [];
   for (const [key, value] of Object.entries(mod)) {
@@ -1105,9 +1259,11 @@ Make sure your eval file:
     }
   }
   if (results.length === 0) {
-    throw new Error(
-      `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`
-    );
+    throw new EvalRunnerError({
+      code: "EVAL_FILE_EMPTY",
+      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
+      expected: true
+    });
   }
   return results;
 }
@@ -1125,7 +1281,12 @@ async function loadEvalsFromDir(dirPath) {
   const seen = /* @__PURE__ */ new Set();
   for (const e of evals) {
     if (seen.has(e.name)) {
-      throw new Error(`Duplicate eval name "${e.name}" found in ${dirPath}`);
+      throw new EvalRunnerError({
+        code: "EVAL_DUPLICATE_NAME",
+        message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
+        expected: true,
+        details: { name: e.name }
+      });
     }
     seen.add(e.name);
   }
@@ -1148,6 +1309,7 @@ function filterEvals(evals, filter) {
 }
 
 // src/runner.ts
+import { isAdkError } from "@botpress/analytics";
 import { randomUUID } from "crypto";
 var DEFAULT_IDLE_TIMEOUT = 3e4;
 var DEFAULT_DEV_SERVER_URL = "http://localhost:3001";
@@ -1189,7 +1351,10 @@ async function seedEvalState(client, ctx, setup) {
   }
   if (state.conversation) {
     if (!ctx.conversationId) {
-      throw new Error("Cannot seed conversation state before a conversation is created.");
+      throw new EvalRunnerError({
+        code: "EVAL_SEED_NO_CONVERSATION",
+        message: "Cannot seed conversation state before a conversation is created."
+      });
     }
     writes.push(
       client.setState({
@@ -1225,32 +1390,28 @@ function evalMetadata(evalDef) {
 async function runEval(evalDef, connection, options = {}) {
   const devServerUrl = options.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = options.devServerHeaders ?? {};
+  const logger = options.logger ?? defaultLogger;
   const idleTimeout = evalDef.options?.idleTimeout ?? options.idleTimeout ?? DEFAULT_IDLE_TIMEOUT;
   const judgePassThreshold = evalDef.options?.judgePassThreshold ?? options.judgePassThreshold;
   const start = Date.now();
   const turns = [];
   let outcomeAssertions = [];
-  const collector = new SSECollector(devServerUrl, devServerHeaders);
+  const collector = options.spanSource ?? new LocalSpanSource(devServerUrl, devServerHeaders);
   try {
-    const session = new ChatSession(connection.webhookId, options.chatClient);
+    const session = new ChatSession(
+      connection.client,
+      connection.botId,
+      options.chatWebhookId,
+      options.chatBaseUrl,
+      options.chatClient
+    );
     await session.connect();
-    let bpClient = null;
-    const getBpClient = () => {
-      if (!bpClient) {
-        bpClient = new BpClient2({
-          token: connection.token,
-          botId: connection.botId,
-          apiUrl: connection.apiUrl
-        });
-      }
-      return bpClient;
-    };
     let conversationId = "";
     if (evalDef.setup?.state?.conversation || evalDef.setup?.workflow) {
       conversationId = await session.ensureConversation();
     }
     await seedEvalState(
-      getBpClient(),
+      connection.client,
       {
         botId: connection.botId,
         userId: session.userId,
@@ -1259,7 +1420,7 @@ async function runEval(evalDef, connection, options = {}) {
       evalDef.setup
     );
     await triggerEvalWorkflow(
-      getBpClient(),
+      connection.client,
       {
         userId: session.userId,
         ...conversationId ? { conversationId } : {}
@@ -1272,38 +1433,57 @@ async function runEval(evalDef, connection, options = {}) {
     if (conversationId) {
       await collector.connect({ conversationId });
     } else if (evalDef.conversation.length > 0) {
-      throw new Error("Cannot run conversation eval without a conversationId \u2014 no chat session or setup created one.");
+      throw new EvalRunnerError({
+        code: "EVAL_NO_CONVERSATION_ID",
+        message: "Cannot run conversation eval without a conversationId \u2014 no chat session or setup created one."
+      });
     }
     for (let i = 0; i < evalDef.conversation.length; i++) {
       if (options.signal?.aborted) break;
       const turn = evalDef.conversation[i];
+      const turnConfigError = (message) => new EvalRunnerError({
+        code: "EVAL_TURN_CONFIG_INVALID",
+        message: `Turn ${i + 1}: ${message}`,
+        expected: true,
+        details: { turn: i + 1 }
+      });
       if (turn.expectSilence && turn.assert?.response) {
-        throw new Error(`Turn ${i + 1}: 'expectSilence' and 'assert.response' are mutually exclusive.`);
+        throw turnConfigError(`'expectSilence' and 'assert.response' are mutually exclusive.`);
       }
       if (turn.user && turn.event) {
-        throw new Error(`Turn ${i + 1}: 'user' and 'event' are mutually exclusive.`);
+        throw turnConfigError(`'user' and 'event' are mutually exclusive.`);
       }
       if (!turn.user && !turn.event) {
-        throw new Error(`Turn ${i + 1}: must have either 'user' or 'event'.`);
+        throw turnConfigError(`must have either 'user' or 'event'.`);
       }
-      const turnLabelForEvent = turn.event ? `[event: ${turn.event.type}]` : turn.user;
-      options.onProgress?.({
+      const turnLabel = turn.event ? "[event]" : turn.user;
+      await options.onProgress?.({
         type: "turn_start",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnNumber: i + 1,
+        turnIndex: i,
         totalTurns: evalDef.conversation.length,
-        userMessage: turnLabelForEvent
+        userMessage: turnLabel
       });
+      if (turn.newConversation && i > 0) {
+        const newConversationId = await session.newConversation();
+        await collector.repoint({ conversationId: newConversationId });
+      }
       collector.startTurn();
       if (turn.event) {
-        await session.sendEvent(turn.event.type, turn.event.payload ?? {});
+        await session.sendEvent(turn.event.payload);
       } else {
         await session.sendMessage(turn.user);
       }
       try {
         await collector.waitForTurnComplete({
           timeout: idleTimeout,
+          acceptSpanNames: turn.event ? /* @__PURE__ */ new Set(["handler.event"]) : /* @__PURE__ */ new Set(["handler.conversation"]),
+          // Long quiet window: a tool-started workflow and its callback only run
+          // AFTER the handler closes; sending the next message into that window
+          // can get it absorbed by their transcript saves and silently skipped.
+          settleQuietMs: 1500,
+          settleMaxMs: 15e3,
           ...options.signal !== void 0 ? { abortSignal: options.signal } : {}
         });
       } catch (err) {
@@ -1327,12 +1507,17 @@ async function runEval(evalDef, connection, options = {}) {
       const turnData = transformSpans(turnSpans);
       const botResponse = turnData.messages.join("\n");
       const botDuration = turnData.handlerDuration;
-      const turnLabel = turn.event ? `[event: ${turn.event.type}]` : turn.user;
-      if (!turn.expectSilence && turnData.messages.length === 0) {
-        throw new Error(`Turn ${i + 1}: bot produced no response.`);
-      }
       const evalStart = Date.now();
       let assertions = [];
+      const noResponse = !turn.expectSilence && turnData.messages.length === 0;
+      if (noResponse) {
+        assertions.push({
+          assertion: "response",
+          pass: false,
+          expected: "Bot produces a response",
+          actual: `Turn ${i + 1}: bot produced no response.`
+        });
+      }
       if (turn.expectSilence) {
         const wasSilent = turnData.messages.length === 0;
         assertions.push({
@@ -1342,7 +1527,7 @@ async function runEval(evalDef, connection, options = {}) {
           actual: wasSilent ? "No response" : `Bot responded: "${botResponse}"`
         });
       }
-      if (turn.assert?.response) {
+      if (turn.assert?.response && !noResponse) {
         assertions = await gradeResponse(botResponse, turn.assert.response, {
           userMessage: turnLabel,
           ...judgePassThreshold !== void 0 ? { judgePassThreshold } : {}
@@ -1376,7 +1561,7 @@ async function runEval(evalDef, connection, options = {}) {
       const turnPass = assertions.every((a) => a.pass);
       const evalDuration = Date.now() - evalStart;
       const turnReport = {
-        turnNumber: i + 1,
+        turnIndex: i,
         userMessage: turnLabel,
         botResponse,
         assertions,
@@ -1385,11 +1570,11 @@ async function runEval(evalDef, connection, options = {}) {
         evalDuration
       };
       turns.push(turnReport);
-      options.onProgress?.({
+      await options.onProgress?.({
         type: "turn_complete",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnNumber: i + 1,
+        turnIndex: i,
         totalTurns: evalDef.conversation.length,
         turnReport
       });
@@ -1434,13 +1619,27 @@ async function runEval(evalDef, connection, options = {}) {
       ...aborted ? { error: "Eval aborted" } : {}
     };
   } catch (err) {
+    const adkErr = isAdkError(err) ? err : void 0;
+    if (!adkErr || !adkErr.expected) {
+      const errMsg = err.message ?? String(err);
+      const errStack = err.stack ?? "";
+      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${errMsg}`);
+      logger.error(`  stack: ${errStack}`);
+      logger.error(`  eval: ${evalDef.name}, turn: ${turns.length}/${evalDef.conversation.length}`);
+      logger.error(`  connection: botId=${connection.botId}`);
+      logger.error(
+        `  spanSource: ${options.spanSource?.constructor?.name ?? "default"}, devServerUrl: ${options.devServerUrl ?? "n/a"}`
+      );
+      options.onException?.(err, { source: "evals", eval_name: evalDef.name });
+    }
     return {
       ...evalMetadata(evalDef),
       turns,
       outcomeAssertions,
       pass: false,
       duration: Date.now() - start,
-      error: err.message
+      error: err.message,
+      ...adkErr ? { errorCode: adkErr.code } : {}
     };
   } finally {
     collector.disconnect();
@@ -1448,9 +1647,15 @@ async function runEval(evalDef, connection, options = {}) {
 }
 async function runEvalSuite(config, filter) {
   const start = Date.now();
-  const runId = randomUUID().replace(/-/g, "").slice(0, 26);
-  const evalsDir = `${config.agentPath}/evals`;
-  const allEvals = await loadEvalsFromDir(evalsDir);
+  const runId = config.runId ?? randomUUID().replace(/-/g, "").slice(0, 26);
+  let allEvals;
+  if (config.definitions) {
+    allEvals = config.definitions;
+  } else if (config.agentPath) {
+    allEvals = await loadEvalsFromDir(`${config.agentPath}/evals`);
+  } else {
+    throw new Error("Either `definitions` or `agentPath` must be provided in EvalRunnerConfig");
+  }
   const evals = filterEvals(allEvals, filter);
   if (evals.length === 0) {
     return {
@@ -1464,37 +1669,26 @@ async function runEvalSuite(config, filter) {
       ...filter !== void 0 ? { filter } : {}
     };
   }
-  await initLLMJudge(
-    {
-      token: config.credentials.token,
-      apiUrl: config.credentials.apiUrl,
-      botId: config.credentials.botId,
-      ...config.credentials.workspaceId !== void 0 ? { workspaceId: config.credentials.workspaceId } : {}
-    },
-    {
-      ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
-      ...config.logger !== void 0 ? { logger: config.logger } : {}
-    }
-  );
-  let webhookId = config.credentials.webhookId;
-  if (!webhookId) {
-    webhookId = await discoverWebhookId(config.credentials.botId, config.credentials.token, config.credentials.apiUrl);
-  }
+  await initLLMJudge(config.client, {
+    ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
+    ...config.logger !== void 0 ? { logger: config.logger } : {}
+  });
   const connection = {
-    webhookId,
-    botId: config.credentials.botId,
-    token: config.credentials.token,
-    apiUrl: config.credentials.apiUrl
+    client: config.client,
+    botId: config.botId
   };
   const devServerUrl = config.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = config.devServerHeaders ?? {};
-  await assertChatChannelBound(devServerUrl, devServerHeaders);
+  if (!config.createSpanSource) {
+    await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
+    await assertTraceStreamReadable(devServerUrl, devServerHeaders);
+  }
   const reports = [];
-  config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
+  await config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
   for (let i = 0; i < evals.length; i++) {
     if (config.signal?.aborted) break;
     const evalDef = evals[i];
-    config.onProgress?.({
+    await config.onProgress?.({
       type: "eval_start",
       evalName: evalDef.name,
       index: i,
@@ -1508,10 +1702,15 @@ async function runEvalSuite(config, filter) {
       ...config.evalOptions?.judgePassThreshold !== void 0 ? { judgePassThreshold: config.evalOptions.judgePassThreshold } : {},
       ...config.onProgress !== void 0 ? { onProgress: config.onProgress } : {},
       ...config.signal !== void 0 ? { signal: config.signal } : {},
-      ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {}
+      ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {},
+      ...config.logger !== void 0 ? { logger: config.logger } : {},
+      ...config.onException !== void 0 ? { onException: config.onException } : {},
+      ...config.createSpanSource !== void 0 ? { spanSource: config.createSpanSource() } : {},
+      ...config.chatWebhookId !== void 0 ? { chatWebhookId: config.chatWebhookId } : {},
+      ...config.chatBaseUrl !== void 0 ? { chatBaseUrl: config.chatBaseUrl } : {}
     });
     reports.push(report);
-    config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
+    await config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
   }
   const runReport = {
     id: runId,
@@ -1526,7 +1725,7 @@ async function runEvalSuite(config, filter) {
   if (config.signal?.aborted) {
     runReport.aborted = true;
   }
-  config.onProgress?.({ type: "suite_complete", report: runReport });
+  await config.onProgress?.({ type: "suite_complete", report: runReport });
   return runReport;
 }
 export {