@botpress/evals 1.19.0-beta.6 → 1.19.0-beta.9

package/dist/runner.js

@@ -1,9 +1,84 @@
 // src/runner.ts
 import { Client as BpClient2 } from "@botpress/client";
 
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// src/types.ts
+var defaultLogger = console;
+
 // src/client.ts
 import { Client as BpClient } from "@botpress/client";
 import { createRequire } from "module";
+
+// ../analytics/src/errors.ts
+var AdkError = class extends Error {
+  /**
+   * Marker for cross-module-instance checks — `instanceof` breaks when a
+   * package is duplicated in the dependency graph.
+   * @internal
+   */
+  static __IS_ADK_BASE_ERROR = true;
+  /**
+   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
+   * pin `Code` to a literal union (`class EvalRunnerError extends
+   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
+   * in switches.
+   */
+  code;
+  /**
+   * True for user/environment conditions the developer can fix themselves
+   * (validation, auth, connectivity). False for internal bugs. Defaults to
+   * false so unclassified errors surface as bugs until someone triages them.
+   */
+  expected;
+  /** Structured context. Sanitized before any telemetry leaves the machine. */
+  details;
+  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
+  suggestion;
+  constructor(opts) {
+    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
+    this.name = this.constructor.name;
+    this.code = opts.code;
+    this.expected = opts.expected ?? false;
+    if (opts.details !== void 0) {
+      this.details = opts.details;
+    }
+    if (opts.suggestion !== void 0) {
+      this.suggestion = opts.suggestion;
+    }
+  }
+};
+function isAdkError(error) {
+  if (error instanceof AdkError) return true;
+  if (!(error instanceof Error)) return false;
+  return error.constructor?.__IS_ADK_BASE_ERROR === true;
+}
+
+// src/errors.ts
+var EvalRunnerError = class extends AdkError {
+};
+
+// src/client.ts
 var defaultChatClient = null;
 function getDefaultChatClient() {
   if (defaultChatClient) return defaultChatClient;
@@ -11,7 +86,10 @@ function getDefaultChatClient() {
   const mod = require2("@botpress/chat");
   const ChatClient = mod.default?.Client ?? mod.Client;
   if (!ChatClient) {
-    throw new Error("@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.");
+    throw new EvalRunnerError({
+      code: "CHAT_CLIENT_MISSING",
+      message: "@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly."
+    });
   }
   return ChatClient;
 }
@@ -26,18 +104,23 @@ var ChatSession = class {
     const ChatClient = this.chatClient ?? getDefaultChatClient();
     this.client = await ChatClient.connect({ webhookId: this.webhookId });
   }
-  get userId() {
+  /** Invariant guard — calling any session method before connect() is a runner bug. */
+  assertConnected() {
     if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
+      throw new EvalRunnerError({
+        code: "CHAT_NOT_CONNECTED",
+        message: "ChatSession not connected. Call connect() first."
+      });
     }
-    return this.client.user.id;
+    return this.client;
+  }
+  get userId() {
+    return this.assertConnected().user.id;
   }
   async ensureConversation() {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     if (!this.conversationId) {
-      const conv = await this.client.createConversation({});
+      const conv = await client.createConversation({});
       this.conversationId = conv.conversation.id;
       return conv.conversation.id;
     }
@@ -47,21 +130,17 @@ var ChatSession = class {
    * Send a user message. Fire-and-forget — observation is via SSECollector.
    */
   async sendMessage(message) {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await this.client.createMessage({ conversationId, payload: { type: "text", text: message } });
+    await client.createMessage({ conversationId, payload: { type: "text", text: message } });
   }
   /**
    * Send an event. Fire-and-forget — observation is via SSECollector.
    */
   async sendEvent(type, payload) {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await this.client.createEvent({ type, payload, conversationId });
+    await client.createEvent({ type, payload, conversationId });
   }
 };
 async function discoverWebhookId(botId, token, apiUrl) {
@@ -71,26 +150,31 @@ async function discoverWebhookId(botId, token, apiUrl) {
   const chat = Object.values(integrations).find((int) => int.name === "chat");
   const webhookId = chat?.webhookId;
   if (!webhookId || typeof webhookId !== "string") {
-    throw new Error(
-      [
+    throw new EvalRunnerError({
+      code: "CHAT_INTEGRATION_MISSING",
+      message: [
         "The `chat` integration is not installed on this bot \u2014 evals require it.",
         "",
         "To fix:",
         "  1. Install it:    adk add chat",
         "  2. Redeploy:      adk dev   (or: adk deploy)"
-      ].join("\n")
-    );
+      ].join("\n"),
+      expected: true
+    });
   }
   return webhookId;
 }
-async function assertChatChannelBound(devServerUrl, headers = {}) {
+async function assertChatChannelBound(devServerUrl, headers = {}, logger = defaultLogger) {
   let conversations = [];
   try {
     const res = await fetch(`${devServerUrl}/api/agent`, { headers });
     if (!res.ok) return;
     const agent = await res.json();
     conversations = agent.conversations ?? [];
-  } catch {
+  } catch (err) {
+    logger?.warn(
+      `Skipping chat.channel binding pre-flight \u2014 could not query ${devServerUrl}/api/agent: ${err instanceof Error ? err.message : String(err)}`
+    );
     return;
   }
   if (conversations.length === 0) return;
@@ -101,8 +185,9 @@ async function assertChatChannelBound(devServerUrl, headers = {}) {
   };
   const bound = conversations.some((c) => matchesChatChannel(c.channel));
   if (bound) return;
-  throw new Error(
-    [
+  throw new EvalRunnerError({
+    code: "CHAT_CHANNEL_UNBOUND",
+    message: [
       "No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.",
       "",
       "To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):",
@@ -111,8 +196,9 @@ async function assertChatChannelBound(devServerUrl, headers = {}) {
       "  channel: ['chat.channel', ...],    // array form",
       "",
       "The `chat` integration is how evals and `adk chat --single` send messages programmatically."
-    ].join("\n")
-  );
+    ].join("\n"),
+    expected: true
+  });
 }
 
 // src/sse-collector.ts
@@ -159,10 +245,15 @@ var SSECollector = class _SSECollector {
       signal: this.abortController.signal
     });
     if (!response.ok) {
-      throw new Error(`SSE connection failed: ${response.status} ${response.statusText}`);
+      throw new EvalRunnerError({
+        code: "SSE_CONNECT_FAILED",
+        message: `SSE connection failed: ${response.status} ${response.statusText}`,
+        expected: true,
+        suggestion: "Make sure the dev server is running (`adk dev`) and reachable."
+      });
     }
     if (!response.body) {
-      throw new Error("SSE response has no body");
+      throw new EvalRunnerError({ code: "SSE_NO_BODY", message: "SSE response has no body" });
     }
     this.connected = true;
     this._consumeStream(response.body).catch((err) => {
@@ -628,13 +719,15 @@ var MODEL_ALIASES = ["fast", "best"];
 var _cognitive = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
 async function initLLMJudge(credentials, options) {
-  const log = options?.logger ?? console;
+  const log = options?.logger ?? defaultLogger;
+  _log = log;
   const client = new Client({
     token: credentials.token,
     apiUrl: credentials.apiUrl,
@@ -659,11 +752,12 @@ async function gradeLLMJudge(botResponse, criteria, context) {
   const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
   try {
     if (!_cognitive) {
+      _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+        actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
@@ -693,11 +787,12 @@ Criteria: ${criteria}`
     const rawContent = output?.choices?.[0]?.content;
     const content = typeof rawContent === "string" ? rawContent : null;
     if (!content) {
+      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge returned empty response"
+        actual: "FAILED \u2014 LLM judge returned empty response"
       };
     }
     const verdict = JSON.parse(content);
@@ -708,11 +803,12 @@ Criteria: ${criteria}`
       actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
     };
   } catch (err) {
+    _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
       assertion: `llm_judge: "${criteria}"`,
-      pass: true,
+      pass: false,
       expected: criteria,
-      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+      actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
   }
 }
@@ -1053,30 +1149,6 @@ function gradeOutcome(mutations, evalDef, workflowSpans) {
 // src/loader.ts
 import { readdirSync, existsSync } from "fs";
 import { resolve } from "path";
-
-// src/definition.ts
-var Eval = class {
-  name;
-  description;
-  tags;
-  type;
-  setup;
-  conversation;
-  outcome;
-  options;
-  constructor(def) {
-    this.name = def.name;
-    this.conversation = def.conversation;
-    if (def.description !== void 0) this.description = def.description;
-    if (def.tags !== void 0) this.tags = def.tags;
-    if (def.type !== void 0) this.type = def.type;
-    if (def.setup !== void 0) this.setup = def.setup;
-    if (def.outcome !== void 0) this.outcome = def.outcome;
-    if (def.options !== void 0) this.options = def.options;
-  }
-};
-
-// src/loader.ts
 function isEvalDefinition(value) {
   return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
 }
@@ -1087,15 +1159,17 @@ async function loadEvalFile(filePath) {
     mod = await import(absPath);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
-    throw new Error(
-      `Failed to load eval file ${filePath}: ${msg}
+    throw new EvalRunnerError({
+      code: "EVAL_LOAD_FAILED",
+      message: `Failed to load eval file ${filePath}: ${msg}
 
 Make sure your eval file:
   - Has no syntax or type errors
   - Exports one or more \`new Eval({...})\` instances
   - Has all dependencies installed (\`bun install\`)`,
-      { cause: err }
-    );
+      expected: true,
+      cause: err
+    });
   }
   const results = [];
   for (const [key, value] of Object.entries(mod)) {
@@ -1105,9 +1179,11 @@ Make sure your eval file:
     }
   }
   if (results.length === 0) {
-    throw new Error(
-      `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`
-    );
+    throw new EvalRunnerError({
+      code: "EVAL_FILE_EMPTY",
+      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
+      expected: true
+    });
   }
   return results;
 }
@@ -1125,7 +1201,12 @@ async function loadEvalsFromDir(dirPath) {
   const seen = /* @__PURE__ */ new Set();
   for (const e of evals) {
     if (seen.has(e.name)) {
-      throw new Error(`Duplicate eval name "${e.name}" found in ${dirPath}`);
+      throw new EvalRunnerError({
+        code: "EVAL_DUPLICATE_NAME",
+        message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
+        expected: true,
+        details: { name: e.name }
+      });
     }
     seen.add(e.name);
   }
@@ -1189,7 +1270,10 @@ async function seedEvalState(client, ctx, setup) {
   }
   if (state.conversation) {
     if (!ctx.conversationId) {
-      throw new Error("Cannot seed conversation state before a conversation is created.");
+      throw new EvalRunnerError({
+        code: "EVAL_SEED_NO_CONVERSATION",
+        message: "Cannot seed conversation state before a conversation is created."
+      });
     }
     writes.push(
       client.setState({
@@ -1225,6 +1309,7 @@ function evalMetadata(evalDef) {
 async function runEval(evalDef, connection, options = {}) {
   const devServerUrl = options.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = options.devServerHeaders ?? {};
+  const logger = options.logger ?? defaultLogger;
   const idleTimeout = evalDef.options?.idleTimeout ?? options.idleTimeout ?? DEFAULT_IDLE_TIMEOUT;
   const judgePassThreshold = evalDef.options?.judgePassThreshold ?? options.judgePassThreshold;
   const start = Date.now();
@@ -1272,19 +1357,28 @@ async function runEval(evalDef, connection, options = {}) {
     if (conversationId) {
       await collector.connect({ conversationId });
     } else if (evalDef.conversation.length > 0) {
-      throw new Error("Cannot run conversation eval without a conversationId \u2014 no chat session or setup created one.");
+      throw new EvalRunnerError({
+        code: "EVAL_NO_CONVERSATION_ID",
+        message: "Cannot run conversation eval without a conversationId \u2014 no chat session or setup created one."
+      });
     }
     for (let i = 0; i < evalDef.conversation.length; i++) {
       if (options.signal?.aborted) break;
       const turn = evalDef.conversation[i];
+      const turnConfigError = (message) => new EvalRunnerError({
+        code: "EVAL_TURN_CONFIG_INVALID",
+        message: `Turn ${i + 1}: ${message}`,
+        expected: true,
+        details: { turn: i + 1 }
+      });
       if (turn.expectSilence && turn.assert?.response) {
-        throw new Error(`Turn ${i + 1}: 'expectSilence' and 'assert.response' are mutually exclusive.`);
+        throw turnConfigError(`'expectSilence' and 'assert.response' are mutually exclusive.`);
       }
       if (turn.user && turn.event) {
-        throw new Error(`Turn ${i + 1}: 'user' and 'event' are mutually exclusive.`);
+        throw turnConfigError(`'user' and 'event' are mutually exclusive.`);
       }
       if (!turn.user && !turn.event) {
-        throw new Error(`Turn ${i + 1}: must have either 'user' or 'event'.`);
+        throw turnConfigError(`must have either 'user' or 'event'.`);
       }
       const turnLabelForEvent = turn.event ? `[event: ${turn.event.type}]` : turn.user;
       options.onProgress?.({
@@ -1328,11 +1422,17 @@ async function runEval(evalDef, connection, options = {}) {
       const botResponse = turnData.messages.join("\n");
       const botDuration = turnData.handlerDuration;
       const turnLabel = turn.event ? `[event: ${turn.event.type}]` : turn.user;
-      if (!turn.expectSilence && turnData.messages.length === 0) {
-        throw new Error(`Turn ${i + 1}: bot produced no response.`);
-      }
       const evalStart = Date.now();
       let assertions = [];
+      const noResponse = !turn.expectSilence && turnData.messages.length === 0;
+      if (noResponse) {
+        assertions.push({
+          assertion: "response",
+          pass: false,
+          expected: "Bot produces a response",
+          actual: `Turn ${i + 1}: bot produced no response.`
+        });
+      }
       if (turn.expectSilence) {
         const wasSilent = turnData.messages.length === 0;
         assertions.push({
@@ -1342,7 +1442,7 @@ async function runEval(evalDef, connection, options = {}) {
           actual: wasSilent ? "No response" : `Bot responded: "${botResponse}"`
         });
       }
-      if (turn.assert?.response) {
+      if (turn.assert?.response && !noResponse) {
         assertions = await gradeResponse(botResponse, turn.assert.response, {
           userMessage: turnLabel,
           ...judgePassThreshold !== void 0 ? { judgePassThreshold } : {}
@@ -1434,13 +1534,19 @@ async function runEval(evalDef, connection, options = {}) {
       ...aborted ? { error: "Eval aborted" } : {}
     };
   } catch (err) {
+    const adkErr = isAdkError(err) ? err : void 0;
+    if (!adkErr || !adkErr.expected) {
+      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${err.message}`);
+      options.onException?.(err, { source: "evals", eval_name: evalDef.name });
+    }
     return {
       ...evalMetadata(evalDef),
       turns,
       outcomeAssertions,
       pass: false,
       duration: Date.now() - start,
-      error: err.message
+      error: err.message,
+      ...adkErr ? { errorCode: adkErr.code } : {}
     };
   } finally {
     collector.disconnect();
@@ -1488,7 +1594,7 @@ async function runEvalSuite(config, filter) {
   };
   const devServerUrl = config.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = config.devServerHeaders ?? {};
-  await assertChatChannelBound(devServerUrl, devServerHeaders);
+  await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
   const reports = [];
   config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
   for (let i = 0; i < evals.length; i++) {
@@ -1508,7 +1614,9 @@ async function runEvalSuite(config, filter) {
       ...config.evalOptions?.judgePassThreshold !== void 0 ? { judgePassThreshold: config.evalOptions.judgePassThreshold } : {},
       ...config.onProgress !== void 0 ? { onProgress: config.onProgress } : {},
       ...config.signal !== void 0 ? { signal: config.signal } : {},
-      ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {}
+      ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {},
+      ...config.logger !== void 0 ? { logger: config.logger } : {},
+      ...config.onException !== void 0 ? { onException: config.onException } : {}
     });
     reports.push(report);
     config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });