@botpress/evals 1.19.0-beta.7 → 1.19.0-beta.9

package/dist/client.d.ts

@@ -5,7 +5,7 @@
  * This is a send-only client. Observation (bot responses, tool calls, state)
  * is handled by the SSECollector via trace spans — not via WebSocket listeners.
  */
-import type { ChatClient } from './types';
+import type { ChatClient, EvalLogger } from './types';
 export declare function setDefaultChatClient(chatClient: ChatClient | null): void;
 /**
  * A send-only chat session that maintains a single client connection across turns.
@@ -18,6 +18,8 @@ export declare class ChatSession {
     private conversationId;
     constructor(webhookId: string, chatClient?: ChatClient);
     connect(): Promise<void>;
+    /** Invariant guard — calling any session method before connect() is a runner bug. */
+    private assertConnected;
     get userId(): string;
     ensureConversation(): Promise<string>;
     /**
@@ -46,5 +48,5 @@ export declare function discoverWebhookId(botId: string, token: string, apiUrl:
  * Uses the dev server's `/api/agent` endpoint, so this is a no-op against
  * production bots (no devServerUrl).
  */
-export declare function assertChatChannelBound(devServerUrl: string, headers?: Record<string, string>): Promise<void>;
+export declare function assertChatChannelBound(devServerUrl: string, headers?: Record<string, string>, logger?: EvalLogger): Promise<void>;
 //# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAwB,MAAM,SAAS,CAAA;AAI/D,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAExE;AAcD;;;GAGG;AACH,qBAAa,WAAW;IAKpB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,UAAU,CAAC;IALrB,OAAO,CAAC,MAAM,CAAoC;IAClD,OAAO,CAAC,cAAc,CAAsB;gBAGlC,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,UAAU;IAG3B,OAAO;IAKb,IAAI,MAAM,IAAI,MAAM,CAKnB;IAEK,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC;IAc3C;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASjD;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAQ/E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAqBrG;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GACnC,OAAO,CAAC,IAAI,CAAC,CAsCf"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAwB,UAAU,EAAE,MAAM,SAAS,CAAA;AAM3E,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAExE;AAiBD;;;GAGG;AACH,qBAAa,WAAW;IAKpB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,UAAU,CAAC;IALrB,OAAO,CAAC,MAAM,CAAoC;IAClD,OAAO,CAAC,cAAc,CAAsB;gBAGlC,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,UAAU;IAG3B,OAAO;IAKb,qFAAqF;IACrF,OAAO,CAAC,eAAe;IAUvB,IAAI,MAAM,IAAI,MAAM,CAEnB;IAEK,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC;IAY3C;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjD;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAK/E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBrG;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,EACpC,MAAM,GAAE,UAA0B,GACjC,OAAO,CAAC,IAAI,CAAC,CA8Cf"}

package/dist/client.js

@@ -1,6 +1,54 @@
 // src/client.ts
 import { Client as BpClient } from "@botpress/client";
 import { createRequire } from "module";
+
+// src/types.ts
+var defaultLogger = console;
+
+// ../analytics/src/errors.ts
+var AdkError = class extends Error {
+  /**
+   * Marker for cross-module-instance checks — `instanceof` breaks when a
+   * package is duplicated in the dependency graph.
+   * @internal
+   */
+  static __IS_ADK_BASE_ERROR = true;
+  /**
+   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
+   * pin `Code` to a literal union (`class EvalRunnerError extends
+   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
+   * in switches.
+   */
+  code;
+  /**
+   * True for user/environment conditions the developer can fix themselves
+   * (validation, auth, connectivity). False for internal bugs. Defaults to
+   * false so unclassified errors surface as bugs until someone triages them.
+   */
+  expected;
+  /** Structured context. Sanitized before any telemetry leaves the machine. */
+  details;
+  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
+  suggestion;
+  constructor(opts) {
+    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
+    this.name = this.constructor.name;
+    this.code = opts.code;
+    this.expected = opts.expected ?? false;
+    if (opts.details !== void 0) {
+      this.details = opts.details;
+    }
+    if (opts.suggestion !== void 0) {
+      this.suggestion = opts.suggestion;
+    }
+  }
+};
+
+// src/errors.ts
+var EvalRunnerError = class extends AdkError {
+};
+
+// src/client.ts
 var defaultChatClient = null;
 function setDefaultChatClient(chatClient) {
   defaultChatClient = chatClient;
@@ -11,7 +59,10 @@ function getDefaultChatClient() {
   const mod = require2("@botpress/chat");
   const ChatClient = mod.default?.Client ?? mod.Client;
   if (!ChatClient) {
-    throw new Error("@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.");
+    throw new EvalRunnerError({
+      code: "CHAT_CLIENT_MISSING",
+      message: "@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly."
+    });
   }
   return ChatClient;
 }
@@ -26,18 +77,23 @@ var ChatSession = class {
     const ChatClient = this.chatClient ?? getDefaultChatClient();
     this.client = await ChatClient.connect({ webhookId: this.webhookId });
   }
-  get userId() {
+  /** Invariant guard — calling any session method before connect() is a runner bug. */
+  assertConnected() {
     if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
+      throw new EvalRunnerError({
+        code: "CHAT_NOT_CONNECTED",
+        message: "ChatSession not connected. Call connect() first."
+      });
     }
-    return this.client.user.id;
+    return this.client;
+  }
+  get userId() {
+    return this.assertConnected().user.id;
   }
   async ensureConversation() {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     if (!this.conversationId) {
-      const conv = await this.client.createConversation({});
+      const conv = await client.createConversation({});
       this.conversationId = conv.conversation.id;
       return conv.conversation.id;
     }
@@ -47,21 +103,17 @@ var ChatSession = class {
    * Send a user message. Fire-and-forget — observation is via SSECollector.
    */
   async sendMessage(message) {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await this.client.createMessage({ conversationId, payload: { type: "text", text: message } });
+    await client.createMessage({ conversationId, payload: { type: "text", text: message } });
   }
   /**
    * Send an event. Fire-and-forget — observation is via SSECollector.
    */
   async sendEvent(type, payload) {
-    if (!this.client) {
-      throw new Error("ChatSession not connected. Call connect() first.");
-    }
+    const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await this.client.createEvent({ type, payload, conversationId });
+    await client.createEvent({ type, payload, conversationId });
   }
 };
 async function discoverWebhookId(botId, token, apiUrl) {
@@ -71,26 +123,31 @@ async function discoverWebhookId(botId, token, apiUrl) {
   const chat = Object.values(integrations).find((int) => int.name === "chat");
   const webhookId = chat?.webhookId;
   if (!webhookId || typeof webhookId !== "string") {
-    throw new Error(
-      [
+    throw new EvalRunnerError({
+      code: "CHAT_INTEGRATION_MISSING",
+      message: [
         "The `chat` integration is not installed on this bot \u2014 evals require it.",
         "",
         "To fix:",
         "  1. Install it:    adk add chat",
         "  2. Redeploy:      adk dev   (or: adk deploy)"
-      ].join("\n")
-    );
+      ].join("\n"),
+      expected: true
+    });
   }
   return webhookId;
 }
-async function assertChatChannelBound(devServerUrl, headers = {}) {
+async function assertChatChannelBound(devServerUrl, headers = {}, logger = defaultLogger) {
   let conversations = [];
   try {
     const res = await fetch(`${devServerUrl}/api/agent`, { headers });
     if (!res.ok) return;
     const agent = await res.json();
     conversations = agent.conversations ?? [];
-  } catch {
+  } catch (err) {
+    logger?.warn(
+      `Skipping chat.channel binding pre-flight \u2014 could not query ${devServerUrl}/api/agent: ${err instanceof Error ? err.message : String(err)}`
+    );
     return;
   }
   if (conversations.length === 0) return;
@@ -101,8 +158,9 @@ async function assertChatChannelBound(devServerUrl, headers = {}) {
   };
   const bound = conversations.some((c) => matchesChatChannel(c.channel));
   if (bound) return;
-  throw new Error(
-    [
+  throw new EvalRunnerError({
+    code: "CHAT_CHANNEL_UNBOUND",
+    message: [
       "No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.",
       "",
       "To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):",
@@ -111,8 +169,9 @@ async function assertChatChannelBound(devServerUrl, headers = {}) {
       "  channel: ['chat.channel', ...],    // array form",
       "",
       "The `chat` integration is how evals and `adk chat --single` send messages programmatically."
-    ].join("\n")
-  );
+    ].join("\n"),
+    expected: true
+  });
 }
 export {
   ChatSession,

package/dist/client.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../src/client.ts"],
-  "sourcesContent": ["/**\n * Chat client for eval conversations.\n * Drives conversations against a running ADK bot via @botpress/chat.\n *\n * This is a send-only client. Observation (bot responses, tool calls, state)\n * is handled by the SSECollector via trace spans \u2014 not via WebSocket listeners.\n */\n\nimport { Client as BpClient } from '@botpress/client'\nimport { createRequire } from 'module'\nimport type { ChatClient, ChatClientConnection } from './types'\n\nlet defaultChatClient: ChatClient | null = null\n\nexport function setDefaultChatClient(chatClient: ChatClient | null): void {\n  defaultChatClient = chatClient\n}\n\nfunction getDefaultChatClient(): ChatClient {\n  if (defaultChatClient) return defaultChatClient\n\n  const require = createRequire(import.meta.url)\n  const mod = require('@botpress/chat') as { Client?: ChatClient; default?: { Client?: ChatClient } }\n  const ChatClient = mod.default?.Client ?? mod.Client\n  if (!ChatClient) {\n    throw new Error('@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.')\n  }\n  return ChatClient\n}\n\n/**\n * A send-only chat session that maintains a single client connection across turns.\n * Messages and events are fire-and-forget; the SSECollector observes results via traces.\n */\nexport class ChatSession {\n  private client: ChatClientConnection | null = null\n  private conversationId: string | null = null\n\n  constructor(\n    private webhookId: string,\n    private chatClient?: ChatClient\n  ) {}\n\n  async connect() {\n    const ChatClient = this.chatClient ?? getDefaultChatClient()\n    this.client = await ChatClient.connect({ webhookId: this.webhookId })\n  }\n\n  get userId(): string {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n    return this.client.user.id\n  }\n\n  async ensureConversation(): Promise<string> {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n\n    if (!this.conversationId) {\n      const conv = await this.client.createConversation({})\n      this.conversationId = conv.conversation.id\n      return conv.conversation.id\n    }\n\n    return this.conversationId\n  }\n\n  /**\n   * Send a user message. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendMessage(message: string): Promise<void> {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n\n    const conversationId = await this.ensureConversation()\n    await this.client.createMessage({ conversationId, payload: { type: 'text', text: message } })\n  }\n\n  /**\n   * Send an event. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendEvent(type: string, payload: Record<string, unknown>): Promise<void> {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n\n    const conversationId = await this.ensureConversation()\n    await this.client.createEvent({ type, payload, conversationId })\n  }\n}\n\n/**\n * Discover the chat integration's webhookId from a bot.\n *\n * Evals talk to the bot through the `chat` integration (via @botpress/chat).\n * If the bot doesn't have that integration installed, evals can't run.\n */\nexport async function discoverWebhookId(botId: string, token: string, apiUrl: string): Promise<string> {\n  const client = new BpClient({ token, botId, apiUrl })\n  const { bot } = await client.getBot({ id: botId })\n  const integrations = bot.integrations || {}\n\n  const chat = Object.values(integrations).find((int) => (int as Record<string, unknown>).name === 'chat')\n  const webhookId = (chat as Record<string, unknown> | undefined)?.webhookId\n\n  if (!webhookId || typeof webhookId !== 'string') {\n    throw new Error(\n      [\n        'The `chat` integration is not installed on this bot \u2014 evals require it.',\n        '',\n        'To fix:',\n        '  1. Install it:    adk add chat',\n        '  2. Redeploy:      adk dev   (or: adk deploy)',\n      ].join('\\n')\n    )\n  }\n\n  return webhookId\n}\n\n/**\n * Verify the bot has at least one Conversation bound to `chat.channel`.\n *\n * The `chat` integration can be installed without any handler listening on its\n * channel \u2014 in that case messages sent via @botpress/chat produce no response\n * and the caller times out silently. Fail fast with actionable guidance instead.\n *\n * Uses the dev server's `/api/agent` endpoint, so this is a no-op against\n * production bots (no devServerUrl).\n */\nexport async function assertChatChannelBound(\n  devServerUrl: string,\n  headers: Record<string, string> = {}\n): Promise<void> {\n  type ConversationSummary = { channel?: string | string[] }\n  let conversations: ConversationSummary[] = []\n  try {\n    const res = await fetch(`${devServerUrl}/api/agent`, { headers })\n    if (!res.ok) return // Dev server unreachable \u2014 skip, let downstream errors surface\n    const agent = (await res.json()) as { conversations?: ConversationSummary[] }\n    conversations = agent.conversations ?? []\n  } catch {\n    return // Network error \u2014 skip check\n  }\n\n  if (conversations.length === 0) return // Bot has no conversations at all \u2014 different problem\n\n  // Mirrors the runtime's matchesChannel semantics: a handler matches `chat.channel`\n  // if it is the wildcard '*', the literal string 'chat.channel', or an array\n  // containing either.\n  const matchesChatChannel = (ch: string | string[] | undefined): boolean => {\n    if (!ch) return false\n    if (ch === '*' || ch === 'chat.channel') return true\n    return Array.isArray(ch) && (ch.includes('chat.channel') || ch.includes('*'))\n  }\n\n  const bound = conversations.some((c) => matchesChatChannel(c.channel))\n  if (bound) return\n\n  throw new Error(\n    [\n      'No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.',\n      '',\n      'To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):',\n      \"  channel: 'chat.channel',           // explicit\",\n      \"  channel: '*',                      // wildcard \u2014 matches every channel\",\n      \"  channel: ['chat.channel', ...],    // array form\",\n      '',\n      'The `chat` integration is how evals and `adk chat --single` send messages programmatically.',\n    ].join('\\n')\n  )\n}\n"],
-  "mappings": ";AAQA,SAAS,UAAU,gBAAgB;AACnC,SAAS,qBAAqB;AAG9B,IAAI,oBAAuC;AAEpC,SAAS,qBAAqB,YAAqC;AACxE,sBAAoB;AACtB;AAEA,SAAS,uBAAmC;AAC1C,MAAI,kBAAmB,QAAO;AAE9B,QAAMA,WAAU,cAAc,YAAY,GAAG;AAC7C,QAAM,MAAMA,SAAQ,gBAAgB;AACpC,QAAM,aAAa,IAAI,SAAS,UAAU,IAAI;AAC9C,MAAI,CAAC,YAAY;AACf,UAAM,IAAI,MAAM,6FAA6F;AAAA,EAC/G;AACA,SAAO;AACT;AAMO,IAAM,cAAN,MAAkB;AAAA,EAIvB,YACU,WACA,YACR;AAFQ;AACA;AAAA,EACP;AAAA,EANK,SAAsC;AAAA,EACtC,iBAAgC;AAAA,EAOxC,MAAM,UAAU;AACd,UAAM,aAAa,KAAK,cAAc,qBAAqB;AAC3D,SAAK,SAAS,MAAM,WAAW,QAAQ,EAAE,WAAW,KAAK,UAAU,CAAC;AAAA,EACtE;AAAA,EAEA,IAAI,SAAiB;AACnB,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AACA,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,qBAAsC;AAC1C,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,OAAO,MAAM,KAAK,OAAO,mBAAmB,CAAC,CAAC;AACpD,WAAK,iBAAiB,KAAK,aAAa;AACxC,aAAO,KAAK,aAAa;AAAA,IAC3B;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,YAAY,SAAgC;AAChD,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AAEA,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,KAAK,OAAO,cAAc,EAAE,gBAAgB,SAAS,EAAE,MAAM,QAAQ,MAAM,QAAQ,EAAE,CAAC;AAAA,EAC9F;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,UAAU,MAAc,SAAiD;AAC7E,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AAEA,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,KAAK,OAAO,YAAY,EAAE,MAAM,SAAS,eAAe,CAAC;AAAA,EACjE;AACF;AAQA,eAAsB,kBAAkB,OAAe,OAAe,QAAiC;AACrG,QAAM,SAAS,IAAI,SAAS,EAAE,OAAO,OAAO,OAAO,CAAC;AACpD,QAAM,EAAE,IAAI,IAAI,MAAM,OAAO,OAAO,EAAE,IAAI,MAAM,CAAC;AACjD,QAAM,eAAe,IAAI,gBAAgB,CAAC;AAE1C,QAAM,OAAO,OAAO,OAAO,YAAY,EAAE,KAAK,CAAC,QAAS,IAAgC,SAAS,MAAM;AACvG,QAAM,YAAa,MAA8C;AAEjE,MAAI,CAAC,aAAa,OAAO,cAAc,UAAU;AAC/C,UAAM,IAAI;AAAA,MACR;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,IACb;AAAA,EACF;AAEA,SAAO;AACT;AAYA,eAAsB,uBACpB,cACA,UAAkC,CAAC,GACpB;AAEf,MAAI,gBAAuC,CAAC;AAC5C,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,YAAY,cAAc,EAAE,QAAQ,CAAC;AAChE,QAAI,CAAC,IAAI,GAAI;AACb,UAAM,QAAS,MAAM,IAAI,KAAK;AAC9B,oBAAgB,MAAM,iBAAiB,CAAC;AAAA,EAC1C,QAAQ;AACN;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,EAAG;AAKhC,QAAM,qBAAqB,CAAC,OAA+C;AACzE,QAAI,CAAC,GAAI,QAAO;AAChB,QAAI,OAAO,OAAO,OAAO,eAAgB,QAAO;AAChD,WAAO,MAAM,QAAQ,EAAE,MAAM,GAAG,SAAS,cAAc,KAAK,GAAG,SAAS,GAAG;AAAA,EAC7E;AAEA,QAAM,QAAQ,cAAc,KAAK,CAAC,MAAM,mBAAmB,EAAE,OAAO,CAAC;AACrE,MAAI,MAAO;AAEX,QAAM,IAAI;AAAA,IACR;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,EACb;AACF;",
+  "sources": ["../src/client.ts", "../src/types.ts", "../../analytics/src/errors.ts", "../src/errors.ts"],
+  "sourcesContent": ["/**\n * Chat client for eval conversations.\n * Drives conversations against a running ADK bot via @botpress/chat.\n *\n * This is a send-only client. Observation (bot responses, tool calls, state)\n * is handled by the SSECollector via trace spans \u2014 not via WebSocket listeners.\n */\n\nimport { Client as BpClient } from '@botpress/client'\nimport { createRequire } from 'module'\nimport type { ChatClient, ChatClientConnection, EvalLogger } from './types'\nimport { defaultLogger } from './types'\nimport { EvalRunnerError } from './errors'\n\nlet defaultChatClient: ChatClient | null = null\n\nexport function setDefaultChatClient(chatClient: ChatClient | null): void {\n  defaultChatClient = chatClient\n}\n\nfunction getDefaultChatClient(): ChatClient {\n  if (defaultChatClient) return defaultChatClient\n\n  const require = createRequire(import.meta.url)\n  const mod = require('@botpress/chat') as { Client?: ChatClient; default?: { Client?: ChatClient } }\n  const ChatClient = mod.default?.Client ?? mod.Client\n  if (!ChatClient) {\n    throw new EvalRunnerError({\n      code: 'CHAT_CLIENT_MISSING',\n      message: '@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.',\n    })\n  }\n  return ChatClient\n}\n\n/**\n * A send-only chat session that maintains a single client connection across turns.\n * Messages and events are fire-and-forget; the SSECollector observes results via traces.\n */\nexport class ChatSession {\n  private client: ChatClientConnection | null = null\n  private conversationId: string | null = null\n\n  constructor(\n    private webhookId: string,\n    private chatClient?: ChatClient\n  ) {}\n\n  async connect() {\n    const ChatClient = this.chatClient ?? getDefaultChatClient()\n    this.client = await ChatClient.connect({ webhookId: this.webhookId })\n  }\n\n  /** Invariant guard \u2014 calling any session method before connect() is a runner bug. */\n  private assertConnected(): ChatClientConnection {\n    if (!this.client) {\n      throw new EvalRunnerError({\n        code: 'CHAT_NOT_CONNECTED',\n        message: 'ChatSession not connected. Call connect() first.',\n      })\n    }\n    return this.client\n  }\n\n  get userId(): string {\n    return this.assertConnected().user.id\n  }\n\n  async ensureConversation(): Promise<string> {\n    const client = this.assertConnected()\n\n    if (!this.conversationId) {\n      const conv = await client.createConversation({})\n      this.conversationId = conv.conversation.id\n      return conv.conversation.id\n    }\n\n    return this.conversationId\n  }\n\n  /**\n   * Send a user message. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendMessage(message: string): Promise<void> {\n    const client = this.assertConnected()\n    const conversationId = await this.ensureConversation()\n    await client.createMessage({ conversationId, payload: { type: 'text', text: message } })\n  }\n\n  /**\n   * Send an event. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendEvent(type: string, payload: Record<string, unknown>): Promise<void> {\n    const client = this.assertConnected()\n    const conversationId = await this.ensureConversation()\n    await client.createEvent({ type, payload, conversationId })\n  }\n}\n\n/**\n * Discover the chat integration's webhookId from a bot.\n *\n * Evals talk to the bot through the `chat` integration (via @botpress/chat).\n * If the bot doesn't have that integration installed, evals can't run.\n */\nexport async function discoverWebhookId(botId: string, token: string, apiUrl: string): Promise<string> {\n  const client = new BpClient({ token, botId, apiUrl })\n  const { bot } = await client.getBot({ id: botId })\n  const integrations = bot.integrations || {}\n\n  const chat = Object.values(integrations).find((int) => (int as Record<string, unknown>).name === 'chat')\n  const webhookId = (chat as Record<string, unknown> | undefined)?.webhookId\n\n  if (!webhookId || typeof webhookId !== 'string') {\n    throw new EvalRunnerError({\n      code: 'CHAT_INTEGRATION_MISSING',\n      message: [\n        'The `chat` integration is not installed on this bot \u2014 evals require it.',\n        '',\n        'To fix:',\n        '  1. Install it:    adk add chat',\n        '  2. Redeploy:      adk dev   (or: adk deploy)',\n      ].join('\\n'),\n      expected: true,\n    })\n  }\n\n  return webhookId\n}\n\n/**\n * Verify the bot has at least one Conversation bound to `chat.channel`.\n *\n * The `chat` integration can be installed without any handler listening on its\n * channel \u2014 in that case messages sent via @botpress/chat produce no response\n * and the caller times out silently. Fail fast with actionable guidance instead.\n *\n * Uses the dev server's `/api/agent` endpoint, so this is a no-op against\n * production bots (no devServerUrl).\n */\nexport async function assertChatChannelBound(\n  devServerUrl: string,\n  headers: Record<string, string> = {},\n  logger: EvalLogger = defaultLogger\n): Promise<void> {\n  type ConversationSummary = { channel?: string | string[] }\n  let conversations: ConversationSummary[] = []\n  try {\n    const res = await fetch(`${devServerUrl}/api/agent`, { headers })\n    if (!res.ok) return // Dev server unreachable \u2014 skip, let downstream errors surface\n    const agent = (await res.json()) as { conversations?: ConversationSummary[] }\n    conversations = agent.conversations ?? []\n  } catch (err) {\n    // Soft pre-flight only \u2014 a missing dev server is normal (e.g. prod bots),\n    // but log the skip so a *broken* dev server (malformed /api/agent JSON)\n    // isn't invisible while every eval afterwards times out mysteriously.\n    logger?.warn(\n      `Skipping chat.channel binding pre-flight \u2014 could not query ${devServerUrl}/api/agent: ${err instanceof Error ? err.message : String(err)}`\n    )\n    return\n  }\n\n  if (conversations.length === 0) return // Bot has no conversations at all \u2014 different problem\n\n  // Mirrors the runtime's matchesChannel semantics: a handler matches `chat.channel`\n  // if it is the wildcard '*', the literal string 'chat.channel', or an array\n  // containing either.\n  const matchesChatChannel = (ch: string | string[] | undefined): boolean => {\n    if (!ch) return false\n    if (ch === '*' || ch === 'chat.channel') return true\n    return Array.isArray(ch) && (ch.includes('chat.channel') || ch.includes('*'))\n  }\n\n  const bound = conversations.some((c) => matchesChatChannel(c.channel))\n  if (bound) return\n\n  throw new EvalRunnerError({\n    code: 'CHAT_CHANNEL_UNBOUND',\n    message: [\n      'No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.',\n      '',\n      'To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):',\n      \"  channel: 'chat.channel',           // explicit\",\n      \"  channel: '*',                      // wildcard \u2014 matches every channel\",\n      \"  channel: ['chat.channel', ...],    // array form\",\n      '',\n      'The `chat` integration is how evals and `adk chat --single` send messages programmatically.',\n    ].join('\\n'),\n    expected: true,\n  })\n}\n", "/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './trace'\n\nexport interface ChatClientConnection {\n  user: { id: string }\n  createConversation: (props: Record<string, unknown>) => Promise<{ conversation: { id: string } }>\n  // Broad parameter types keep this compatible with @botpress/chat's narrower\n  // generated payload unions while avoiding a dependency on those definitions.\n  createMessage: (props: any) => Promise<unknown>\n  createEvent: (props: any) => Promise<unknown>\n}\n\nexport interface ChatClient {\n  connect: (props: { webhookId: string }) => Promise<ChatClientConnection>\n}\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnNumber: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  webhookId: string\n  botId: string\n  token: string\n  apiUrl: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  credentials: {\n    token: string\n    apiUrl: string\n    botId: string\n    workspaceId?: string\n    webhookId?: string\n  }\n  agentPath: string\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n", "/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity). False for internal bugs. Defaults to\n   * false so unclassified errors surface as bugs until someone triages them.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? false\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n"],
+  "mappings": ";AAQA,SAAS,UAAU,gBAAgB;AACnC,SAAS,qBAAqB;;;ACoJvB,IAAM,gBAA4B;;;AC5IlC,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;ACpCO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;AHd9D,IAAI,oBAAuC;AAEpC,SAAS,qBAAqB,YAAqC;AACxE,sBAAoB;AACtB;AAEA,SAAS,uBAAmC;AAC1C,MAAI,kBAAmB,QAAO;AAE9B,QAAMA,WAAU,cAAc,YAAY,GAAG;AAC7C,QAAM,MAAMA,SAAQ,gBAAgB;AACpC,QAAM,aAAa,IAAI,SAAS,UAAU,IAAI;AAC9C,MAAI,CAAC,YAAY;AACf,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS;AAAA,IACX,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAMO,IAAM,cAAN,MAAkB;AAAA,EAIvB,YACU,WACA,YACR;AAFQ;AACA;AAAA,EACP;AAAA,EANK,SAAsC;AAAA,EACtC,iBAAgC;AAAA,EAOxC,MAAM,UAAU;AACd,UAAM,aAAa,KAAK,cAAc,qBAAqB;AAC3D,SAAK,SAAS,MAAM,WAAW,QAAQ,EAAE,WAAW,KAAK,UAAU,CAAC;AAAA,EACtE;AAAA;AAAA,EAGQ,kBAAwC;AAC9C,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,gBAAgB,EAAE,KAAK;AAAA,EACrC;AAAA,EAEA,MAAM,qBAAsC;AAC1C,UAAM,SAAS,KAAK,gBAAgB;AAEpC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,OAAO,MAAM,OAAO,mBAAmB,CAAC,CAAC;AAC/C,WAAK,iBAAiB,KAAK,aAAa;AACxC,aAAO,KAAK,aAAa;AAAA,IAC3B;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,YAAY,SAAgC;AAChD,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,OAAO,cAAc,EAAE,gBAAgB,SAAS,EAAE,MAAM,QAAQ,MAAM,QAAQ,EAAE,CAAC;AAAA,EACzF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,UAAU,MAAc,SAAiD;AAC7E,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,OAAO,YAAY,EAAE,MAAM,SAAS,eAAe,CAAC;AAAA,EAC5D;AACF;AAQA,eAAsB,kBAAkB,OAAe,OAAe,QAAiC;AACrG,QAAM,SAAS,IAAI,SAAS,EAAE,OAAO,OAAO,OAAO,CAAC;AACpD,QAAM,EAAE,IAAI,IAAI,MAAM,OAAO,OAAO,EAAE,IAAI,MAAM,CAAC;AACjD,QAAM,eAAe,IAAI,gBAAgB,CAAC;AAE1C,QAAM,OAAO,OAAO,OAAO,YAAY,EAAE,KAAK,CAAC,QAAS,IAAgC,SAAS,MAAM;AACvG,QAAM,YAAa,MAA8C;AAEjE,MAAI,CAAC,aAAa,OAAO,cAAc,UAAU;AAC/C,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,MACX,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAYA,eAAsB,uBACpB,cACA,UAAkC,CAAC,GACnC,SAAqB,eACN;AAEf,MAAI,gBAAuC,CAAC;AAC5C,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,YAAY,cAAc,EAAE,QAAQ,CAAC;AAChE,QAAI,CAAC,IAAI,GAAI;AACb,UAAM,QAAS,MAAM,IAAI,KAAK;AAC9B,oBAAgB,MAAM,iBAAiB,CAAC;AAAA,EAC1C,SAAS,KAAK;AAIZ,YAAQ;AAAA,MACN,mEAA8D,YAAY,eAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IAC3I;AACA;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,EAAG;AAKhC,QAAM,qBAAqB,CAAC,OAA+C;AACzE,QAAI,CAAC,GAAI,QAAO;AAChB,QAAI,OAAO,OAAO,OAAO,eAAgB,QAAO;AAChD,WAAO,MAAM,QAAQ,EAAE,MAAM,GAAG,SAAS,cAAc,KAAK,GAAG,SAAS,GAAG;AAAA,EAC7E;AAEA,QAAM,QAAQ,cAAc,KAAK,CAAC,MAAM,mBAAmB,EAAE,OAAO,CAAC;AACrE,MAAI,MAAO;AAEX,QAAM,IAAI,gBAAgB;AAAA,IACxB,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ,CAAC;AACH;",
   "names": ["require"]
 }

package/dist/errors.d.ts

@@ -0,0 +1,15 @@
+import { AdkError } from '@botpress/analytics';
+/**
+ * Stable error codes for the eval engine. Typed as a literal union so a typo'd
+ * code is a compile error and PostHog issue-slicing stays consistent.
+ * Conventions per docs/ERROR-HANDLING.md.
+ */
+export declare const EVAL_ERROR_CODES: readonly ["EVAL_LOAD_FAILED", "EVAL_FILE_EMPTY", "EVAL_DUPLICATE_NAME", "EVAL_SEED_NO_CONVERSATION", "EVAL_NO_CONVERSATION_ID", "EVAL_TURN_CONFIG_INVALID", "SSE_CONNECT_FAILED", "SSE_NO_BODY", "CHAT_CLIENT_MISSING", "CHAT_NOT_CONNECTED", "CHAT_INTEGRATION_MISSING", "CHAT_CHANNEL_UNBOUND"];
+export type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number];
+/**
+ * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to
+ * avoid shadowing the JS built-in `EvalError`.
+ */
+export declare class EvalRunnerError extends AdkError<EvalErrorCode> {
+}
+//# sourceMappingURL=errors.d.ts.map

package/dist/errors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAE9C;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,mSAanB,CAAA;AAEV,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAA;AAE7D;;;GAGG;AACH,qBAAa,eAAgB,SAAQ,QAAQ,CAAC,aAAa,CAAC;CAAG"}

package/dist/graders/index.js

@@ -45,6 +45,11 @@ function operatorToString(operator) {
 // src/graders/llm.ts
 import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
+
+// src/types.ts
+var defaultLogger = console;
+
+// src/graders/llm.ts
 var DEFAULT_PASS_THRESHOLD = 3;
 function buildJudgeSystemPrompt(passThreshold) {
   return `You are an evaluation judge for a chatbot. You will be given:
@@ -74,13 +79,15 @@ var MODEL_ALIASES = ["fast", "best"];
 var _cognitive = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
 async function initLLMJudge(credentials, options) {
-  const log = options?.logger ?? console;
+  const log = options?.logger ?? defaultLogger;
+  _log = log;
   const client = new Client({
     token: credentials.token,
     apiUrl: credentials.apiUrl,
@@ -105,11 +112,12 @@ async function gradeLLMJudge(botResponse, criteria, context) {
   const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
   try {
     if (!_cognitive) {
+      _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+        actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
@@ -139,11 +147,12 @@ Criteria: ${criteria}`
     const rawContent = output?.choices?.[0]?.content;
     const content = typeof rawContent === "string" ? rawContent : null;
     if (!content) {
+      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge returned empty response"
+        actual: "FAILED \u2014 LLM judge returned empty response"
       };
     }
     const verdict = JSON.parse(content);
@@ -154,11 +163,12 @@ Criteria: ${criteria}`
       actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
     };
   } catch (err) {
+    _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
       assertion: `llm_judge: "${criteria}"`,
-      pass: true,
+      pass: false,
       expected: criteria,
-      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+      actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
   }
 }

package/dist/graders/index.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../../src/graders/match.ts", "../../src/graders/llm.ts", "../../src/graders/response.ts", "../../src/graders/tools.ts", "../../src/graders/state.ts", "../../src/graders/workflow.ts", "../../src/graders/timing.ts", "../../src/graders/outcome.ts"],
-  "sourcesContent": ["/**\n * Shared match utilities for assertion grading.\n * Used by tool, state, and table graders.\n */\n\nimport type { MatchOperator } from '../types'\n\n/**\n * Evaluate a single match operator against a value.\n */\nexport function matchValue(operator: MatchOperator, actual: unknown): boolean {\n  // Shorthand: string means exact match\n  if (typeof operator === 'string') {\n    return String(actual) === operator\n  }\n\n  if ('equals' in operator) {\n    return actual === operator.equals\n  }\n\n  if ('contains' in operator) {\n    return String(actual).toLowerCase().includes(operator.contains.toLowerCase())\n  }\n\n  if ('not_contains' in operator) {\n    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase())\n  }\n\n  if ('matches' in operator) {\n    return new RegExp(operator.matches, 'i').test(String(actual))\n  }\n\n  if ('in' in operator) {\n    return operator.in.includes(actual)\n  }\n\n  if ('exists' in operator) {\n    return operator.exists ? actual !== undefined && actual !== null : actual === undefined || actual === null\n  }\n\n  if ('gte' in operator) {\n    return Number(actual) >= operator.gte\n  }\n\n  if ('lte' in operator) {\n    return Number(actual) <= operator.lte\n  }\n\n  return false\n}\n\nexport function operatorToString(operator: MatchOperator): string {\n  if (typeof operator === 'string') return `equals \"${operator}\"`\n  if ('equals' in operator) return `equals ${JSON.stringify(operator.equals)}`\n  if ('contains' in operator) return `contains \"${operator.contains}\"`\n  if ('not_contains' in operator) return `not_contains \"${operator.not_contains}\"`\n  if ('matches' in operator) return `matches /${operator.matches}/`\n  if ('in' in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(', ')}]`\n  if ('exists' in operator) return operator.exists ? 'exists' : 'does not exist'\n  if ('gte' in operator) return `>= ${operator.gte}`\n  if ('lte' in operator) return `<= ${operator.lte}`\n  return JSON.stringify(operator)\n}\n", "/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? console\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    if (!_cognitive) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: true,\n      expected: criteria,\n      actual: `SKIPPED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n", "/**\n * Response assertion graders.\n * Checks bot response text against expected assertions.\n */\n\nimport type { ResponseAssertion, GraderResult } from '../types'\nimport { gradeLLMJudge } from './llm'\n\nexport async function gradeResponse(\n  botResponse: string,\n  assertions: ResponseAssertion[],\n  context: { userMessage: string; judgePassThreshold?: number }\n): Promise<GraderResult[]> {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    if ('contains' in assertion) {\n      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase())\n      results.push({\n        assertion: `contains \"${assertion.contains}\"`,\n        pass,\n        expected: `Response contains \"${assertion.contains}\"`,\n        actual: pass ? `Found in response` : `Not found in response`,\n      })\n      continue\n    }\n\n    if ('not_contains' in assertion) {\n      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase())\n      results.push({\n        assertion: `not_contains \"${assertion.not_contains}\"`,\n        pass,\n        expected: `Response does not contain \"${assertion.not_contains}\"`,\n        actual: pass ? `Not found in response` : `Found in response`,\n      })\n      continue\n    }\n\n    if ('matches' in assertion) {\n      const regex = new RegExp(assertion.matches, 'i')\n      const pass = regex.test(botResponse)\n      results.push({\n        assertion: `matches ${assertion.matches}`,\n        pass,\n        expected: `Response matches /${assertion.matches}/`,\n        actual: pass ? `Matched` : `No match`,\n      })\n      continue\n    }\n\n    if ('llm_judge' in assertion) {\n      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {\n        userMessage: context.userMessage,\n        ...(context.judgePassThreshold !== undefined ? { passThreshold: context.judgePassThreshold } : {}),\n      })\n      results.push(result)\n      continue\n    }\n\n    // TODO: uncomment this once we support it\n    // if ('similar_to' in assertion) {\n    //   results.push({\n    //     assertion: `similar_to: \"${assertion.similar_to}\"`,\n    //     pass: true,\n    //     expected: assertion.similar_to,\n    //     actual: 'SKIPPED \u2014 similar_to not yet implemented',\n    //   })\n    //   continue\n    // }\n\n    results.push({\n      assertion: 'unknown',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown assertion: ${JSON.stringify(assertion)}`,\n    })\n  }\n\n  return results\n}\n", "/**\n * Tool assertion graders.\n * Checks tool calls from traces against expected assertions.\n */\n\nimport type { ToolAssertion, ToolCall, GraderResult } from '../types'\nimport { matchValue, operatorToString } from './match'\n\nexport function gradeTools(toolCalls: ToolCall[], assertions: ToolAssertion[]): GraderResult[] {\n  return assertions.map((assertion) => {\n    // --- called ---\n    if ('called' in assertion && !('not_called' in assertion) && !('call_order' in assertion)) {\n      const matches = toolCalls.filter((tc) => tc.name === assertion.called)\n      const wasCalled = matches.length > 0\n\n      if (!wasCalled) {\n        return {\n          assertion: `tool called: ${assertion.called}`,\n          pass: false,\n          expected: `${assertion.called} was called`,\n          actual: `Not called. Tools called: [${toolCalls.map((tc) => tc.name).join(', ') || 'none'}]`,\n        }\n      }\n\n      // Check params if specified\n      if (assertion.params) {\n        const paramResults: { key: string; pass: boolean; detail: string }[] = []\n\n        for (const [key, operator] of Object.entries(assertion.params)) {\n          // Check across all matching calls \u2014 pass if any call matches\n          const anyMatch = matches.some((tc) => matchValue(operator, tc.input[key]))\n          paramResults.push({\n            key,\n            pass: anyMatch,\n            detail: anyMatch\n              ? `matched`\n              : `expected ${key} ${operatorToString(operator)}, got ${JSON.stringify(matches.map((tc) => tc.input[key]))}`,\n          })\n        }\n\n        const allParamsPass = paramResults.every((p) => p.pass)\n        const failedParams = paramResults.filter((p) => !p.pass)\n\n        return {\n          assertion: `tool called: ${assertion.called} with params`,\n          pass: allParamsPass,\n          expected: `${assertion.called} called with ${Object.entries(assertion.params)\n            .map(([k, v]) => `${k} ${operatorToString(v)}`)\n            .join(', ')}`,\n          actual: allParamsPass ? `Matched` : failedParams.map((p) => p.detail).join('; '),\n        }\n      }\n\n      return {\n        assertion: `tool called: ${assertion.called}`,\n        pass: true,\n        expected: `${assertion.called} was called`,\n        actual: `Called ${matches.length} time(s)`,\n      }\n    }\n\n    // --- not_called ---\n    if ('not_called' in assertion) {\n      const wasCalled = toolCalls.some((tc) => tc.name === assertion.not_called)\n      return {\n        assertion: `tool not_called: ${assertion.not_called}`,\n        pass: !wasCalled,\n        expected: `${assertion.not_called} was NOT called`,\n        actual: wasCalled ? `Was called` : `Not called`,\n      }\n    }\n\n    // --- call_order ---\n    if ('call_order' in assertion) {\n      const calledNames = toolCalls.map((tc) => tc.name)\n      const expectedOrder = assertion.call_order as string[]\n\n      // Check that the expected sequence appears in order (not necessarily contiguous)\n      let cursor = 0\n      for (const name of calledNames) {\n        if (cursor < expectedOrder.length && name === expectedOrder[cursor]) {\n          cursor++\n        }\n      }\n      const inOrder = cursor === expectedOrder.length\n\n      return {\n        assertion: `call_order: [${expectedOrder.join(' \u2192 ')}]`,\n        pass: inOrder,\n        expected: `Tools called in order: [${expectedOrder.join(' \u2192 ')}]`,\n        actual: `Actual order: [${calledNames.join(' \u2192 ') || 'none'}]`,\n      }\n    }\n\n    return {\n      assertion: 'unknown tool assertion',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown: ${JSON.stringify(assertion)}`,\n    }\n  })\n}\n", "/**\n * State assertion graders.\n * Pure functions that grade state assertions against StateMutation[] from the transformer.\n * No API calls, no retries, no BpClient.\n */\n\nimport type { StateMutation, StateAssertion, GraderResult } from '../types'\n\n/**\n * Parse a state path like \"conversation.lastCity\" into its components.\n *   bot.*            -> type: \"bot\",          field: \"*\"\n *   user.*           -> type: \"user\",         field: \"*\"\n *   conversation.*   -> type: \"conversation\", field: \"*\"\n */\nfunction parseStatePath(path: string): { type: string; field: string } {\n  const dot = path.indexOf('.')\n  if (dot === -1) {\n    throw new Error(`Invalid state path \"${path}\" \u2014 expected \"type.field\" format`)\n  }\n\n  const type = path.slice(0, dot)\n  const field = path.slice(dot + 1)\n\n  if (type !== 'bot' && type !== 'user' && type !== 'conversation') {\n    throw new Error(`Unknown state type \"${type}\" in path \"${path}\" \u2014 expected bot, user, or conversation`)\n  }\n\n  return { type, field }\n}\n\n/**\n * Grade state assertions using StateMutation[] from the transformer.\n *\n * - `equals`: find the LAST mutation matching the state type, extract field from `current`, deep-compare\n * - `changed: true`: check if the field appears in ANY mutation's `changedKeys` for that state type\n * - `changed: false`: check that the field appears in NO mutation's `changedKeys`\n * - Swapped state: when `swappedToFile` is true, current/previous are undefined \u2014 report a clear message\n */\nexport function gradeState(mutations: StateMutation[], assertions: StateAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const parsed = parseStatePath(assertion.path)\n    const matchingMutations = mutations.filter((m) => m.type === parsed.type)\n\n    if (assertion.equals !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual:\n            `No state.save span found for \"${parsed.type}\" \u2014 the bot did not mutate this state during the turn. ` +\n            'If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.',\n        })\n        continue\n      }\n\n      const lastMutation = matchingMutations[matchingMutations.length - 1]!\n\n      if (lastMutation.swappedToFile) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const currentState = lastMutation.current as Record<string, unknown> | undefined\n      const actualValue = currentState ? currentState[parsed.field] : undefined\n      const pass = deepEqual(actualValue, assertion.equals)\n\n      results.push({\n        assertion: `state: ${assertion.path} equals`,\n        pass,\n        expected: JSON.stringify(assertion.equals),\n        actual: JSON.stringify(actualValue),\n      })\n    }\n\n    if (assertion.changed !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: !assertion.changed,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'No state mutation found for type \"' + parsed.type + '\"',\n        })\n        continue\n      }\n\n      // Check for swapped state in any matching mutation\n      const hasSwapped = matchingMutations.some((m) => m.swappedToFile)\n      if (hasSwapped) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: false,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field))\n      const pass = assertion.changed ? fieldChanged : !fieldChanged\n\n      results.push({\n        assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n        pass,\n        expected: assertion.changed\n          ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n          : `Field \"${parsed.field}\" unchanged`,\n        actual: fieldChanged ? `Field \"${parsed.field}\" was changed` : `Field \"${parsed.field}\" was not changed`,\n      })\n    }\n  }\n\n  return results\n}\n\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true\n  if (a === null || b === null) return false\n  if (typeof a !== typeof b) return false\n\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false\n    return a.every((v, i) => deepEqual(v, b[i]))\n  }\n\n  if (Array.isArray(a) !== Array.isArray(b)) return false\n\n  if (typeof a === 'object' && typeof b === 'object') {\n    const keysA = Object.keys(a as object)\n    const keysB = Object.keys(b as object)\n    if (keysA.length !== keysB.length) return false\n    return keysA.every((k) => deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k]))\n  }\n\n  return false\n}\n", "/**\n * Workflow assertion graders.\n * Checks workflow execution via transformer-produced WorkflowSpan data.\n */\n\nimport type { WorkflowAssertion, WorkflowSpan, GraderResult } from '../types'\n\nexport function gradeWorkflows(workflows: WorkflowSpan[], assertions: WorkflowAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const matching = workflows.filter((wf) => wf.name === assertion.name)\n\n    // entered \u2014 was the workflow triggered at all?\n    if (assertion.entered !== undefined) {\n      const wasEntered = matching.length > 0\n      const pass = assertion.entered ? wasEntered : !wasEntered\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.entered ? 'entered' : 'not entered'}`,\n        pass,\n        expected: assertion.entered\n          ? `Workflow \"${assertion.name}\" was entered`\n          : `Workflow \"${assertion.name}\" was not entered`,\n        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`,\n      })\n    }\n\n    // completed \u2014 did the workflow complete?\n    if (assertion.completed !== undefined) {\n      const didComplete = matching.some(\n        (wf) => wf.statusFinal === 'completed' || wf.status === 'ok' || wf.status === 'error'\n      )\n      const pass = assertion.completed ? didComplete : !didComplete\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.completed ? 'completed' : 'not completed'}`,\n        pass,\n        expected: assertion.completed\n          ? `Workflow \"${assertion.name}\" completed`\n          : `Workflow \"${assertion.name}\" did not complete`,\n        actual: didComplete ? `Completed` : `Not completed`,\n      })\n    }\n  }\n\n  return results\n}\n", "/**\n * Timing assertion graders.\n * Checks bot response duration against expected thresholds.\n */\n\nimport type { TimingAssertion, GraderResult } from '../types'\nimport { matchValue, operatorToString } from './match'\n\nexport function gradeTiming(botDuration: number, assertions: TimingAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const pass = matchValue(assertion.response_time, botDuration)\n    const expected = operatorToString(assertion.response_time)\n\n    results.push({\n      assertion: `response_time ${expected}`,\n      pass,\n      expected: `Response time ${expected}`,\n      actual: `${botDuration}ms`,\n    })\n  }\n\n  return results\n}\n", "/**\n * Outcome assertion orchestrator.\n * Runs state and workflow graders after all conversation turns complete.\n */\n\nimport type { EvalDefinition, GraderResult, StateMutation, WorkflowSpan } from '../types'\nimport { gradeState } from './state'\nimport { gradeWorkflows } from './workflow'\n\n/**\n * Grade all outcome assertions after the conversation completes.\n * Takes transformer output (StateMutation[], WorkflowSpan[]) \u2014 no raw spans, no API calls.\n */\nexport function gradeOutcome(\n  mutations: StateMutation[],\n  evalDef: EvalDefinition,\n  workflowSpans: WorkflowSpan[]\n): GraderResult[] {\n  const outcome = evalDef.outcome\n  if (!outcome) return []\n\n  const results: GraderResult[] = []\n\n  if (outcome.state && outcome.state.length > 0) {\n    const stateResults = gradeState(mutations, outcome.state)\n    results.push(...stateResults)\n  }\n\n  if (outcome.workflow && outcome.workflow.length > 0) {\n    const workflowResults = gradeWorkflows(workflowSpans, outcome.workflow)\n    results.push(...workflowResults)\n  }\n\n  return results\n}\n"],
-  "mappings": ";AAUO,SAAS,WAAW,UAAyB,QAA0B;AAE5E,MAAI,OAAO,aAAa,UAAU;AAChC,WAAO,OAAO,MAAM,MAAM;AAAA,EAC5B;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,MAAI,cAAc,UAAU;AAC1B,WAAO,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,SAAS,YAAY,CAAC;AAAA,EAC9E;AAEA,MAAI,kBAAkB,UAAU;AAC9B,WAAO,CAAC,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,aAAa,YAAY,CAAC;AAAA,EACnF;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,IAAI,OAAO,SAAS,SAAS,GAAG,EAAE,KAAK,OAAO,MAAM,CAAC;AAAA,EAC9D;AAEA,MAAI,QAAQ,UAAU;AACpB,WAAO,SAAS,GAAG,SAAS,MAAM;AAAA,EACpC;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,SAAS,SAAS,WAAW,UAAa,WAAW,OAAO,WAAW,UAAa,WAAW;AAAA,EACxG;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,UAAiC;AAChE,MAAI,OAAO,aAAa,SAAU,QAAO,WAAW,QAAQ;AAC5D,MAAI,YAAY,SAAU,QAAO,UAAU,KAAK,UAAU,SAAS,MAAM,CAAC;AAC1E,MAAI,cAAc,SAAU,QAAO,aAAa,SAAS,QAAQ;AACjE,MAAI,kBAAkB,SAAU,QAAO,iBAAiB,SAAS,YAAY;AAC7E,MAAI,aAAa,SAAU,QAAO,YAAY,SAAS,OAAO;AAC9D,MAAI,QAAQ,SAAU,QAAO,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACxF,MAAI,YAAY,SAAU,QAAO,SAAS,SAAS,WAAW;AAC9D,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,SAAO,KAAK,UAAU,QAAQ;AAChC;;;ACzDA,SAAS,cAAc;AACvB,SAAS,iBAAiB;AAG1B,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAE1B,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAMA,eAAsB,aACpB,aACA,SACA;AACA,QAAM,MAAM,SAAS,UAAU;AAE/B,QAAM,SAAS,IAAI,OAAO;AAAA,IACxB,OAAO,YAAY;AAAA,IACnB,QAAQ,YAAY;AAAA,IACpB,OAAO,YAAY;AAAA,IACnB,GAAI,YAAY,gBAAgB,SAAY,EAAE,aAAa,YAAY,YAAY,IAAI,CAAC;AAAA,EAC1F,CAAC;AACD,eAAa,IAAI,UAAU,EAAE,QAAQ,qBAAqB,KAAK,CAAC;AAEhE,MAAI;AACF,uBAAmB,MAAM,MAAM,MAAM,WAAW,kBAAkB,GAAG,KAAK,CAAC;AAAA,EAC7E,SAAS,KAAK;AACZ,QAAI,KAAK,qCAAsC,IAAc,OAAO,kCAAkC;AACtG,uBAAmB,CAAC;AAAA,EACtB;AAEA,gBAAc,SAAS,SAAS;AAEhC,MAAI,iBAAiB,WAAW,GAAG;AACjC,QAAI,KAAK,6DAA6D;AAAA,EACxE,WAAW,CAAC,cAAc,WAAW,GAAG;AACtC,QAAI,KAAK,+BAA+B,WAAW,0DAA0D;AAAA,EAC/G;AACF;AAEA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AACF,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,yCAAqC,IAAc,OAAO;AAAA,IACpE;AAAA,EACF;AACF;;;ACnJA,eAAsB,cACpB,aACA,YACA,SACyB;AACzB,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,YAAY,YAAY,EAAE,SAAS,UAAU,SAAS,YAAY,CAAC;AAChF,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,QAAQ;AAAA,QAC1C;AAAA,QACA,UAAU,sBAAsB,UAAU,QAAQ;AAAA,QAClD,QAAQ,OAAO,sBAAsB;AAAA,MACvC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,kBAAkB,WAAW;AAC/B,YAAM,OAAO,CAAC,YAAY,YAAY,EAAE,SAAS,UAAU,aAAa,YAAY,CAAC;AACrF,cAAQ,KAAK;AAAA,QACX,WAAW,iBAAiB,UAAU,YAAY;AAAA,QAClD;AAAA,QACA,UAAU,8BAA8B,UAAU,YAAY;AAAA,QAC9D,QAAQ,OAAO,0BAA0B;AAAA,MAC3C,CAAC;AACD;AAAA,IACF;AAEA,QAAI,aAAa,WAAW;AAC1B,YAAM,QAAQ,IAAI,OAAO,UAAU,SAAS,GAAG;AAC/C,YAAM,OAAO,MAAM,KAAK,WAAW;AACnC,cAAQ,KAAK;AAAA,QACX,WAAW,WAAW,UAAU,OAAO;AAAA,QACvC;AAAA,QACA,UAAU,qBAAqB,UAAU,OAAO;AAAA,QAChD,QAAQ,OAAO,YAAY;AAAA,MAC7B,CAAC;AACD;AAAA,IACF;AAEA,QAAI,eAAe,WAAW;AAC5B,YAAM,SAAS,MAAM,cAAc,aAAa,UAAU,WAAW;AAAA,QACnE,aAAa,QAAQ;AAAA,QACrB,GAAI,QAAQ,uBAAuB,SAAY,EAAE,eAAe,QAAQ,mBAAmB,IAAI,CAAC;AAAA,MAClG,CAAC;AACD,cAAQ,KAAK,MAAM;AACnB;AAAA,IACF;AAaA,YAAQ,KAAK;AAAA,MACX,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,sBAAsB,KAAK,UAAU,SAAS,CAAC;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;;;ACvEO,SAAS,WAAW,WAAuB,YAA6C;AAC7F,SAAO,WAAW,IAAI,CAAC,cAAc;AAEnC,QAAI,YAAY,aAAa,EAAE,gBAAgB,cAAc,EAAE,gBAAgB,YAAY;AACzF,YAAM,UAAU,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,MAAM;AACrE,YAAM,YAAY,QAAQ,SAAS;AAEnC,UAAI,CAAC,WAAW;AACd,eAAO;AAAA,UACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,UAC3C,MAAM;AAAA,UACN,UAAU,GAAG,UAAU,MAAM;AAAA,UAC7B,QAAQ,8BAA8B,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,KAAK,MAAM;AAAA,QAC3F;AAAA,MACF;AAGA,UAAI,UAAU,QAAQ;AACpB,cAAM,eAAiE,CAAC;AAExE,mBAAW,CAAC,KAAK,QAAQ,KAAK,OAAO,QAAQ,UAAU,MAAM,GAAG;AAE9D,gBAAM,WAAW,QAAQ,KAAK,CAAC,OAAO,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AACzE,uBAAa,KAAK;AAAA,YAChB;AAAA,YACA,MAAM;AAAA,YACN,QAAQ,WACJ,YACA,YAAY,GAAG,IAAI,iBAAiB,QAAQ,CAAC,SAAS,KAAK,UAAU,QAAQ,IAAI,CAAC,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;AAAA,UAC9G,CAAC;AAAA,QACH;AAEA,cAAM,gBAAgB,aAAa,MAAM,CAAC,MAAM,EAAE,IAAI;AACtD,cAAM,eAAe,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;AAEvD,eAAO;AAAA,UACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,UAC3C,MAAM;AAAA,UACN,UAAU,GAAG,UAAU,MAAM,gBAAgB,OAAO,QAAQ,UAAU,MAAM,EACzE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,CAAC,EAAE,EAC7C,KAAK,IAAI,CAAC;AAAA,UACb,QAAQ,gBAAgB,YAAY,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI;AAAA,QACjF;AAAA,MACF;AAEA,aAAO;AAAA,QACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,QAC3C,MAAM;AAAA,QACN,UAAU,GAAG,UAAU,MAAM;AAAA,QAC7B,QAAQ,UAAU,QAAQ,MAAM;AAAA,MAClC;AAAA,IACF;AAGA,QAAI,gBAAgB,WAAW;AAC7B,YAAM,YAAY,UAAU,KAAK,CAAC,OAAO,GAAG,SAAS,UAAU,UAAU;AACzE,aAAO;AAAA,QACL,WAAW,oBAAoB,UAAU,UAAU;AAAA,QACnD,MAAM,CAAC;AAAA,QACP,UAAU,GAAG,UAAU,UAAU;AAAA,QACjC,QAAQ,YAAY,eAAe;AAAA,MACrC;AAAA,IACF;AAGA,QAAI,gBAAgB,WAAW;AAC7B,YAAM,cAAc,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI;AACjD,YAAM,gBAAgB,UAAU;AAGhC,UAAI,SAAS;AACb,iBAAW,QAAQ,aAAa;AAC9B,YAAI,SAAS,cAAc,UAAU,SAAS,cAAc,MAAM,GAAG;AACnE;AAAA,QACF;AAAA,MACF;AACA,YAAM,UAAU,WAAW,cAAc;AAEzC,aAAO;AAAA,QACL,WAAW,gBAAgB,cAAc,KAAK,UAAK,CAAC;AAAA,QACpD,MAAM;AAAA,QACN,UAAU,2BAA2B,cAAc,KAAK,UAAK,CAAC;AAAA,QAC9D,QAAQ,kBAAkB,YAAY,KAAK,UAAK,KAAK,MAAM;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,YAAY,KAAK,UAAU,SAAS,CAAC;AAAA,IAC/C;AAAA,EACF,CAAC;AACH;;;ACvFA,SAAS,eAAe,MAA+C;AACrE,QAAM,MAAM,KAAK,QAAQ,GAAG;AAC5B,MAAI,QAAQ,IAAI;AACd,UAAM,IAAI,MAAM,uBAAuB,IAAI,uCAAkC;AAAA,EAC/E;AAEA,QAAM,OAAO,KAAK,MAAM,GAAG,GAAG;AAC9B,QAAM,QAAQ,KAAK,MAAM,MAAM,CAAC;AAEhC,MAAI,SAAS,SAAS,SAAS,UAAU,SAAS,gBAAgB;AAChE,UAAM,IAAI,MAAM,uBAAuB,IAAI,cAAc,IAAI,8CAAyC;AAAA,EACxG;AAEA,SAAO,EAAE,MAAM,MAAM;AACvB;AAUO,SAAS,WAAW,WAA4B,YAA8C;AACnG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,SAAS,eAAe,UAAU,IAAI;AAC5C,UAAM,oBAAoB,UAAU,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,IAAI;AAExE,QAAI,UAAU,WAAW,QAAW;AAClC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QACE,iCAAiC,OAAO,IAAI;AAAA,QAEhD,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,kBAAkB,SAAS,CAAC;AAEnE,UAAI,aAAa,eAAe;AAC9B,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,aAAa;AAClC,YAAM,cAAc,eAAe,aAAa,OAAO,KAAK,IAAI;AAChE,YAAM,OAAO,UAAU,aAAa,UAAU,MAAM;AAEpD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI;AAAA,QACnC;AAAA,QACA,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,QACzC,QAAQ,KAAK,UAAU,WAAW;AAAA,MACpC,CAAC;AAAA,IACH;AAEA,QAAI,UAAU,YAAY,QAAW;AACnC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM,CAAC,UAAU;AAAA,UACjB,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ,uCAAuC,OAAO,OAAO;AAAA,QAC/D,CAAC;AACD;AAAA,MACF;AAGA,YAAM,aAAa,kBAAkB,KAAK,CAAC,MAAM,EAAE,aAAa;AAChE,UAAI,YAAY;AACd,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM;AAAA,UACN,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,KAAK,CAAC,MAAM,EAAE,YAAY,SAAS,OAAO,KAAK,CAAC;AACvF,YAAM,OAAO,UAAU,UAAU,eAAe,CAAC;AAEjD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,QAClF;AAAA,QACA,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,QAC1B,QAAQ,eAAe,UAAU,OAAO,KAAK,kBAAkB,UAAU,OAAO,KAAK;AAAA,MACvF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,UAAU,GAAY,GAAqB;AAClD,MAAI,MAAM,EAAG,QAAO;AACpB,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,MAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAElC,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,WAAO,EAAE,MAAM,CAAC,GAAG,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;AAAA,EAC7C;AAEA,MAAI,MAAM,QAAQ,CAAC,MAAM,MAAM,QAAQ,CAAC,EAAG,QAAO;AAElD,MAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;AAClD,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,QAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,WAAO,MAAM,MAAM,CAAC,MAAM,UAAW,EAA8B,CAAC,GAAI,EAA8B,CAAC,CAAC,CAAC;AAAA,EAC3G;AAEA,SAAO;AACT;;;AC3IO,SAAS,eAAe,WAA2B,YAAiD;AACzG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,WAAW,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,IAAI;AAGpE,QAAI,UAAU,YAAY,QAAW;AACnC,YAAM,aAAa,SAAS,SAAS;AACrC,YAAM,OAAO,UAAU,UAAU,aAAa,CAAC;AAE/C,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,aAAa;AAAA,QACvF;AAAA,QACA,UAAU,UAAU,UAChB,aAAa,UAAU,IAAI,kBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,aAAa,SAAS,SAAS,MAAM,sBAAsB;AAAA,MACrE,CAAC;AAAA,IACH;AAGA,QAAI,UAAU,cAAc,QAAW;AACrC,YAAM,cAAc,SAAS;AAAA,QAC3B,CAAC,OAAO,GAAG,gBAAgB,eAAe,GAAG,WAAW,QAAQ,GAAG,WAAW;AAAA,MAChF;AACA,YAAM,OAAO,UAAU,YAAY,cAAc,CAAC;AAElD,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,YAAY,cAAc,eAAe;AAAA,QAC7F;AAAA,QACA,UAAU,UAAU,YAChB,aAAa,UAAU,IAAI,gBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,cAAc,cAAc;AAAA,MACtC,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;;;ACvCO,SAAS,YAAY,aAAqB,YAA+C;AAC9F,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,OAAO,WAAW,UAAU,eAAe,WAAW;AAC5D,UAAM,WAAW,iBAAiB,UAAU,aAAa;AAEzD,YAAQ,KAAK;AAAA,MACX,WAAW,iBAAiB,QAAQ;AAAA,MACpC;AAAA,MACA,UAAU,iBAAiB,QAAQ;AAAA,MACnC,QAAQ,GAAG,WAAW;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AACT;;;ACXO,SAAS,aACd,WACA,SACA,eACgB;AAChB,QAAM,UAAU,QAAQ;AACxB,MAAI,CAAC,QAAS,QAAO,CAAC;AAEtB,QAAM,UAA0B,CAAC;AAEjC,MAAI,QAAQ,SAAS,QAAQ,MAAM,SAAS,GAAG;AAC7C,UAAM,eAAe,WAAW,WAAW,QAAQ,KAAK;AACxD,YAAQ,KAAK,GAAG,YAAY;AAAA,EAC9B;AAEA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,UAAM,kBAAkB,eAAe,eAAe,QAAQ,QAAQ;AACtE,YAAQ,KAAK,GAAG,eAAe;AAAA,EACjC;AAEA,SAAO;AACT;",
+  "sources": ["../../src/graders/match.ts", "../../src/graders/llm.ts", "../../src/types.ts", "../../src/graders/response.ts", "../../src/graders/tools.ts", "../../src/graders/state.ts", "../../src/graders/workflow.ts", "../../src/graders/timing.ts", "../../src/graders/outcome.ts"],
+  "sourcesContent": ["/**\n * Shared match utilities for assertion grading.\n * Used by tool, state, and table graders.\n */\n\nimport type { MatchOperator } from '../types'\n\n/**\n * Evaluate a single match operator against a value.\n */\nexport function matchValue(operator: MatchOperator, actual: unknown): boolean {\n  // Shorthand: string means exact match\n  if (typeof operator === 'string') {\n    return String(actual) === operator\n  }\n\n  if ('equals' in operator) {\n    return actual === operator.equals\n  }\n\n  if ('contains' in operator) {\n    return String(actual).toLowerCase().includes(operator.contains.toLowerCase())\n  }\n\n  if ('not_contains' in operator) {\n    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase())\n  }\n\n  if ('matches' in operator) {\n    return new RegExp(operator.matches, 'i').test(String(actual))\n  }\n\n  if ('in' in operator) {\n    return operator.in.includes(actual)\n  }\n\n  if ('exists' in operator) {\n    return operator.exists ? actual !== undefined && actual !== null : actual === undefined || actual === null\n  }\n\n  if ('gte' in operator) {\n    return Number(actual) >= operator.gte\n  }\n\n  if ('lte' in operator) {\n    return Number(actual) <= operator.lte\n  }\n\n  return false\n}\n\nexport function operatorToString(operator: MatchOperator): string {\n  if (typeof operator === 'string') return `equals \"${operator}\"`\n  if ('equals' in operator) return `equals ${JSON.stringify(operator.equals)}`\n  if ('contains' in operator) return `contains \"${operator.contains}\"`\n  if ('not_contains' in operator) return `not_contains \"${operator.not_contains}\"`\n  if ('matches' in operator) return `matches /${operator.matches}/`\n  if ('in' in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(', ')}]`\n  if ('exists' in operator) return operator.exists ? 'exists' : 'does not exist'\n  if ('gte' in operator) return `>= ${operator.gte}`\n  if ('lte' in operator) return `<= ${operator.lte}`\n  return JSON.stringify(operator)\n}\n", "/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\nimport { defaultLogger } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\nlet _log: EvalLogger = defaultLogger\n\n/** @internal Test-only: clears module judge state so tests stay hermetic in the shared bun test process. */\nexport function _resetLLMJudgeForTests(): void {\n  _cognitive = null\n  _availableModels = []\n  _judgeModel = 'fast'\n  _log = defaultLogger\n}\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? defaultLogger\n  _log = log\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    // A judge that can't run must NOT silently pass the assertion \u2014 a flaky\n    // LLM call or missing credentials would turn real assertions green.\n    // Unavailability is a visible failure with a clear reason.\n    if (!_cognitive) {\n      _log.warn(`LLM judge unavailable for \"${criteria}\": no credentials configured (initLLMJudge not called)`)\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      _log.warn(`LLM judge returned an empty response for \"${criteria}\"`)\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    _log.warn(`LLM judge failed for \"${criteria}\": ${(err as Error).message}`)\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: false,\n      expected: criteria,\n      actual: `FAILED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n", "/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './trace'\n\nexport interface ChatClientConnection {\n  user: { id: string }\n  createConversation: (props: Record<string, unknown>) => Promise<{ conversation: { id: string } }>\n  // Broad parameter types keep this compatible with @botpress/chat's narrower\n  // generated payload unions while avoiding a dependency on those definitions.\n  createMessage: (props: any) => Promise<unknown>\n  createEvent: (props: any) => Promise<unknown>\n}\n\nexport interface ChatClient {\n  connect: (props: { webhookId: string }) => Promise<ChatClientConnection>\n}\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnNumber: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  webhookId: string\n  botId: string\n  token: string\n  apiUrl: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  credentials: {\n    token: string\n    apiUrl: string\n    botId: string\n    workspaceId?: string\n    webhookId?: string\n  }\n  agentPath: string\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n", "/**\n * Response assertion graders.\n * Checks bot response text against expected assertions.\n */\n\nimport type { ResponseAssertion, GraderResult } from '../types'\nimport { gradeLLMJudge } from './llm'\n\nexport async function gradeResponse(\n  botResponse: string,\n  assertions: ResponseAssertion[],\n  context: { userMessage: string; judgePassThreshold?: number }\n): Promise<GraderResult[]> {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    if ('contains' in assertion) {\n      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase())\n      results.push({\n        assertion: `contains \"${assertion.contains}\"`,\n        pass,\n        expected: `Response contains \"${assertion.contains}\"`,\n        actual: pass ? `Found in response` : `Not found in response`,\n      })\n      continue\n    }\n\n    if ('not_contains' in assertion) {\n      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase())\n      results.push({\n        assertion: `not_contains \"${assertion.not_contains}\"`,\n        pass,\n        expected: `Response does not contain \"${assertion.not_contains}\"`,\n        actual: pass ? `Not found in response` : `Found in response`,\n      })\n      continue\n    }\n\n    if ('matches' in assertion) {\n      const regex = new RegExp(assertion.matches, 'i')\n      const pass = regex.test(botResponse)\n      results.push({\n        assertion: `matches ${assertion.matches}`,\n        pass,\n        expected: `Response matches /${assertion.matches}/`,\n        actual: pass ? `Matched` : `No match`,\n      })\n      continue\n    }\n\n    if ('llm_judge' in assertion) {\n      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {\n        userMessage: context.userMessage,\n        ...(context.judgePassThreshold !== undefined ? { passThreshold: context.judgePassThreshold } : {}),\n      })\n      results.push(result)\n      continue\n    }\n\n    // TODO: uncomment this once we support it\n    // if ('similar_to' in assertion) {\n    //   results.push({\n    //     assertion: `similar_to: \"${assertion.similar_to}\"`,\n    //     pass: true,\n    //     expected: assertion.similar_to,\n    //     actual: 'SKIPPED \u2014 similar_to not yet implemented',\n    //   })\n    //   continue\n    // }\n\n    results.push({\n      assertion: 'unknown',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown assertion: ${JSON.stringify(assertion)}`,\n    })\n  }\n\n  return results\n}\n", "/**\n * Tool assertion graders.\n * Checks tool calls from traces against expected assertions.\n */\n\nimport type { ToolAssertion, ToolCall, GraderResult } from '../types'\nimport { matchValue, operatorToString } from './match'\n\nexport function gradeTools(toolCalls: ToolCall[], assertions: ToolAssertion[]): GraderResult[] {\n  return assertions.map((assertion) => {\n    // --- called ---\n    if ('called' in assertion && !('not_called' in assertion) && !('call_order' in assertion)) {\n      const matches = toolCalls.filter((tc) => tc.name === assertion.called)\n      const wasCalled = matches.length > 0\n\n      if (!wasCalled) {\n        return {\n          assertion: `tool called: ${assertion.called}`,\n          pass: false,\n          expected: `${assertion.called} was called`,\n          actual: `Not called. Tools called: [${toolCalls.map((tc) => tc.name).join(', ') || 'none'}]`,\n        }\n      }\n\n      // Check params if specified\n      if (assertion.params) {\n        const paramResults: { key: string; pass: boolean; detail: string }[] = []\n\n        for (const [key, operator] of Object.entries(assertion.params)) {\n          // Check across all matching calls \u2014 pass if any call matches\n          const anyMatch = matches.some((tc) => matchValue(operator, tc.input[key]))\n          paramResults.push({\n            key,\n            pass: anyMatch,\n            detail: anyMatch\n              ? `matched`\n              : `expected ${key} ${operatorToString(operator)}, got ${JSON.stringify(matches.map((tc) => tc.input[key]))}`,\n          })\n        }\n\n        const allParamsPass = paramResults.every((p) => p.pass)\n        const failedParams = paramResults.filter((p) => !p.pass)\n\n        return {\n          assertion: `tool called: ${assertion.called} with params`,\n          pass: allParamsPass,\n          expected: `${assertion.called} called with ${Object.entries(assertion.params)\n            .map(([k, v]) => `${k} ${operatorToString(v)}`)\n            .join(', ')}`,\n          actual: allParamsPass ? `Matched` : failedParams.map((p) => p.detail).join('; '),\n        }\n      }\n\n      return {\n        assertion: `tool called: ${assertion.called}`,\n        pass: true,\n        expected: `${assertion.called} was called`,\n        actual: `Called ${matches.length} time(s)`,\n      }\n    }\n\n    // --- not_called ---\n    if ('not_called' in assertion) {\n      const wasCalled = toolCalls.some((tc) => tc.name === assertion.not_called)\n      return {\n        assertion: `tool not_called: ${assertion.not_called}`,\n        pass: !wasCalled,\n        expected: `${assertion.not_called} was NOT called`,\n        actual: wasCalled ? `Was called` : `Not called`,\n      }\n    }\n\n    // --- call_order ---\n    if ('call_order' in assertion) {\n      const calledNames = toolCalls.map((tc) => tc.name)\n      const expectedOrder = assertion.call_order as string[]\n\n      // Check that the expected sequence appears in order (not necessarily contiguous)\n      let cursor = 0\n      for (const name of calledNames) {\n        if (cursor < expectedOrder.length && name === expectedOrder[cursor]) {\n          cursor++\n        }\n      }\n      const inOrder = cursor === expectedOrder.length\n\n      return {\n        assertion: `call_order: [${expectedOrder.join(' \u2192 ')}]`,\n        pass: inOrder,\n        expected: `Tools called in order: [${expectedOrder.join(' \u2192 ')}]`,\n        actual: `Actual order: [${calledNames.join(' \u2192 ') || 'none'}]`,\n      }\n    }\n\n    return {\n      assertion: 'unknown tool assertion',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown: ${JSON.stringify(assertion)}`,\n    }\n  })\n}\n", "/**\n * State assertion graders.\n * Pure functions that grade state assertions against StateMutation[] from the transformer.\n * No API calls, no retries, no BpClient.\n */\n\nimport type { StateMutation, StateAssertion, GraderResult } from '../types'\n\n/**\n * Parse a state path like \"conversation.lastCity\" into its components.\n *   bot.*            -> type: \"bot\",          field: \"*\"\n *   user.*           -> type: \"user\",         field: \"*\"\n *   conversation.*   -> type: \"conversation\", field: \"*\"\n */\nfunction parseStatePath(path: string): { type: string; field: string } {\n  const dot = path.indexOf('.')\n  if (dot === -1) {\n    throw new Error(`Invalid state path \"${path}\" \u2014 expected \"type.field\" format`)\n  }\n\n  const type = path.slice(0, dot)\n  const field = path.slice(dot + 1)\n\n  if (type !== 'bot' && type !== 'user' && type !== 'conversation') {\n    throw new Error(`Unknown state type \"${type}\" in path \"${path}\" \u2014 expected bot, user, or conversation`)\n  }\n\n  return { type, field }\n}\n\n/**\n * Grade state assertions using StateMutation[] from the transformer.\n *\n * - `equals`: find the LAST mutation matching the state type, extract field from `current`, deep-compare\n * - `changed: true`: check if the field appears in ANY mutation's `changedKeys` for that state type\n * - `changed: false`: check that the field appears in NO mutation's `changedKeys`\n * - Swapped state: when `swappedToFile` is true, current/previous are undefined \u2014 report a clear message\n */\nexport function gradeState(mutations: StateMutation[], assertions: StateAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const parsed = parseStatePath(assertion.path)\n    const matchingMutations = mutations.filter((m) => m.type === parsed.type)\n\n    if (assertion.equals !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual:\n            `No state.save span found for \"${parsed.type}\" \u2014 the bot did not mutate this state during the turn. ` +\n            'If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.',\n        })\n        continue\n      }\n\n      const lastMutation = matchingMutations[matchingMutations.length - 1]!\n\n      if (lastMutation.swappedToFile) {\n        results.push({\n          assertion: `state: ${assertion.path} equals`,\n          pass: false,\n          expected: JSON.stringify(assertion.equals),\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const currentState = lastMutation.current as Record<string, unknown> | undefined\n      const actualValue = currentState ? currentState[parsed.field] : undefined\n      const pass = deepEqual(actualValue, assertion.equals)\n\n      results.push({\n        assertion: `state: ${assertion.path} equals`,\n        pass,\n        expected: JSON.stringify(assertion.equals),\n        actual: JSON.stringify(actualValue),\n      })\n    }\n\n    if (assertion.changed !== undefined) {\n      if (matchingMutations.length === 0) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: !assertion.changed,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'No state mutation found for type \"' + parsed.type + '\"',\n        })\n        continue\n      }\n\n      // Check for swapped state in any matching mutation\n      const hasSwapped = matchingMutations.some((m) => m.swappedToFile)\n      if (hasSwapped) {\n        results.push({\n          assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n          pass: false,\n          expected: assertion.changed\n            ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n            : `Field \"${parsed.field}\" unchanged`,\n          actual: 'State was too large for trace-based assertion (swapped to file)',\n        })\n        continue\n      }\n\n      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field))\n      const pass = assertion.changed ? fieldChanged : !fieldChanged\n\n      results.push({\n        assertion: `state: ${assertion.path} ${assertion.changed ? 'changed' : 'unchanged'}`,\n        pass,\n        expected: assertion.changed\n          ? `Field \"${parsed.field}\" changed in a \"${parsed.type}\" state mutation`\n          : `Field \"${parsed.field}\" unchanged`,\n        actual: fieldChanged ? `Field \"${parsed.field}\" was changed` : `Field \"${parsed.field}\" was not changed`,\n      })\n    }\n  }\n\n  return results\n}\n\nfunction deepEqual(a: unknown, b: unknown): boolean {\n  if (a === b) return true\n  if (a === null || b === null) return false\n  if (typeof a !== typeof b) return false\n\n  if (Array.isArray(a) && Array.isArray(b)) {\n    if (a.length !== b.length) return false\n    return a.every((v, i) => deepEqual(v, b[i]))\n  }\n\n  if (Array.isArray(a) !== Array.isArray(b)) return false\n\n  if (typeof a === 'object' && typeof b === 'object') {\n    const keysA = Object.keys(a as object)\n    const keysB = Object.keys(b as object)\n    if (keysA.length !== keysB.length) return false\n    return keysA.every((k) => deepEqual((a as Record<string, unknown>)[k], (b as Record<string, unknown>)[k]))\n  }\n\n  return false\n}\n", "/**\n * Workflow assertion graders.\n * Checks workflow execution via transformer-produced WorkflowSpan data.\n */\n\nimport type { WorkflowAssertion, WorkflowSpan, GraderResult } from '../types'\n\nexport function gradeWorkflows(workflows: WorkflowSpan[], assertions: WorkflowAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const matching = workflows.filter((wf) => wf.name === assertion.name)\n\n    // entered \u2014 was the workflow triggered at all?\n    if (assertion.entered !== undefined) {\n      const wasEntered = matching.length > 0\n      const pass = assertion.entered ? wasEntered : !wasEntered\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.entered ? 'entered' : 'not entered'}`,\n        pass,\n        expected: assertion.entered\n          ? `Workflow \"${assertion.name}\" was entered`\n          : `Workflow \"${assertion.name}\" was not entered`,\n        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`,\n      })\n    }\n\n    // completed \u2014 did the workflow complete?\n    if (assertion.completed !== undefined) {\n      const didComplete = matching.some(\n        (wf) => wf.statusFinal === 'completed' || wf.status === 'ok' || wf.status === 'error'\n      )\n      const pass = assertion.completed ? didComplete : !didComplete\n\n      results.push({\n        assertion: `workflow: ${assertion.name} ${assertion.completed ? 'completed' : 'not completed'}`,\n        pass,\n        expected: assertion.completed\n          ? `Workflow \"${assertion.name}\" completed`\n          : `Workflow \"${assertion.name}\" did not complete`,\n        actual: didComplete ? `Completed` : `Not completed`,\n      })\n    }\n  }\n\n  return results\n}\n", "/**\n * Timing assertion graders.\n * Checks bot response duration against expected thresholds.\n */\n\nimport type { TimingAssertion, GraderResult } from '../types'\nimport { matchValue, operatorToString } from './match'\n\nexport function gradeTiming(botDuration: number, assertions: TimingAssertion[]): GraderResult[] {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    const pass = matchValue(assertion.response_time, botDuration)\n    const expected = operatorToString(assertion.response_time)\n\n    results.push({\n      assertion: `response_time ${expected}`,\n      pass,\n      expected: `Response time ${expected}`,\n      actual: `${botDuration}ms`,\n    })\n  }\n\n  return results\n}\n", "/**\n * Outcome assertion orchestrator.\n * Runs state and workflow graders after all conversation turns complete.\n */\n\nimport type { EvalDefinition, GraderResult, StateMutation, WorkflowSpan } from '../types'\nimport { gradeState } from './state'\nimport { gradeWorkflows } from './workflow'\n\n/**\n * Grade all outcome assertions after the conversation completes.\n * Takes transformer output (StateMutation[], WorkflowSpan[]) \u2014 no raw spans, no API calls.\n */\nexport function gradeOutcome(\n  mutations: StateMutation[],\n  evalDef: EvalDefinition,\n  workflowSpans: WorkflowSpan[]\n): GraderResult[] {\n  const outcome = evalDef.outcome\n  if (!outcome) return []\n\n  const results: GraderResult[] = []\n\n  if (outcome.state && outcome.state.length > 0) {\n    const stateResults = gradeState(mutations, outcome.state)\n    results.push(...stateResults)\n  }\n\n  if (outcome.workflow && outcome.workflow.length > 0) {\n    const workflowResults = gradeWorkflows(workflowSpans, outcome.workflow)\n    results.push(...workflowResults)\n  }\n\n  return results\n}\n"],
+  "mappings": ";AAUO,SAAS,WAAW,UAAyB,QAA0B;AAE5E,MAAI,OAAO,aAAa,UAAU;AAChC,WAAO,OAAO,MAAM,MAAM;AAAA,EAC5B;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,WAAW,SAAS;AAAA,EAC7B;AAEA,MAAI,cAAc,UAAU;AAC1B,WAAO,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,SAAS,YAAY,CAAC;AAAA,EAC9E;AAEA,MAAI,kBAAkB,UAAU;AAC9B,WAAO,CAAC,OAAO,MAAM,EAAE,YAAY,EAAE,SAAS,SAAS,aAAa,YAAY,CAAC;AAAA,EACnF;AAEA,MAAI,aAAa,UAAU;AACzB,WAAO,IAAI,OAAO,SAAS,SAAS,GAAG,EAAE,KAAK,OAAO,MAAM,CAAC;AAAA,EAC9D;AAEA,MAAI,QAAQ,UAAU;AACpB,WAAO,SAAS,GAAG,SAAS,MAAM;AAAA,EACpC;AAEA,MAAI,YAAY,UAAU;AACxB,WAAO,SAAS,SAAS,WAAW,UAAa,WAAW,OAAO,WAAW,UAAa,WAAW;AAAA,EACxG;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,MAAI,SAAS,UAAU;AACrB,WAAO,OAAO,MAAM,KAAK,SAAS;AAAA,EACpC;AAEA,SAAO;AACT;AAEO,SAAS,iBAAiB,UAAiC;AAChE,MAAI,OAAO,aAAa,SAAU,QAAO,WAAW,QAAQ;AAC5D,MAAI,YAAY,SAAU,QAAO,UAAU,KAAK,UAAU,SAAS,MAAM,CAAC;AAC1E,MAAI,cAAc,SAAU,QAAO,aAAa,SAAS,QAAQ;AACjE,MAAI,kBAAkB,SAAU,QAAO,iBAAiB,SAAS,YAAY;AAC7E,MAAI,aAAa,SAAU,QAAO,YAAY,SAAS,OAAO;AAC9D,MAAI,QAAQ,SAAU,QAAO,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC;AACxF,MAAI,YAAY,SAAU,QAAO,SAAS,SAAS,WAAW;AAC9D,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,MAAI,SAAS,SAAU,QAAO,MAAM,SAAS,GAAG;AAChD,SAAO,KAAK,UAAU,QAAQ;AAChC;;;ACzDA,SAAS,cAAc;AACvB,SAAS,iBAAiB;;;ACuJnB,IAAM,gBAA4B;;;ADnJzC,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAC1B,IAAI,OAAmB;AAUvB,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAMA,eAAsB,aACpB,aACA,SACA;AACA,QAAM,MAAM,SAAS,UAAU;AAC/B,SAAO;AAEP,QAAM,SAAS,IAAI,OAAO;AAAA,IACxB,OAAO,YAAY;AAAA,IACnB,QAAQ,YAAY;AAAA,IACpB,OAAO,YAAY;AAAA,IACnB,GAAI,YAAY,gBAAgB,SAAY,EAAE,aAAa,YAAY,YAAY,IAAI,CAAC;AAAA,EAC1F,CAAC;AACD,eAAa,IAAI,UAAU,EAAE,QAAQ,qBAAqB,KAAK,CAAC;AAEhE,MAAI;AACF,uBAAmB,MAAM,MAAM,MAAM,WAAW,kBAAkB,GAAG,KAAK,CAAC;AAAA,EAC7E,SAAS,KAAK;AACZ,QAAI,KAAK,qCAAsC,IAAc,OAAO,kCAAkC;AACtG,uBAAmB,CAAC;AAAA,EACtB;AAEA,gBAAc,SAAS,SAAS;AAEhC,MAAI,iBAAiB,WAAW,GAAG;AACjC,QAAI,KAAK,6DAA6D;AAAA,EACxE,WAAW,CAAC,cAAc,WAAW,GAAG;AACtC,QAAI,KAAK,+BAA+B,WAAW,0DAA0D;AAAA,EAC/G;AACF;AAEA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AAIF,QAAI,CAAC,YAAY;AACf,WAAK,KAAK,8BAA8B,QAAQ,wDAAwD;AACxG,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,WAAK,KAAK,6CAA6C,QAAQ,GAAG;AAClE,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,SAAK,KAAK,yBAAyB,QAAQ,MAAO,IAAc,OAAO,EAAE;AACzE,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,wCAAoC,IAAc,OAAO;AAAA,IACnE;AAAA,EACF;AACF;;;AEpKA,eAAsB,cACpB,aACA,YACA,SACyB;AACzB,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,YAAY,YAAY,EAAE,SAAS,UAAU,SAAS,YAAY,CAAC;AAChF,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,QAAQ;AAAA,QAC1C;AAAA,QACA,UAAU,sBAAsB,UAAU,QAAQ;AAAA,QAClD,QAAQ,OAAO,sBAAsB;AAAA,MACvC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,kBAAkB,WAAW;AAC/B,YAAM,OAAO,CAAC,YAAY,YAAY,EAAE,SAAS,UAAU,aAAa,YAAY,CAAC;AACrF,cAAQ,KAAK;AAAA,QACX,WAAW,iBAAiB,UAAU,YAAY;AAAA,QAClD;AAAA,QACA,UAAU,8BAA8B,UAAU,YAAY;AAAA,QAC9D,QAAQ,OAAO,0BAA0B;AAAA,MAC3C,CAAC;AACD;AAAA,IACF;AAEA,QAAI,aAAa,WAAW;AAC1B,YAAM,QAAQ,IAAI,OAAO,UAAU,SAAS,GAAG;AAC/C,YAAM,OAAO,MAAM,KAAK,WAAW;AACnC,cAAQ,KAAK;AAAA,QACX,WAAW,WAAW,UAAU,OAAO;AAAA,QACvC;AAAA,QACA,UAAU,qBAAqB,UAAU,OAAO;AAAA,QAChD,QAAQ,OAAO,YAAY;AAAA,MAC7B,CAAC;AACD;AAAA,IACF;AAEA,QAAI,eAAe,WAAW;AAC5B,YAAM,SAAS,MAAM,cAAc,aAAa,UAAU,WAAW;AAAA,QACnE,aAAa,QAAQ;AAAA,QACrB,GAAI,QAAQ,uBAAuB,SAAY,EAAE,eAAe,QAAQ,mBAAmB,IAAI,CAAC;AAAA,MAClG,CAAC;AACD,cAAQ,KAAK,MAAM;AACnB;AAAA,IACF;AAaA,YAAQ,KAAK;AAAA,MACX,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,sBAAsB,KAAK,UAAU,SAAS,CAAC;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;;;ACvEO,SAAS,WAAW,WAAuB,YAA6C;AAC7F,SAAO,WAAW,IAAI,CAAC,cAAc;AAEnC,QAAI,YAAY,aAAa,EAAE,gBAAgB,cAAc,EAAE,gBAAgB,YAAY;AACzF,YAAM,UAAU,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,MAAM;AACrE,YAAM,YAAY,QAAQ,SAAS;AAEnC,UAAI,CAAC,WAAW;AACd,eAAO;AAAA,UACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,UAC3C,MAAM;AAAA,UACN,UAAU,GAAG,UAAU,MAAM;AAAA,UAC7B,QAAQ,8BAA8B,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,KAAK,IAAI,KAAK,MAAM;AAAA,QAC3F;AAAA,MACF;AAGA,UAAI,UAAU,QAAQ;AACpB,cAAM,eAAiE,CAAC;AAExE,mBAAW,CAAC,KAAK,QAAQ,KAAK,OAAO,QAAQ,UAAU,MAAM,GAAG;AAE9D,gBAAM,WAAW,QAAQ,KAAK,CAAC,OAAO,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AACzE,uBAAa,KAAK;AAAA,YAChB;AAAA,YACA,MAAM;AAAA,YACN,QAAQ,WACJ,YACA,YAAY,GAAG,IAAI,iBAAiB,QAAQ,CAAC,SAAS,KAAK,UAAU,QAAQ,IAAI,CAAC,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;AAAA,UAC9G,CAAC;AAAA,QACH;AAEA,cAAM,gBAAgB,aAAa,MAAM,CAAC,MAAM,EAAE,IAAI;AACtD,cAAM,eAAe,aAAa,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI;AAEvD,eAAO;AAAA,UACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,UAC3C,MAAM;AAAA,UACN,UAAU,GAAG,UAAU,MAAM,gBAAgB,OAAO,QAAQ,UAAU,MAAM,EACzE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,CAAC,EAAE,EAC7C,KAAK,IAAI,CAAC;AAAA,UACb,QAAQ,gBAAgB,YAAY,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI;AAAA,QACjF;AAAA,MACF;AAEA,aAAO;AAAA,QACL,WAAW,gBAAgB,UAAU,MAAM;AAAA,QAC3C,MAAM;AAAA,QACN,UAAU,GAAG,UAAU,MAAM;AAAA,QAC7B,QAAQ,UAAU,QAAQ,MAAM;AAAA,MAClC;AAAA,IACF;AAGA,QAAI,gBAAgB,WAAW;AAC7B,YAAM,YAAY,UAAU,KAAK,CAAC,OAAO,GAAG,SAAS,UAAU,UAAU;AACzE,aAAO;AAAA,QACL,WAAW,oBAAoB,UAAU,UAAU;AAAA,QACnD,MAAM,CAAC;AAAA,QACP,UAAU,GAAG,UAAU,UAAU;AAAA,QACjC,QAAQ,YAAY,eAAe;AAAA,MACrC;AAAA,IACF;AAGA,QAAI,gBAAgB,WAAW;AAC7B,YAAM,cAAc,UAAU,IAAI,CAAC,OAAO,GAAG,IAAI;AACjD,YAAM,gBAAgB,UAAU;AAGhC,UAAI,SAAS;AACb,iBAAW,QAAQ,aAAa;AAC9B,YAAI,SAAS,cAAc,UAAU,SAAS,cAAc,MAAM,GAAG;AACnE;AAAA,QACF;AAAA,MACF;AACA,YAAM,UAAU,WAAW,cAAc;AAEzC,aAAO;AAAA,QACL,WAAW,gBAAgB,cAAc,KAAK,UAAK,CAAC;AAAA,QACpD,MAAM;AAAA,QACN,UAAU,2BAA2B,cAAc,KAAK,UAAK,CAAC;AAAA,QAC9D,QAAQ,kBAAkB,YAAY,KAAK,UAAK,KAAK,MAAM;AAAA,MAC7D;AAAA,IACF;AAEA,WAAO;AAAA,MACL,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,YAAY,KAAK,UAAU,SAAS,CAAC;AAAA,IAC/C;AAAA,EACF,CAAC;AACH;;;ACvFA,SAAS,eAAe,MAA+C;AACrE,QAAM,MAAM,KAAK,QAAQ,GAAG;AAC5B,MAAI,QAAQ,IAAI;AACd,UAAM,IAAI,MAAM,uBAAuB,IAAI,uCAAkC;AAAA,EAC/E;AAEA,QAAM,OAAO,KAAK,MAAM,GAAG,GAAG;AAC9B,QAAM,QAAQ,KAAK,MAAM,MAAM,CAAC;AAEhC,MAAI,SAAS,SAAS,SAAS,UAAU,SAAS,gBAAgB;AAChE,UAAM,IAAI,MAAM,uBAAuB,IAAI,cAAc,IAAI,8CAAyC;AAAA,EACxG;AAEA,SAAO,EAAE,MAAM,MAAM;AACvB;AAUO,SAAS,WAAW,WAA4B,YAA8C;AACnG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,SAAS,eAAe,UAAU,IAAI;AAC5C,UAAM,oBAAoB,UAAU,OAAO,CAAC,MAAM,EAAE,SAAS,OAAO,IAAI;AAExE,QAAI,UAAU,WAAW,QAAW;AAClC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QACE,iCAAiC,OAAO,IAAI;AAAA,QAEhD,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,kBAAkB,SAAS,CAAC;AAEnE,UAAI,aAAa,eAAe;AAC9B,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI;AAAA,UACnC,MAAM;AAAA,UACN,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,UACzC,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,aAAa;AAClC,YAAM,cAAc,eAAe,aAAa,OAAO,KAAK,IAAI;AAChE,YAAM,OAAO,UAAU,aAAa,UAAU,MAAM;AAEpD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI;AAAA,QACnC;AAAA,QACA,UAAU,KAAK,UAAU,UAAU,MAAM;AAAA,QACzC,QAAQ,KAAK,UAAU,WAAW;AAAA,MACpC,CAAC;AAAA,IACH;AAEA,QAAI,UAAU,YAAY,QAAW;AACnC,UAAI,kBAAkB,WAAW,GAAG;AAClC,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM,CAAC,UAAU;AAAA,UACjB,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ,uCAAuC,OAAO,OAAO;AAAA,QAC/D,CAAC;AACD;AAAA,MACF;AAGA,YAAM,aAAa,kBAAkB,KAAK,CAAC,MAAM,EAAE,aAAa;AAChE,UAAI,YAAY;AACd,gBAAQ,KAAK;AAAA,UACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,UAClF,MAAM;AAAA,UACN,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,UAC1B,QAAQ;AAAA,QACV,CAAC;AACD;AAAA,MACF;AAEA,YAAM,eAAe,kBAAkB,KAAK,CAAC,MAAM,EAAE,YAAY,SAAS,OAAO,KAAK,CAAC;AACvF,YAAM,OAAO,UAAU,UAAU,eAAe,CAAC;AAEjD,cAAQ,KAAK;AAAA,QACX,WAAW,UAAU,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,WAAW;AAAA,QAClF;AAAA,QACA,UAAU,UAAU,UAChB,UAAU,OAAO,KAAK,mBAAmB,OAAO,IAAI,qBACpD,UAAU,OAAO,KAAK;AAAA,QAC1B,QAAQ,eAAe,UAAU,OAAO,KAAK,kBAAkB,UAAU,OAAO,KAAK;AAAA,MACvF,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,UAAU,GAAY,GAAqB;AAClD,MAAI,MAAM,EAAG,QAAO;AACpB,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,MAAI,OAAO,MAAM,OAAO,EAAG,QAAO;AAElC,MAAI,MAAM,QAAQ,CAAC,KAAK,MAAM,QAAQ,CAAC,GAAG;AACxC,QAAI,EAAE,WAAW,EAAE,OAAQ,QAAO;AAClC,WAAO,EAAE,MAAM,CAAC,GAAG,MAAM,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;AAAA,EAC7C;AAEA,MAAI,MAAM,QAAQ,CAAC,MAAM,MAAM,QAAQ,CAAC,EAAG,QAAO;AAElD,MAAI,OAAO,MAAM,YAAY,OAAO,MAAM,UAAU;AAClD,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,UAAM,QAAQ,OAAO,KAAK,CAAW;AACrC,QAAI,MAAM,WAAW,MAAM,OAAQ,QAAO;AAC1C,WAAO,MAAM,MAAM,CAAC,MAAM,UAAW,EAA8B,CAAC,GAAI,EAA8B,CAAC,CAAC,CAAC;AAAA,EAC3G;AAEA,SAAO;AACT;;;AC3IO,SAAS,eAAe,WAA2B,YAAiD;AACzG,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,WAAW,UAAU,OAAO,CAAC,OAAO,GAAG,SAAS,UAAU,IAAI;AAGpE,QAAI,UAAU,YAAY,QAAW;AACnC,YAAM,aAAa,SAAS,SAAS;AACrC,YAAM,OAAO,UAAU,UAAU,aAAa,CAAC;AAE/C,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,UAAU,YAAY,aAAa;AAAA,QACvF;AAAA,QACA,UAAU,UAAU,UAChB,aAAa,UAAU,IAAI,kBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,aAAa,SAAS,SAAS,MAAM,sBAAsB;AAAA,MACrE,CAAC;AAAA,IACH;AAGA,QAAI,UAAU,cAAc,QAAW;AACrC,YAAM,cAAc,SAAS;AAAA,QAC3B,CAAC,OAAO,GAAG,gBAAgB,eAAe,GAAG,WAAW,QAAQ,GAAG,WAAW;AAAA,MAChF;AACA,YAAM,OAAO,UAAU,YAAY,cAAc,CAAC;AAElD,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,IAAI,IAAI,UAAU,YAAY,cAAc,eAAe;AAAA,QAC7F;AAAA,QACA,UAAU,UAAU,YAChB,aAAa,UAAU,IAAI,gBAC3B,aAAa,UAAU,IAAI;AAAA,QAC/B,QAAQ,cAAc,cAAc;AAAA,MACtC,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO;AACT;;;ACvCO,SAAS,YAAY,aAAqB,YAA+C;AAC9F,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,UAAM,OAAO,WAAW,UAAU,eAAe,WAAW;AAC5D,UAAM,WAAW,iBAAiB,UAAU,aAAa;AAEzD,YAAQ,KAAK;AAAA,MACX,WAAW,iBAAiB,QAAQ;AAAA,MACpC;AAAA,MACA,UAAU,iBAAiB,QAAQ;AAAA,MACnC,QAAQ,GAAG,WAAW;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AACT;;;ACXO,SAAS,aACd,WACA,SACA,eACgB;AAChB,QAAM,UAAU,QAAQ;AACxB,MAAI,CAAC,QAAS,QAAO,CAAC;AAEtB,QAAM,UAA0B,CAAC;AAEjC,MAAI,QAAQ,SAAS,QAAQ,MAAM,SAAS,GAAG;AAC7C,UAAM,eAAe,WAAW,WAAW,QAAQ,KAAK;AACxD,YAAQ,KAAK,GAAG,YAAY;AAAA,EAC9B;AAEA,MAAI,QAAQ,YAAY,QAAQ,SAAS,SAAS,GAAG;AACnD,UAAM,kBAAkB,eAAe,eAAe,QAAQ,QAAQ;AACtE,YAAQ,KAAK,GAAG,eAAe;AAAA,EACjC;AAEA,SAAO;AACT;",
   "names": []
 }

package/dist/graders/llm.d.ts

@@ -3,6 +3,8 @@
  * Replaces the OpenAI-based implementation from the standalone project.
  */
 import type { EvalLogger, GraderResult } from '../types';
+/** @internal Test-only: clears module judge state so tests stay hermetic in the shared bun test process. */
+export declare function _resetLLMJudgeForTests(): void;
 /**
  * Initialize the LLM judge with explicit credentials.
  * Call this before running evals so the judge doesn't need env vars.

package/dist/graders/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/graders/llm.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AA0CxD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,WAAW,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EACnF,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,UAAU,CAAA;CAAE,iBA0BlD;AAED,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GACvD,OAAO,CAAC,YAAY,CAAC,CAoEvB"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/graders/llm.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAqCxD,4GAA4G;AAC5G,wBAAgB,sBAAsB,IAAI,IAAI,CAK7C;AASD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,WAAW,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EACnF,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,UAAU,CAAA;CAAE,iBA2BlD;AAED,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GACvD,OAAO,CAAC,YAAY,CAAC,CA0EvB"}