@botpress/evals 1.19.0-beta.7 → 1.19.0-beta.9

package/dist/graders/llm.js

@@ -1,6 +1,11 @@
 // src/graders/llm.ts
 import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
+
+// src/types.ts
+var defaultLogger = console;
+
+// src/graders/llm.ts
 var DEFAULT_PASS_THRESHOLD = 3;
 function buildJudgeSystemPrompt(passThreshold) {
   return `You are an evaluation judge for a chatbot. You will be given:
@@ -30,13 +35,21 @@ var MODEL_ALIASES = ["fast", "best"];
 var _cognitive = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _log = defaultLogger;
+function _resetLLMJudgeForTests() {
+  _cognitive = null;
+  _availableModels = [];
+  _judgeModel = "fast";
+  _log = defaultLogger;
+}
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
 async function initLLMJudge(credentials, options) {
-  const log = options?.logger ?? console;
+  const log = options?.logger ?? defaultLogger;
+  _log = log;
   const client = new Client({
     token: credentials.token,
     apiUrl: credentials.apiUrl,
@@ -61,11 +74,12 @@ async function gradeLLMJudge(botResponse, criteria, context) {
   const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
   try {
     if (!_cognitive) {
+      _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+        actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
@@ -95,11 +109,12 @@ Criteria: ${criteria}`
     const rawContent = output?.choices?.[0]?.content;
     const content = typeof rawContent === "string" ? rawContent : null;
     if (!content) {
+      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge returned empty response"
+        actual: "FAILED \u2014 LLM judge returned empty response"
       };
     }
     const verdict = JSON.parse(content);
@@ -110,15 +125,17 @@ Criteria: ${criteria}`
       actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
     };
   } catch (err) {
+    _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
       assertion: `llm_judge: "${criteria}"`,
-      pass: true,
+      pass: false,
       expected: criteria,
-      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+      actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
   }
 }
 export {
+  _resetLLMJudgeForTests,
   gradeLLMJudge,
   initLLMJudge
 };

package/dist/graders/llm.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../../src/graders/llm.ts"],
-  "sourcesContent": ["/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? console\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    if (!_cognitive) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: true,\n      expected: criteria,\n      actual: `SKIPPED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n"],
-  "mappings": ";AAKA,SAAS,cAAc;AACvB,SAAS,iBAAiB;AAG1B,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAE1B,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAMA,eAAsB,aACpB,aACA,SACA;AACA,QAAM,MAAM,SAAS,UAAU;AAE/B,QAAM,SAAS,IAAI,OAAO;AAAA,IACxB,OAAO,YAAY;AAAA,IACnB,QAAQ,YAAY;AAAA,IACpB,OAAO,YAAY;AAAA,IACnB,GAAI,YAAY,gBAAgB,SAAY,EAAE,aAAa,YAAY,YAAY,IAAI,CAAC;AAAA,EAC1F,CAAC;AACD,eAAa,IAAI,UAAU,EAAE,QAAQ,qBAAqB,KAAK,CAAC;AAEhE,MAAI;AACF,uBAAmB,MAAM,MAAM,MAAM,WAAW,kBAAkB,GAAG,KAAK,CAAC;AAAA,EAC7E,SAAS,KAAK;AACZ,QAAI,KAAK,qCAAsC,IAAc,OAAO,kCAAkC;AACtG,uBAAmB,CAAC;AAAA,EACtB;AAEA,gBAAc,SAAS,SAAS;AAEhC,MAAI,iBAAiB,WAAW,GAAG;AACjC,QAAI,KAAK,6DAA6D;AAAA,EACxE,WAAW,CAAC,cAAc,WAAW,GAAG;AACtC,QAAI,KAAK,+BAA+B,WAAW,0DAA0D;AAAA,EAC/G;AACF;AAEA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AACF,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,yCAAqC,IAAc,OAAO;AAAA,IACpE;AAAA,EACF;AACF;",
+  "sources": ["../../src/graders/llm.ts", "../../src/types.ts"],
+  "sourcesContent": ["/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\nimport { defaultLogger } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\nlet _log: EvalLogger = defaultLogger\n\n/** @internal Test-only: clears module judge state so tests stay hermetic in the shared bun test process. */\nexport function _resetLLMJudgeForTests(): void {\n  _cognitive = null\n  _availableModels = []\n  _judgeModel = 'fast'\n  _log = defaultLogger\n}\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? defaultLogger\n  _log = log\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    // A judge that can't run must NOT silently pass the assertion \u2014 a flaky\n    // LLM call or missing credentials would turn real assertions green.\n    // Unavailability is a visible failure with a clear reason.\n    if (!_cognitive) {\n      _log.warn(`LLM judge unavailable for \"${criteria}\": no credentials configured (initLLMJudge not called)`)\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      _log.warn(`LLM judge returned an empty response for \"${criteria}\"`)\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    _log.warn(`LLM judge failed for \"${criteria}\": ${(err as Error).message}`)\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: false,\n      expected: criteria,\n      actual: `FAILED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n", "/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './trace'\n\nexport interface ChatClientConnection {\n  user: { id: string }\n  createConversation: (props: Record<string, unknown>) => Promise<{ conversation: { id: string } }>\n  // Broad parameter types keep this compatible with @botpress/chat's narrower\n  // generated payload unions while avoiding a dependency on those definitions.\n  createMessage: (props: any) => Promise<unknown>\n  createEvent: (props: any) => Promise<unknown>\n}\n\nexport interface ChatClient {\n  connect: (props: { webhookId: string }) => Promise<ChatClientConnection>\n}\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnNumber: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  webhookId: string\n  botId: string\n  token: string\n  apiUrl: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  credentials: {\n    token: string\n    apiUrl: string\n    botId: string\n    workspaceId?: string\n    webhookId?: string\n  }\n  agentPath: string\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n"],
+  "mappings": ";AAKA,SAAS,cAAc;AACvB,SAAS,iBAAiB;;;ACuJnB,IAAM,gBAA4B;;;ADnJzC,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAC1B,IAAI,OAAmB;AAGhB,SAAS,yBAA+B;AAC7C,eAAa;AACb,qBAAmB,CAAC;AACpB,gBAAc;AACd,SAAO;AACT;AAEA,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAMA,eAAsB,aACpB,aACA,SACA;AACA,QAAM,MAAM,SAAS,UAAU;AAC/B,SAAO;AAEP,QAAM,SAAS,IAAI,OAAO;AAAA,IACxB,OAAO,YAAY;AAAA,IACnB,QAAQ,YAAY;AAAA,IACpB,OAAO,YAAY;AAAA,IACnB,GAAI,YAAY,gBAAgB,SAAY,EAAE,aAAa,YAAY,YAAY,IAAI,CAAC;AAAA,EAC1F,CAAC;AACD,eAAa,IAAI,UAAU,EAAE,QAAQ,qBAAqB,KAAK,CAAC;AAEhE,MAAI;AACF,uBAAmB,MAAM,MAAM,MAAM,WAAW,kBAAkB,GAAG,KAAK,CAAC;AAAA,EAC7E,SAAS,KAAK;AACZ,QAAI,KAAK,qCAAsC,IAAc,OAAO,kCAAkC;AACtG,uBAAmB,CAAC;AAAA,EACtB;AAEA,gBAAc,SAAS,SAAS;AAEhC,MAAI,iBAAiB,WAAW,GAAG;AACjC,QAAI,KAAK,6DAA6D;AAAA,EACxE,WAAW,CAAC,cAAc,WAAW,GAAG;AACtC,QAAI,KAAK,+BAA+B,WAAW,0DAA0D;AAAA,EAC/G;AACF;AAEA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AAIF,QAAI,CAAC,YAAY;AACf,WAAK,KAAK,8BAA8B,QAAQ,wDAAwD;AACxG,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,WAAK,KAAK,6CAA6C,QAAQ,GAAG;AAClE,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,SAAK,KAAK,yBAAyB,QAAQ,MAAO,IAAc,OAAO,EAAE;AACzE,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,wCAAoC,IAAc,OAAO;AAAA,IACnE;AAAA,EACF;AACF;",
   "names": []
 }

package/dist/graders/response.js

@@ -1,6 +1,11 @@
 // src/graders/llm.ts
 import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
+
+// src/types.ts
+var defaultLogger = console;
+
+// src/graders/llm.ts
 var DEFAULT_PASS_THRESHOLD = 3;
 function buildJudgeSystemPrompt(passThreshold) {
   return `You are an evaluation judge for a chatbot. You will be given:
@@ -30,6 +35,7 @@ var MODEL_ALIASES = ["fast", "best"];
 var _cognitive = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
@@ -39,11 +45,12 @@ async function gradeLLMJudge(botResponse, criteria, context) {
   const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
   try {
     if (!_cognitive) {
+      _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+        actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
@@ -73,11 +80,12 @@ Criteria: ${criteria}`
     const rawContent = output?.choices?.[0]?.content;
     const content = typeof rawContent === "string" ? rawContent : null;
     if (!content) {
+      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
       return {
         assertion: `llm_judge: "${criteria}"`,
-        pass: true,
+        pass: false,
         expected: criteria,
-        actual: "SKIPPED \u2014 LLM judge returned empty response"
+        actual: "FAILED \u2014 LLM judge returned empty response"
       };
     }
     const verdict = JSON.parse(content);
@@ -88,11 +96,12 @@ Criteria: ${criteria}`
       actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
     };
   } catch (err) {
+    _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
       assertion: `llm_judge: "${criteria}"`,
-      pass: true,
+      pass: false,
       expected: criteria,
-      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+      actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
   }
 }

package/dist/graders/response.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../../src/graders/llm.ts", "../../src/graders/response.ts"],
-  "sourcesContent": ["/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? console\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    if (!_cognitive) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: true,\n        expected: criteria,\n        actual: 'SKIPPED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: true,\n      expected: criteria,\n      actual: `SKIPPED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n", "/**\n * Response assertion graders.\n * Checks bot response text against expected assertions.\n */\n\nimport type { ResponseAssertion, GraderResult } from '../types'\nimport { gradeLLMJudge } from './llm'\n\nexport async function gradeResponse(\n  botResponse: string,\n  assertions: ResponseAssertion[],\n  context: { userMessage: string; judgePassThreshold?: number }\n): Promise<GraderResult[]> {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    if ('contains' in assertion) {\n      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase())\n      results.push({\n        assertion: `contains \"${assertion.contains}\"`,\n        pass,\n        expected: `Response contains \"${assertion.contains}\"`,\n        actual: pass ? `Found in response` : `Not found in response`,\n      })\n      continue\n    }\n\n    if ('not_contains' in assertion) {\n      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase())\n      results.push({\n        assertion: `not_contains \"${assertion.not_contains}\"`,\n        pass,\n        expected: `Response does not contain \"${assertion.not_contains}\"`,\n        actual: pass ? `Not found in response` : `Found in response`,\n      })\n      continue\n    }\n\n    if ('matches' in assertion) {\n      const regex = new RegExp(assertion.matches, 'i')\n      const pass = regex.test(botResponse)\n      results.push({\n        assertion: `matches ${assertion.matches}`,\n        pass,\n        expected: `Response matches /${assertion.matches}/`,\n        actual: pass ? `Matched` : `No match`,\n      })\n      continue\n    }\n\n    if ('llm_judge' in assertion) {\n      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {\n        userMessage: context.userMessage,\n        ...(context.judgePassThreshold !== undefined ? { passThreshold: context.judgePassThreshold } : {}),\n      })\n      results.push(result)\n      continue\n    }\n\n    // TODO: uncomment this once we support it\n    // if ('similar_to' in assertion) {\n    //   results.push({\n    //     assertion: `similar_to: \"${assertion.similar_to}\"`,\n    //     pass: true,\n    //     expected: assertion.similar_to,\n    //     actual: 'SKIPPED \u2014 similar_to not yet implemented',\n    //   })\n    //   continue\n    // }\n\n    results.push({\n      assertion: 'unknown',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown assertion: ${JSON.stringify(assertion)}`,\n    })\n  }\n\n  return results\n}\n"],
-  "mappings": ";AAKA,SAAS,cAAc;AACvB,SAAS,iBAAiB;AAG1B,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAE1B,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAoCA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AACF,QAAI,CAAC,YAAY;AACf,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,yCAAqC,IAAc,OAAO;AAAA,IACpE;AAAA,EACF;AACF;;;ACnJA,eAAsB,cACpB,aACA,YACA,SACyB;AACzB,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,YAAY,YAAY,EAAE,SAAS,UAAU,SAAS,YAAY,CAAC;AAChF,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,QAAQ;AAAA,QAC1C;AAAA,QACA,UAAU,sBAAsB,UAAU,QAAQ;AAAA,QAClD,QAAQ,OAAO,sBAAsB;AAAA,MACvC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,kBAAkB,WAAW;AAC/B,YAAM,OAAO,CAAC,YAAY,YAAY,EAAE,SAAS,UAAU,aAAa,YAAY,CAAC;AACrF,cAAQ,KAAK;AAAA,QACX,WAAW,iBAAiB,UAAU,YAAY;AAAA,QAClD;AAAA,QACA,UAAU,8BAA8B,UAAU,YAAY;AAAA,QAC9D,QAAQ,OAAO,0BAA0B;AAAA,MAC3C,CAAC;AACD;AAAA,IACF;AAEA,QAAI,aAAa,WAAW;AAC1B,YAAM,QAAQ,IAAI,OAAO,UAAU,SAAS,GAAG;AAC/C,YAAM,OAAO,MAAM,KAAK,WAAW;AACnC,cAAQ,KAAK;AAAA,QACX,WAAW,WAAW,UAAU,OAAO;AAAA,QACvC;AAAA,QACA,UAAU,qBAAqB,UAAU,OAAO;AAAA,QAChD,QAAQ,OAAO,YAAY;AAAA,MAC7B,CAAC;AACD;AAAA,IACF;AAEA,QAAI,eAAe,WAAW;AAC5B,YAAM,SAAS,MAAM,cAAc,aAAa,UAAU,WAAW;AAAA,QACnE,aAAa,QAAQ;AAAA,QACrB,GAAI,QAAQ,uBAAuB,SAAY,EAAE,eAAe,QAAQ,mBAAmB,IAAI,CAAC;AAAA,MAClG,CAAC;AACD,cAAQ,KAAK,MAAM;AACnB;AAAA,IACF;AAaA,YAAQ,KAAK;AAAA,MACX,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,sBAAsB,KAAK,UAAU,SAAS,CAAC;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;",
+  "sources": ["../../src/graders/llm.ts", "../../src/types.ts", "../../src/graders/response.ts"],
+  "sourcesContent": ["/**\n * LLM judge grader using @botpress/cognitive.\n * Replaces the OpenAI-based implementation from the standalone project.\n */\n\nimport { Client } from '@botpress/client'\nimport { Cognitive } from '@botpress/cognitive'\nimport type { EvalLogger, GraderResult } from '../types'\nimport { defaultLogger } from '../types'\n\nconst DEFAULT_PASS_THRESHOLD = 3\n\nfunction buildJudgeSystemPrompt(passThreshold: number): string {\n  return `You are an evaluation judge for a chatbot. You will be given:\n- The user's message\n- The bot's response\n- Grading criteria\n\nYour job is to determine if the bot's response meets the criteria.\n\nRespond with a JSON object:\n{\n  \"pass\": true/false,\n  \"score\": 1-5,\n  \"reason\": \"brief explanation\"\n}\n\nScoring guide:\n- 5: Fully meets criteria, excellent response\n- 4: Mostly meets criteria, minor issues\n- 3: Partially meets criteria, notable gaps\n- 2: Barely meets criteria, significant issues\n- 1: Does not meet criteria\n\nA score of ${passThreshold} or above is a pass.`\n}\n\nconst MODEL_ALIASES = ['fast', 'best']\n\nlet _cognitive: Cognitive | null = null\nlet _availableModels: string[] = []\nlet _judgeModel: string = 'fast'\nlet _log: EvalLogger = defaultLogger\n\n/** @internal Test-only: clears module judge state so tests stay hermetic in the shared bun test process. */\nexport function _resetLLMJudgeForTests(): void {\n  _cognitive = null\n  _availableModels = []\n  _judgeModel = 'fast'\n  _log = defaultLogger\n}\n\nfunction _isModelValid(model: string): boolean {\n  if (MODEL_ALIASES.includes(model)) return true\n  if (_availableModels.length === 0) return false\n\n  return _availableModels.includes(model)\n}\n\n/**\n * Initialize the LLM judge with explicit credentials.\n * Call this before running evals so the judge doesn't need env vars.\n */\nexport async function initLLMJudge(\n  credentials: { token: string; apiUrl: string; botId: string; workspaceId?: string },\n  options?: { model?: string; logger?: EvalLogger }\n) {\n  const log = options?.logger ?? defaultLogger\n  _log = log\n\n  const client = new Client({\n    token: credentials.token,\n    apiUrl: credentials.apiUrl,\n    botId: credentials.botId,\n    ...(credentials.workspaceId !== undefined ? { workspaceId: credentials.workspaceId } : {}),\n  })\n  _cognitive = new Cognitive({ client, __experimental_beta: true })\n\n  try {\n    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys())\n  } catch (err) {\n    log.warn(`Failed to fetch available models: ${(err as Error).message}. LLM judge will be unavailable.`)\n    _availableModels = []\n  }\n\n  _judgeModel = options?.model ?? 'fast'\n\n  if (_availableModels.length === 0) {\n    log.warn('No available Cognitive models: cannot validate judge model.')\n  } else if (!_isModelValid(_judgeModel)) {\n    log.warn(`Configured LLM judge model \"${_judgeModel}\" is invalid. Run \"adk models\" to list available models.`)\n  }\n}\n\nexport async function gradeLLMJudge(\n  botResponse: string,\n  criteria: string,\n  context: { userMessage: string; passThreshold?: number }\n): Promise<GraderResult> {\n  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD))\n\n  try {\n    // A judge that can't run must NOT silently pass the assertion \u2014 a flaky\n    // LLM call or missing credentials would turn real assertions green.\n    // Unavailability is a visible failure with a clear reason.\n    if (!_cognitive) {\n      _log.warn(`LLM judge unavailable for \"${criteria}\": no credentials configured (initLLMJudge not called)`)\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 LLM judge unavailable: no credentials configured',\n      }\n    }\n\n    if (!_isModelValid(_judgeModel)) {\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 invalid LLM judge model',\n      }\n    }\n\n    type CognitiveModel = Exclude<Parameters<Cognitive['generateContent']>[0]['model'], undefined>\n\n    const { output } = await _cognitive.generateContent({\n      model: _judgeModel as CognitiveModel,\n      temperature: 0,\n      responseFormat: 'json_object',\n      systemPrompt: buildJudgeSystemPrompt(threshold),\n      messages: [\n        {\n          role: 'user',\n          content: `User message: ${context.userMessage}\\n\\nBot response: ${botResponse}\\n\\nCriteria: ${criteria}`,\n        },\n      ],\n    })\n\n    const rawContent = output?.choices?.[0]?.content\n    const content = typeof rawContent === 'string' ? rawContent : null\n    if (!content) {\n      _log.warn(`LLM judge returned an empty response for \"${criteria}\"`)\n      return {\n        assertion: `llm_judge: \"${criteria}\"`,\n        pass: false,\n        expected: criteria,\n        actual: 'FAILED \u2014 LLM judge returned empty response',\n      }\n    }\n\n    const verdict = JSON.parse(content) as {\n      pass: boolean\n      score: number\n      reason: string\n    }\n\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: verdict.score >= threshold,\n      expected: criteria,\n      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`,\n    }\n  } catch (err) {\n    _log.warn(`LLM judge failed for \"${criteria}\": ${(err as Error).message}`)\n    return {\n      assertion: `llm_judge: \"${criteria}\"`,\n      pass: false,\n      expected: criteria,\n      actual: `FAILED \u2014 LLM judge unavailable: ${(err as Error).message}`,\n    }\n  }\n}\n", "/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './trace'\n\nexport interface ChatClientConnection {\n  user: { id: string }\n  createConversation: (props: Record<string, unknown>) => Promise<{ conversation: { id: string } }>\n  // Broad parameter types keep this compatible with @botpress/chat's narrower\n  // generated payload unions while avoiding a dependency on those definitions.\n  createMessage: (props: any) => Promise<unknown>\n  createEvent: (props: any) => Promise<unknown>\n}\n\nexport interface ChatClient {\n  connect: (props: { webhookId: string }) => Promise<ChatClientConnection>\n}\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnNumber: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  webhookId: string\n  botId: string\n  token: string\n  apiUrl: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  credentials: {\n    token: string\n    apiUrl: string\n    botId: string\n    workspaceId?: string\n    webhookId?: string\n  }\n  agentPath: string\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n", "/**\n * Response assertion graders.\n * Checks bot response text against expected assertions.\n */\n\nimport type { ResponseAssertion, GraderResult } from '../types'\nimport { gradeLLMJudge } from './llm'\n\nexport async function gradeResponse(\n  botResponse: string,\n  assertions: ResponseAssertion[],\n  context: { userMessage: string; judgePassThreshold?: number }\n): Promise<GraderResult[]> {\n  const results: GraderResult[] = []\n\n  for (const assertion of assertions) {\n    if ('contains' in assertion) {\n      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase())\n      results.push({\n        assertion: `contains \"${assertion.contains}\"`,\n        pass,\n        expected: `Response contains \"${assertion.contains}\"`,\n        actual: pass ? `Found in response` : `Not found in response`,\n      })\n      continue\n    }\n\n    if ('not_contains' in assertion) {\n      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase())\n      results.push({\n        assertion: `not_contains \"${assertion.not_contains}\"`,\n        pass,\n        expected: `Response does not contain \"${assertion.not_contains}\"`,\n        actual: pass ? `Not found in response` : `Found in response`,\n      })\n      continue\n    }\n\n    if ('matches' in assertion) {\n      const regex = new RegExp(assertion.matches, 'i')\n      const pass = regex.test(botResponse)\n      results.push({\n        assertion: `matches ${assertion.matches}`,\n        pass,\n        expected: `Response matches /${assertion.matches}/`,\n        actual: pass ? `Matched` : `No match`,\n      })\n      continue\n    }\n\n    if ('llm_judge' in assertion) {\n      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {\n        userMessage: context.userMessage,\n        ...(context.judgePassThreshold !== undefined ? { passThreshold: context.judgePassThreshold } : {}),\n      })\n      results.push(result)\n      continue\n    }\n\n    // TODO: uncomment this once we support it\n    // if ('similar_to' in assertion) {\n    //   results.push({\n    //     assertion: `similar_to: \"${assertion.similar_to}\"`,\n    //     pass: true,\n    //     expected: assertion.similar_to,\n    //     actual: 'SKIPPED \u2014 similar_to not yet implemented',\n    //   })\n    //   continue\n    // }\n\n    results.push({\n      assertion: 'unknown',\n      pass: false,\n      expected: 'known assertion type',\n      actual: `Unknown assertion: ${JSON.stringify(assertion)}`,\n    })\n  }\n\n  return results\n}\n"],
+  "mappings": ";AAKA,SAAS,cAAc;AACvB,SAAS,iBAAiB;;;ACuJnB,IAAM,gBAA4B;;;ADnJzC,IAAM,yBAAyB;AAE/B,SAAS,uBAAuB,eAA+B;AAC7D,SAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,aAqBI,aAAa;AAC1B;AAEA,IAAM,gBAAgB,CAAC,QAAQ,MAAM;AAErC,IAAI,aAA+B;AACnC,IAAI,mBAA6B,CAAC;AAClC,IAAI,cAAsB;AAC1B,IAAI,OAAmB;AAUvB,SAAS,cAAc,OAAwB;AAC7C,MAAI,cAAc,SAAS,KAAK,EAAG,QAAO;AAC1C,MAAI,iBAAiB,WAAW,EAAG,QAAO;AAE1C,SAAO,iBAAiB,SAAS,KAAK;AACxC;AAqCA,eAAsB,cACpB,aACA,UACA,SACuB;AACvB,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,iBAAiB,sBAAsB,CAAC;AAE1F,MAAI;AAIF,QAAI,CAAC,YAAY;AACf,WAAK,KAAK,8BAA8B,QAAQ,wDAAwD;AACxG,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,CAAC,cAAc,WAAW,GAAG;AAC/B,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAIA,UAAM,EAAE,OAAO,IAAI,MAAM,WAAW,gBAAgB;AAAA,MAClD,OAAO;AAAA,MACP,aAAa;AAAA,MACb,gBAAgB;AAAA,MAChB,cAAc,uBAAuB,SAAS;AAAA,MAC9C,UAAU;AAAA,QACR;AAAA,UACE,MAAM;AAAA,UACN,SAAS,iBAAiB,QAAQ,WAAW;AAAA;AAAA,gBAAqB,WAAW;AAAA;AAAA,YAAiB,QAAQ;AAAA,QACxG;AAAA,MACF;AAAA,IACF,CAAC;AAED,UAAM,aAAa,QAAQ,UAAU,CAAC,GAAG;AACzC,UAAM,UAAU,OAAO,eAAe,WAAW,aAAa;AAC9D,QAAI,CAAC,SAAS;AACZ,WAAK,KAAK,6CAA6C,QAAQ,GAAG;AAClE,aAAO;AAAA,QACL,WAAW,eAAe,QAAQ;AAAA,QAClC,MAAM;AAAA,QACN,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,UAAM,UAAU,KAAK,MAAM,OAAO;AAMlC,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM,QAAQ,SAAS;AAAA,MACvB,UAAU;AAAA,MACV,QAAQ,SAAS,QAAQ,KAAK,kBAAkB,SAAS,YAAO,QAAQ,MAAM;AAAA,IAChF;AAAA,EACF,SAAS,KAAK;AACZ,SAAK,KAAK,yBAAyB,QAAQ,MAAO,IAAc,OAAO,EAAE;AACzE,WAAO;AAAA,MACL,WAAW,eAAe,QAAQ;AAAA,MAClC,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,wCAAoC,IAAc,OAAO;AAAA,IACnE;AAAA,EACF;AACF;;;AEpKA,eAAsB,cACpB,aACA,YACA,SACyB;AACzB,QAAM,UAA0B,CAAC;AAEjC,aAAW,aAAa,YAAY;AAClC,QAAI,cAAc,WAAW;AAC3B,YAAM,OAAO,YAAY,YAAY,EAAE,SAAS,UAAU,SAAS,YAAY,CAAC;AAChF,cAAQ,KAAK;AAAA,QACX,WAAW,aAAa,UAAU,QAAQ;AAAA,QAC1C;AAAA,QACA,UAAU,sBAAsB,UAAU,QAAQ;AAAA,QAClD,QAAQ,OAAO,sBAAsB;AAAA,MACvC,CAAC;AACD;AAAA,IACF;AAEA,QAAI,kBAAkB,WAAW;AAC/B,YAAM,OAAO,CAAC,YAAY,YAAY,EAAE,SAAS,UAAU,aAAa,YAAY,CAAC;AACrF,cAAQ,KAAK;AAAA,QACX,WAAW,iBAAiB,UAAU,YAAY;AAAA,QAClD;AAAA,QACA,UAAU,8BAA8B,UAAU,YAAY;AAAA,QAC9D,QAAQ,OAAO,0BAA0B;AAAA,MAC3C,CAAC;AACD;AAAA,IACF;AAEA,QAAI,aAAa,WAAW;AAC1B,YAAM,QAAQ,IAAI,OAAO,UAAU,SAAS,GAAG;AAC/C,YAAM,OAAO,MAAM,KAAK,WAAW;AACnC,cAAQ,KAAK;AAAA,QACX,WAAW,WAAW,UAAU,OAAO;AAAA,QACvC;AAAA,QACA,UAAU,qBAAqB,UAAU,OAAO;AAAA,QAChD,QAAQ,OAAO,YAAY;AAAA,MAC7B,CAAC;AACD;AAAA,IACF;AAEA,QAAI,eAAe,WAAW;AAC5B,YAAM,SAAS,MAAM,cAAc,aAAa,UAAU,WAAW;AAAA,QACnE,aAAa,QAAQ;AAAA,QACrB,GAAI,QAAQ,uBAAuB,SAAY,EAAE,eAAe,QAAQ,mBAAmB,IAAI,CAAC;AAAA,MAClG,CAAC;AACD,cAAQ,KAAK,MAAM;AACnB;AAAA,IACF;AAaA,YAAQ,KAAK;AAAA,MACX,WAAW;AAAA,MACX,MAAM;AAAA,MACN,UAAU;AAAA,MACV,QAAQ,sBAAsB,KAAK,UAAU,SAAS,CAAC;AAAA,IACzD,CAAC;AAAA,EACH;AAEA,SAAO;AACT;",
   "names": []
 }

package/dist/loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAazD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAkC9E;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAuBjF;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAalG;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,CAe1F"}
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../src/loader.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAczD,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAyC9E;AAED,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA4BjF;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAalG;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,CAe1F"}

package/dist/loader.js

@@ -24,6 +24,49 @@ var Eval = class {
   }
 };
 
+// ../analytics/src/errors.ts
+var AdkError = class extends Error {
+  /**
+   * Marker for cross-module-instance checks — `instanceof` breaks when a
+   * package is duplicated in the dependency graph.
+   * @internal
+   */
+  static __IS_ADK_BASE_ERROR = true;
+  /**
+   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
+   * pin `Code` to a literal union (`class EvalRunnerError extends
+   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
+   * in switches.
+   */
+  code;
+  /**
+   * True for user/environment conditions the developer can fix themselves
+   * (validation, auth, connectivity). False for internal bugs. Defaults to
+   * false so unclassified errors surface as bugs until someone triages them.
+   */
+  expected;
+  /** Structured context. Sanitized before any telemetry leaves the machine. */
+  details;
+  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
+  suggestion;
+  constructor(opts) {
+    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
+    this.name = this.constructor.name;
+    this.code = opts.code;
+    this.expected = opts.expected ?? false;
+    if (opts.details !== void 0) {
+      this.details = opts.details;
+    }
+    if (opts.suggestion !== void 0) {
+      this.suggestion = opts.suggestion;
+    }
+  }
+};
+
+// src/errors.ts
+var EvalRunnerError = class extends AdkError {
+};
+
 // src/loader.ts
 function isEvalDefinition(value) {
   return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
@@ -35,15 +78,17 @@ async function loadEvalFile(filePath) {
     mod = await import(absPath);
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
-    throw new Error(
-      `Failed to load eval file ${filePath}: ${msg}
+    throw new EvalRunnerError({
+      code: "EVAL_LOAD_FAILED",
+      message: `Failed to load eval file ${filePath}: ${msg}
 
 Make sure your eval file:
   - Has no syntax or type errors
   - Exports one or more \`new Eval({...})\` instances
   - Has all dependencies installed (\`bun install\`)`,
-      { cause: err }
-    );
+      expected: true,
+      cause: err
+    });
   }
   const results = [];
   for (const [key, value] of Object.entries(mod)) {
@@ -53,9 +98,11 @@ Make sure your eval file:
     }
   }
   if (results.length === 0) {
-    throw new Error(
-      `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`
-    );
+    throw new EvalRunnerError({
+      code: "EVAL_FILE_EMPTY",
+      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
+      expected: true
+    });
   }
   return results;
 }
@@ -73,7 +120,12 @@ async function loadEvalsFromDir(dirPath) {
   const seen = /* @__PURE__ */ new Set();
   for (const e of evals) {
     if (seen.has(e.name)) {
-      throw new Error(`Duplicate eval name "${e.name}" found in ${dirPath}`);
+      throw new EvalRunnerError({
+        code: "EVAL_DUPLICATE_NAME",
+        message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
+        expected: true,
+        details: { name: e.name }
+      });
     }
     seen.add(e.name);
   }

package/dist/loader.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../src/loader.ts", "../src/definition.ts"],
-  "sourcesContent": ["/**\n * Eval file loader.\n * Loads *.eval.ts files from a directory using dynamic imports.\n * Supports both default and named exports, and multiple evals per file.\n */\n\nimport { readdirSync, existsSync } from 'fs'\nimport { resolve } from 'path'\nimport type { EvalDefinition, EvalFilter } from './types'\nimport { Eval } from './types'\n\nfunction isEvalDefinition(value: unknown): value is EvalDefinition {\n  return (\n    value !== null &&\n    typeof value === 'object' &&\n    typeof (value as Record<string, unknown>).name === 'string' &&\n    (value as Record<string, unknown>).name !== '' &&\n    Array.isArray((value as Record<string, unknown>).conversation)\n  )\n}\n\nexport async function loadEvalFile(filePath: string): Promise<EvalDefinition[]> {\n  const absPath = resolve(filePath)\n\n  let mod: Record<string, unknown>\n  try {\n    mod = await import(absPath)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    throw new Error(\n      `Failed to load eval file ${filePath}: ${msg}\\n\\n` +\n        'Make sure your eval file:\\n' +\n        '  - Has no syntax or type errors\\n' +\n        '  - Exports one or more `new Eval({...})` instances\\n' +\n        '  - Has all dependencies installed (`bun install`)',\n      { cause: err }\n    )\n  }\n\n  const results: EvalDefinition[] = []\n\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === '__esModule') continue\n    if (value instanceof Eval || isEvalDefinition(value)) {\n      results.push(value)\n    }\n  }\n\n  if (results.length === 0) {\n    throw new Error(\n      `Invalid eval file ${filePath}: no valid evals found. Export one or more \\`new Eval({...})\\` instances (as default or named exports).`\n    )\n  }\n\n  return results\n}\n\nexport async function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) {\n    return []\n  }\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n  const evals: EvalDefinition[] = []\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    evals.push(...defs)\n  }\n\n  const seen = new Set<string>()\n  for (const e of evals) {\n    if (seen.has(e.name)) {\n      throw new Error(`Duplicate eval name \"${e.name}\" found in ${dirPath}`)\n    }\n    seen.add(e.name)\n  }\n\n  return evals\n}\n\nexport async function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) return null\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    const found = defs.find((d) => d.name === name)\n    if (found) return found\n  }\n\n  return null\n}\n\nexport function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[] {\n  if (!filter) return evals\n\n  return evals.filter((e) => {\n    if (filter.names && filter.names.length > 0) {\n      if (!filter.names.includes(e.name)) return false\n    }\n    if (filter.tags && filter.tags.length > 0) {\n      if (!e.tags || !filter.tags.some((t) => e.tags!.includes(t))) return false\n    }\n    if (filter.type) {\n      if (e.type !== filter.type) return false\n    }\n    return true\n  })\n}\n", "export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n"],
-  "mappings": ";AAMA,SAAS,aAAa,kBAAkB;AACxC,SAAS,eAAe;;;ACgFjB,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;ADnGA,SAAS,iBAAiB,OAAyC;AACjE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAQ,MAAkC,SAAS,YAClD,MAAkC,SAAS,MAC5C,MAAM,QAAS,MAAkC,YAAY;AAEjE;AAEA,eAAsB,aAAa,UAA6C;AAC9E,QAAM,UAAU,QAAQ,QAAQ;AAEhC,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,OAAO;AAAA,EACrB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,IAAI;AAAA,MACR,4BAA4B,QAAQ,KAAK,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAK5C,EAAE,OAAO,IAAI;AAAA,IACf;AAAA,EACF;AAEA,QAAM,UAA4B,CAAC;AAEnC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,QAAQ,aAAc;AAC1B,QAAI,iBAAiB,QAAQ,iBAAiB,KAAK,GAAG;AACpD,cAAQ,KAAK,KAAK;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI;AAAA,MACR,qBAAqB,QAAQ;AAAA,IAC/B;AAAA,EACF;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAiB,SAA4C;AACjF,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AACtE,QAAM,QAA0B,CAAC;AAEjC,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,KAAK,GAAG,IAAI;AAAA,EACpB;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,OAAO;AACrB,QAAI,KAAK,IAAI,EAAE,IAAI,GAAG;AACpB,YAAM,IAAI,MAAM,wBAAwB,EAAE,IAAI,cAAc,OAAO,EAAE;AAAA,IACvE;AACA,SAAK,IAAI,EAAE,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;AAEA,eAAsB,eAAe,SAAiB,MAA8C;AAClG,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,EAAG,QAAO;AAEhC,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AAEtE,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI;AAC9C,QAAI,MAAO,QAAO;AAAA,EACpB;AAEA,SAAO;AACT;AAEO,SAAS,YAAY,OAAyB,QAAuC;AAC1F,MAAI,CAAC,OAAQ,QAAO;AAEpB,SAAO,MAAM,OAAO,CAAC,MAAM;AACzB,QAAI,OAAO,SAAS,OAAO,MAAM,SAAS,GAAG;AAC3C,UAAI,CAAC,OAAO,MAAM,SAAS,EAAE,IAAI,EAAG,QAAO;AAAA,IAC7C;AACA,QAAI,OAAO,QAAQ,OAAO,KAAK,SAAS,GAAG;AACzC,UAAI,CAAC,EAAE,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,KAAM,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACvE;AACA,QAAI,OAAO,MAAM;AACf,UAAI,EAAE,SAAS,OAAO,KAAM,QAAO;AAAA,IACrC;AACA,WAAO;AAAA,EACT,CAAC;AACH;",
+  "sources": ["../src/loader.ts", "../src/definition.ts", "../../analytics/src/errors.ts", "../src/errors.ts"],
+  "sourcesContent": ["/**\n * Eval file loader.\n * Loads *.eval.ts files from a directory using dynamic imports.\n * Supports both default and named exports, and multiple evals per file.\n */\n\nimport { readdirSync, existsSync } from 'fs'\nimport { resolve } from 'path'\nimport type { EvalDefinition, EvalFilter } from './types'\nimport { Eval } from './types'\nimport { EvalRunnerError } from './errors'\n\nfunction isEvalDefinition(value: unknown): value is EvalDefinition {\n  return (\n    value !== null &&\n    typeof value === 'object' &&\n    typeof (value as Record<string, unknown>).name === 'string' &&\n    (value as Record<string, unknown>).name !== '' &&\n    Array.isArray((value as Record<string, unknown>).conversation)\n  )\n}\n\nexport async function loadEvalFile(filePath: string): Promise<EvalDefinition[]> {\n  const absPath = resolve(filePath)\n\n  let mod: Record<string, unknown>\n  try {\n    mod = await import(absPath)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    // Guidance stays in the message until the CLI renders `suggestion`\n    // (cli-package remediation) \u2014 report.error/logger.fatal only show message.\n    throw new EvalRunnerError({\n      code: 'EVAL_LOAD_FAILED',\n      message:\n        `Failed to load eval file ${filePath}: ${msg}\\n\\n` +\n        'Make sure your eval file:\\n' +\n        '  - Has no syntax or type errors\\n' +\n        '  - Exports one or more `new Eval({...})` instances\\n' +\n        '  - Has all dependencies installed (`bun install`)',\n      expected: true,\n      cause: err,\n    })\n  }\n\n  const results: EvalDefinition[] = []\n\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === '__esModule') continue\n    if (value instanceof Eval || isEvalDefinition(value)) {\n      results.push(value)\n    }\n  }\n\n  if (results.length === 0) {\n    throw new EvalRunnerError({\n      code: 'EVAL_FILE_EMPTY',\n      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \\`new Eval({...})\\` instances (as default or named exports).`,\n      expected: true,\n    })\n  }\n\n  return results\n}\n\nexport async function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) {\n    return []\n  }\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n  const evals: EvalDefinition[] = []\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    evals.push(...defs)\n  }\n\n  const seen = new Set<string>()\n  for (const e of evals) {\n    if (seen.has(e.name)) {\n      throw new EvalRunnerError({\n        code: 'EVAL_DUPLICATE_NAME',\n        message: `Duplicate eval name \"${e.name}\" found in ${dirPath} \u2014 names must be unique across the evals directory.`,\n        expected: true,\n        details: { name: e.name },\n      })\n    }\n    seen.add(e.name)\n  }\n\n  return evals\n}\n\nexport async function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) return null\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    const found = defs.find((d) => d.name === name)\n    if (found) return found\n  }\n\n  return null\n}\n\nexport function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[] {\n  if (!filter) return evals\n\n  return evals.filter((e) => {\n    if (filter.names && filter.names.length > 0) {\n      if (!filter.names.includes(e.name)) return false\n    }\n    if (filter.tags && filter.tags.length > 0) {\n      if (!e.tags || !filter.tags.some((t) => e.tags!.includes(t))) return false\n    }\n    if (filter.type) {\n      if (e.type !== filter.type) return false\n    }\n    return true\n  })\n}\n", "export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n", "/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity). False for internal bugs. Defaults to\n   * false so unclassified errors surface as bugs until someone triages them.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? false\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n"],
+  "mappings": ";AAMA,SAAS,aAAa,kBAAkB;AACxC,SAAS,eAAe;;;ACgFjB,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;AC7FO,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;ACpCO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;AHhB9D,SAAS,iBAAiB,OAAyC;AACjE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAQ,MAAkC,SAAS,YAClD,MAAkC,SAAS,MAC5C,MAAM,QAAS,MAAkC,YAAY;AAEjE;AAEA,eAAsB,aAAa,UAA6C;AAC9E,QAAM,UAAU,QAAQ,QAAQ;AAEhC,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,OAAO;AAAA,EACrB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAG3D,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SACE,4BAA4B,QAAQ,KAAK,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAK9C,UAAU;AAAA,MACV,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,QAAM,UAA4B,CAAC;AAEnC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,QAAQ,aAAc;AAC1B,QAAI,iBAAiB,QAAQ,iBAAiB,KAAK,GAAG;AACpD,cAAQ,KAAK,KAAK;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS,qBAAqB,QAAQ;AAAA,MACtC,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAiB,SAA4C;AACjF,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AACtE,QAAM,QAA0B,CAAC;AAEjC,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,KAAK,GAAG,IAAI;AAAA,EACpB;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,OAAO;AACrB,QAAI,KAAK,IAAI,EAAE,IAAI,GAAG;AACpB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS,wBAAwB,EAAE,IAAI,cAAc,OAAO;AAAA,QAC5D,UAAU;AAAA,QACV,SAAS,EAAE,MAAM,EAAE,KAAK;AAAA,MAC1B,CAAC;AAAA,IACH;AACA,SAAK,IAAI,EAAE,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;AAEA,eAAsB,eAAe,SAAiB,MAA8C;AAClG,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,EAAG,QAAO;AAEhC,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AAEtE,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI;AAC9C,QAAI,MAAO,QAAO;AAAA,EACpB;AAEA,SAAO;AACT;AAEO,SAAS,YAAY,OAAyB,QAAuC;AAC1F,MAAI,CAAC,OAAQ,QAAO;AAEpB,SAAO,MAAM,OAAO,CAAC,MAAM;AACzB,QAAI,OAAO,SAAS,OAAO,MAAM,SAAS,GAAG;AAC3C,UAAI,CAAC,OAAO,MAAM,SAAS,EAAE,IAAI,EAAG,QAAO;AAAA,IAC7C;AACA,QAAI,OAAO,QAAQ,OAAO,KAAK,SAAS,GAAG;AACzC,UAAI,CAAC,EAAE,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,KAAM,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACvE;AACA,QAAI,OAAO,MAAM;AACf,UAAI,EAAE,SAAS,OAAO,KAAM,QAAO;AAAA,IACrC;AACA,WAAO;AAAA,EACT,CAAC;AACH;",
   "names": []
 }

package/dist/runner.d.ts

@@ -17,6 +17,8 @@ export declare function runEval(evalDef: EvalDefinition, connection: BotConnecti
     signal?: AbortSignal;
     devServerHeaders?: Record<string, string>;
     chatClient?: EvalRunnerConfig['chatClient'];
+    logger?: EvalRunnerConfig['logger'];
+    onException?: EvalRunnerConfig['onException'];
 }): Promise<EvalReport>;
 /**
  * Run a suite of evals.

package/dist/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EAGV,aAAa,EAEd,MAAM,SAAS,CAAA;AAgHhB,wBAAsB,OAAO,CAC3B,OAAO,EAAE,cAAc,EACvB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAA;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,+IAA+I;IAC/I,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,UAAU,CAAC,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAA;CACvC,GACL,OAAO,CAAC,UAAU,CAAC,CA2RrB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CA6GxG"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EAGV,aAAa,EAEd,MAAM,SAAS,CAAA;AAsHhB,wBAAsB,OAAO,CAC3B,OAAO,EAAE,cAAc,EACvB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAA;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,+IAA+I;IAC/I,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,UAAU,CAAC,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAA;IAC3C,MAAM,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACnC,WAAW,CAAC,EAAE,gBAAgB,CAAC,aAAa,CAAC,CAAA;CACzC,GACL,OAAO,CAAC,UAAU,CAAC,CA+TrB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CA+GxG"}