@botpress/evals 2.0.3 → 3.0.0-beta.0

package/dist/client.js

@@ -18,8 +18,9 @@ var AdkError = class extends Error {
   code;
   /**
    * True for user/environment conditions the developer can fix themselves
-   * (validation, auth, connectivity). False for internal bugs. Defaults to
-   * false so unclassified errors surface as bugs until someone triages them.
+   * (validation, auth, connectivity) — reported as `error.expected` events.
+   * False for internal bugs — reported as PostHog `$exception`s. Defaults to
+   * **true**; set `expected: false` explicitly to flag a genuine bug.
    */
   expected;
   /** Structured context. Sanitized before any telemetry leaves the machine. */
@@ -30,7 +31,7 @@ var AdkError = class extends Error {
     super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
     this.name = this.constructor.name;
     this.code = opts.code;
-    this.expected = opts.expected ?? false;
+    this.expected = opts.expected ?? true;
     if (opts.details !== void 0) {
       this.details = opts.details;
     }

package/dist/client.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/types.ts", "../../analytics/src/errors.ts", "../src/errors.ts", "../src/client.ts"],
-  "sourcesContent": ["/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nimport type { EvalDefinition as _EvalDefinition } from './definition'\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './spans/trace'\nexport type { WaitOptions, WorkflowWaitOptions } from './spans/span-source'\nimport type { SpanSource } from './spans/span-source'\nexport type { SpanSource }\n\n/**\n * The @botpress/chat Client *class* (constructor) \u2014 not an instance. Callers\n * inject it (the CLI passes its bundled chat client) and the runner calls the\n * static `.connect()`. Typed via `typeof import(...)` so this types-only module\n * doesn't pull in a value import.\n */\nexport type ChatClient = typeof import('@botpress/chat').Client\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n  /**\n   * True when the assertion could not actually be evaluated (e.g. the llm_judge had no credentials,\n   * returned an empty response, or errored). Distinguishes a genuine pass from a \"didn't run\" pass.\n   * Callers that demand a trustworthy verdict (e.g. the benchmark) should treat `skipped` as a failure.\n   */\n  skipped?: boolean\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnIndex: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnIndex: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnIndex: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  client: import('@botpress/client').Client\n  botId: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  /** Authenticated Botpress client. Used for state seeding, workflow triggers, webhook discovery, and LLM judge. */\n  client: import('@botpress/client').Client\n  botId: string\n  /** Path to the agent directory. Used to discover eval files from `<agentPath>/evals/`. Not needed when `definitions` is provided. */\n  agentPath?: string\n  /** Pre-loaded eval definitions. When provided, `agentPath` is not used for eval discovery. */\n  definitions?: _EvalDefinition[]\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void | Promise<void>\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n  /** Factory for creating span sources. Each eval gets its own instance. Defaults to LocalSpanSource (SSE from dev server). */\n  createSpanSource?: () => SpanSource\n  /** Externally-provided run ID. When set, the runner uses this instead of generating its own. */\n  runId?: string\n  /** Pre-resolved chat integration webhook ID. Skips the getBot() discovery call when provided. */\n  chatWebhookId?: string\n  /** Base URL for the chat service (e.g. https://chat.botpress.dev). Defaults to https://chat.botpress.cloud. */\n  chatBaseUrl?: string\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n", "/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity). False for internal bugs. Defaults to\n   * false so unclassified errors surface as bugs until someone triages them.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? false\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n", "/**\n * Chat client for eval conversations.\n * Drives conversations against a running ADK bot via @botpress/chat.\n *\n * This is a send-only client. Observation (bot responses, tool calls, state)\n * is handled by the SSECollector via trace spans \u2014 not via WebSocket listeners.\n */\n\nimport { Client as BpClient } from '@botpress/client'\nimport type { AuthenticatedClient as AuthedChatClient } from '@botpress/chat'\nimport type { ChatClient, EvalLogger } from './types'\nimport { defaultLogger } from './types'\nimport { EvalRunnerError } from './errors'\n\nconst TRACE_STREAM_PREFLIGHT_TIMEOUT_MS = 10_000\n\n/**\n * A send-only chat session that maintains a single client connection across turns.\n * Messages and events are fire-and-forget; the SSECollector observes results via traces.\n */\nexport class ChatSession {\n  private client: AuthedChatClient | null = null\n  private conversationId: string | null = null\n\n  constructor(\n    private bpClient: BpClient,\n    private botId: string,\n    private _chatWebhookId?: string,\n    private _chatBaseUrl?: string,\n    private _chatClient?: ChatClient\n  ) {}\n\n  async connect() {\n    const webhookId = this._chatWebhookId ?? (await discoverWebhookId(this.bpClient, this.botId))\n\n    // Prefer the host-injected client. The CLI passes its bundled CJS chat client\n    // (working node http adapter); only fall back to a dynamic import when nothing\n    // was injected \u2014 e.g. the cloud runtime, where @botpress/chat resolves to a\n    // real node_modules dep. In a bun-compiled CLI binary the dynamic import\n    // resolves to the browser ESM bundle whose axios http adapter is null\n    // (\"Adapter 'http' is not available in the build\"), so the injected client is\n    // required there.\n    const ChatClientCtor =\n      this._chatClient ?? (await import(/* webpackIgnore: true */ '@botpress/chat' as string)).Client\n\n    this.client = await ChatClientCtor.connect({\n      webhookId,\n      ...(this._chatBaseUrl ? { baseApiUrl: this._chatBaseUrl } : {}),\n    })\n  }\n\n  /** Invariant guard \u2014 calling any session method before connect() is a runner bug. */\n  private assertConnected(): AuthedChatClient {\n    if (!this.client) {\n      throw new EvalRunnerError({\n        code: 'CHAT_NOT_CONNECTED',\n        message: 'ChatSession not connected. Call connect() first.',\n      })\n    }\n    return this.client\n  }\n\n  get userId(): string {\n    return this.assertConnected().user.id\n  }\n\n  async ensureConversation(): Promise<string> {\n    const client = this.assertConnected()\n\n    if (!this.conversationId) {\n      const conv = await client.createConversation({})\n      this.conversationId = conv.conversation.id\n      return conv.conversation.id\n    }\n\n    return this.conversationId\n  }\n\n  /**\n   * Open a fresh conversation under the same user (the user is fixed at\n   * connect()) and make it active. Returns the new conversation id.\n   */\n  async newConversation(): Promise<string> {\n    const client = this.assertConnected()\n    const conv = await client.createConversation({})\n    this.conversationId = conv.conversation.id\n    return conv.conversation.id\n  }\n\n  /**\n   * Send a user message. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendMessage(message: string): Promise<void> {\n    const client = this.assertConnected()\n    const conversationId = await this.ensureConversation()\n    await client.createMessage({ conversationId, payload: { type: 'text', text: message } })\n  }\n\n  async sendEvent(payload: Record<string, unknown>): Promise<void> {\n    const client = this.assertConnected()\n    const conversationId = await this.ensureConversation()\n    await client.createEvent({ payload, conversationId })\n  }\n}\n\n/**\n * Discover the chat integration's webhookId from a bot.\n *\n * Evals talk to the bot through the `chat` integration (via @botpress/chat).\n * If the bot doesn't have that integration installed, evals can't run.\n */\nexport async function discoverWebhookId(client: BpClient, botId: string): Promise<string> {\n  const { bot } = await client.getBot({ id: botId })\n  const integrations = bot.integrations || {}\n\n  const chat = Object.values(integrations).find((int) => (int as Record<string, unknown>).name === 'chat')\n  const webhookId = (chat as Record<string, unknown> | undefined)?.webhookId\n\n  if (!webhookId || typeof webhookId !== 'string') {\n    throw new EvalRunnerError({\n      code: 'CHAT_INTEGRATION_MISSING',\n      message: [\n        'The `chat` integration is not installed on this bot \u2014 evals require it.',\n        '',\n        'To fix:',\n        '  1. Install it:    adk integrations add chat',\n        '  2. Redeploy:      adk dev   (or: adk deploy)',\n      ].join('\\n'),\n      expected: true,\n    })\n  }\n\n  return webhookId\n}\n\n/**\n * Verify the dev server's trace stream can be read before eval turns start.\n *\n * This only checks that `/api/traces/stream` is reachable and emits its initial\n * snapshot. It does not prove the private runtime span-ingest path is healthy.\n */\nexport async function assertTraceStreamReadable(\n  devServerUrl: string,\n  headers: Record<string, string> = {},\n  timeoutMs = TRACE_STREAM_PREFLIGHT_TIMEOUT_MS\n): Promise<void> {\n  const url = `${devServerUrl.replace(/\\/$/, '')}/api/traces/stream?count=1`\n  const abortController = new AbortController()\n  let timedOut = false\n  let reader: ReadableStreamDefaultReader<Uint8Array> | null = null\n\n  const timer = setTimeout(() => {\n    timedOut = true\n    abortController.abort()\n  }, timeoutMs)\n\n  try {\n    const res = await fetch(url, {\n      headers: { ...headers, Accept: 'text/event-stream' },\n      signal: abortController.signal,\n    })\n\n    if (!res.ok) {\n      throw traceStreamPreflightError(`Trace stream pre-flight failed: ${res.status} ${res.statusText}`)\n    }\n\n    if (!res.body) {\n      throw traceStreamPreflightError('Trace stream pre-flight failed: response had no body.')\n    }\n\n    reader = res.body.getReader()\n    const decoder = new TextDecoder()\n    let buffer = ''\n\n    while (true) {\n      const { done, value } = await reader.read()\n      if (done) {\n        throw traceStreamPreflightError('Trace stream pre-flight failed: stream closed before snapshot.')\n      }\n\n      buffer += decoder.decode(value, { stream: true })\n      const parts = buffer.split('\\n\\n')\n      buffer = parts.pop() ?? ''\n\n      for (const part of parts) {\n        if (parseSSEEventName(part) === 'snapshot') {\n          return\n        }\n      }\n    }\n  } catch (err) {\n    if (err instanceof EvalRunnerError) {\n      throw err\n    }\n\n    if (timedOut) {\n      throw traceStreamPreflightError(\n        `Trace stream pre-flight timed out after ${timeoutMs}ms before receiving a snapshot.`\n      )\n    }\n\n    throw traceStreamPreflightError(\n      `Trace stream pre-flight failed: ${err instanceof Error ? err.message : String(err)}`\n    )\n  } finally {\n    clearTimeout(timer)\n    reader?.cancel().catch(() => {})\n  }\n}\n\nfunction parseSSEEventName(block: string): string | null {\n  for (const line of block.split('\\n')) {\n    if (line.startsWith('event:')) {\n      return line.slice('event:'.length).trim()\n    }\n  }\n  return null\n}\n\nfunction traceStreamPreflightError(message: string): EvalRunnerError {\n  return new EvalRunnerError({\n    code: 'SSE_CONNECT_FAILED',\n    message,\n    expected: true,\n    suggestion: 'Make sure the dev server is running (`adk dev`) and trace streaming is available.',\n  })\n}\n\n/**\n * Verify the bot has at least one Conversation bound to `chat.channel`.\n *\n * The `chat` integration can be installed without any handler listening on its\n * channel \u2014 in that case messages sent via @botpress/chat produce no response\n * and the caller times out silently. Fail fast with actionable guidance instead.\n *\n * Uses the dev server's `/api/agent` endpoint, so this is a no-op against\n * production bots (no devServerUrl).\n */\nexport async function assertChatChannelBound(\n  devServerUrl: string,\n  headers: Record<string, string> = {},\n  logger: EvalLogger = defaultLogger\n): Promise<void> {\n  type ConversationSummary = { channel?: string | string[] }\n  let conversations: ConversationSummary[] = []\n  try {\n    const res = await fetch(`${devServerUrl}/api/agent`, { headers })\n    if (!res.ok) return // Dev server unreachable \u2014 skip, let downstream errors surface\n    const agent = (await res.json()) as { conversations?: ConversationSummary[] }\n    conversations = agent.conversations ?? []\n  } catch (err) {\n    // Soft pre-flight only \u2014 a missing dev server is normal (e.g. prod bots),\n    // but log the skip so a *broken* dev server (malformed /api/agent JSON)\n    // isn't invisible while every eval afterwards times out mysteriously.\n    logger?.warn(\n      `Skipping chat.channel binding pre-flight \u2014 could not query ${devServerUrl}/api/agent: ${err instanceof Error ? err.message : String(err)}`\n    )\n    return\n  }\n\n  if (conversations.length === 0) return // Bot has no conversations at all \u2014 different problem\n\n  // Mirrors the runtime's matchesChannel semantics: a handler matches `chat.channel`\n  // if it is the wildcard '*', the literal string 'chat.channel', or an array\n  // containing either.\n  const matchesChatChannel = (ch: string | string[] | undefined): boolean => {\n    if (!ch) return false\n    if (ch === '*' || ch === 'chat.channel') return true\n    return Array.isArray(ch) && (ch.includes('chat.channel') || ch.includes('*'))\n  }\n\n  const bound = conversations.some((c) => matchesChatChannel(c.channel))\n  if (bound) return\n\n  throw new EvalRunnerError({\n    code: 'CHAT_CHANNEL_UNBOUND',\n    message: [\n      'No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.',\n      '',\n      'To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):',\n      \"  channel: 'chat.channel',           // explicit\",\n      \"  channel: '*',                      // wildcard \u2014 matches every channel\",\n      \"  channel: ['chat.channel', ...],    // array form\",\n      '',\n      'The `chat` integration is how evals and `adk chat --single` send messages programmatically.',\n    ].join('\\n'),\n    expected: true,\n  })\n}\n"],
-  "mappings": ";AAiKO,IAAM,gBAA4B;;;AChJlC,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;ACpCO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;ACd9D,IAAM,oCAAoC;AAMnC,IAAM,cAAN,MAAkB;AAAA,EAIvB,YACU,UACA,OACA,gBACA,cACA,aACR;AALQ;AACA;AACA;AACA;AACA;AAAA,EACP;AAAA,EATK,SAAkC;AAAA,EAClC,iBAAgC;AAAA,EAUxC,MAAM,UAAU;AACd,UAAM,YAAY,KAAK,kBAAmB,MAAM,kBAAkB,KAAK,UAAU,KAAK,KAAK;AAS3F,UAAM,iBACJ,KAAK,gBAAgB,MAAM;AAAA;AAAA,MAAiC;AAAA,IAA0B,GAAG;AAE3F,SAAK,SAAS,MAAM,eAAe,QAAQ;AAAA,MACzC;AAAA,MACA,GAAI,KAAK,eAAe,EAAE,YAAY,KAAK,aAAa,IAAI,CAAC;AAAA,IAC/D,CAAC;AAAA,EACH;AAAA;AAAA,EAGQ,kBAAoC;AAC1C,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,gBAAgB,EAAE,KAAK;AAAA,EACrC;AAAA,EAEA,MAAM,qBAAsC;AAC1C,UAAM,SAAS,KAAK,gBAAgB;AAEpC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,OAAO,MAAM,OAAO,mBAAmB,CAAC,CAAC;AAC/C,WAAK,iBAAiB,KAAK,aAAa;AACxC,aAAO,KAAK,aAAa;AAAA,IAC3B;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,kBAAmC;AACvC,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,OAAO,MAAM,OAAO,mBAAmB,CAAC,CAAC;AAC/C,SAAK,iBAAiB,KAAK,aAAa;AACxC,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,YAAY,SAAgC;AAChD,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,OAAO,cAAc,EAAE,gBAAgB,SAAS,EAAE,MAAM,QAAQ,MAAM,QAAQ,EAAE,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,UAAU,SAAiD;AAC/D,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,OAAO,YAAY,EAAE,SAAS,eAAe,CAAC;AAAA,EACtD;AACF;AAQA,eAAsB,kBAAkB,QAAkB,OAAgC;AACxF,QAAM,EAAE,IAAI,IAAI,MAAM,OAAO,OAAO,EAAE,IAAI,MAAM,CAAC;AACjD,QAAM,eAAe,IAAI,gBAAgB,CAAC;AAE1C,QAAM,OAAO,OAAO,OAAO,YAAY,EAAE,KAAK,CAAC,QAAS,IAAgC,SAAS,MAAM;AACvG,QAAM,YAAa,MAA8C;AAEjE,MAAI,CAAC,aAAa,OAAO,cAAc,UAAU;AAC/C,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,MACX,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAQA,eAAsB,0BACpB,cACA,UAAkC,CAAC,GACnC,YAAY,mCACG;AACf,QAAM,MAAM,GAAG,aAAa,QAAQ,OAAO,EAAE,CAAC;AAC9C,QAAM,kBAAkB,IAAI,gBAAgB;AAC5C,MAAI,WAAW;AACf,MAAI,SAAyD;AAE7D,QAAM,QAAQ,WAAW,MAAM;AAC7B,eAAW;AACX,oBAAgB,MAAM;AAAA,EACxB,GAAG,SAAS;AAEZ,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,SAAS,EAAE,GAAG,SAAS,QAAQ,oBAAoB;AAAA,MACnD,QAAQ,gBAAgB;AAAA,IAC1B,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,0BAA0B,mCAAmC,IAAI,MAAM,IAAI,IAAI,UAAU,EAAE;AAAA,IACnG;AAEA,QAAI,CAAC,IAAI,MAAM;AACb,YAAM,0BAA0B,uDAAuD;AAAA,IACzF;AAEA,aAAS,IAAI,KAAK,UAAU;AAC5B,UAAM,UAAU,IAAI,YAAY;AAChC,QAAI,SAAS;AAEb,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,MAAM;AACR,cAAM,0BAA0B,gEAAgE;AAAA,MAClG;AAEA,gBAAU,QAAQ,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;AAChD,YAAM,QAAQ,OAAO,MAAM,MAAM;AACjC,eAAS,MAAM,IAAI,KAAK;AAExB,iBAAW,QAAQ,OAAO;AACxB,YAAI,kBAAkB,IAAI,MAAM,YAAY;AAC1C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,eAAe,iBAAiB;AAClC,YAAM;AAAA,IACR;AAEA,QAAI,UAAU;AACZ,YAAM;AAAA,QACJ,2CAA2C,SAAS;AAAA,MACtD;AAAA,IACF;AAEA,UAAM;AAAA,MACJ,mCAAmC,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IACrF;AAAA,EACF,UAAE;AACA,iBAAa,KAAK;AAClB,YAAQ,OAAO,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACjC;AACF;AAEA,SAAS,kBAAkB,OAA8B;AACvD,aAAW,QAAQ,MAAM,MAAM,IAAI,GAAG;AACpC,QAAI,KAAK,WAAW,QAAQ,GAAG;AAC7B,aAAO,KAAK,MAAM,SAAS,MAAM,EAAE,KAAK;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,0BAA0B,SAAkC;AACnE,SAAO,IAAI,gBAAgB;AAAA,IACzB,MAAM;AAAA,IACN;AAAA,IACA,UAAU;AAAA,IACV,YAAY;AAAA,EACd,CAAC;AACH;AAYA,eAAsB,uBACpB,cACA,UAAkC,CAAC,GACnC,SAAqB,eACN;AAEf,MAAI,gBAAuC,CAAC;AAC5C,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,YAAY,cAAc,EAAE,QAAQ,CAAC;AAChE,QAAI,CAAC,IAAI,GAAI;AACb,UAAM,QAAS,MAAM,IAAI,KAAK;AAC9B,oBAAgB,MAAM,iBAAiB,CAAC;AAAA,EAC1C,SAAS,KAAK;AAIZ,YAAQ;AAAA,MACN,mEAA8D,YAAY,eAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IAC3I;AACA;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,EAAG;AAKhC,QAAM,qBAAqB,CAAC,OAA+C;AACzE,QAAI,CAAC,GAAI,QAAO;AAChB,QAAI,OAAO,OAAO,OAAO,eAAgB,QAAO;AAChD,WAAO,MAAM,QAAQ,EAAE,MAAM,GAAG,SAAS,cAAc,KAAK,GAAG,SAAS,GAAG;AAAA,EAC7E;AAEA,QAAM,QAAQ,cAAc,KAAK,CAAC,MAAM,mBAAmB,EAAE,OAAO,CAAC;AACrE,MAAI,MAAO;AAEX,QAAM,IAAI,gBAAgB;AAAA,IACxB,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ,CAAC;AACH;",
+  "sourcesContent": ["/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nimport type { EvalDefinition as _EvalDefinition } from './definition'\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './spans/trace'\nexport type { WaitOptions, WorkflowWaitOptions } from './spans/span-source'\nimport type { SpanSource } from './spans/span-source'\nexport type { SpanSource }\n\n/**\n * The @botpress/chat Client *class* (constructor) \u2014 not an instance. Callers\n * inject it (the CLI passes its bundled chat client) and the runner calls the\n * static `.connect()`. Typed via `typeof import(...)` so this types-only module\n * doesn't pull in a value import.\n */\nexport type ChatClient = typeof import('@botpress/chat').Client\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n  /**\n   * True when the assertion could not actually be evaluated (e.g. the llm_judge had no credentials,\n   * returned an empty response, or errored). Distinguishes a genuine pass from a \"didn't run\" pass.\n   * Callers that demand a trustworthy verdict (e.g. the benchmark) should treat `skipped` as a failure.\n   */\n  skipped?: boolean\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnIndex: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnIndex: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnIndex: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  client: import('@botpress/client').Client\n  botId: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  /** Authenticated Botpress client. Used for state seeding, workflow triggers, webhook discovery, and LLM judge. */\n  client: import('@botpress/client').Client\n  botId: string\n  /** Path to the agent directory. Used to discover eval files from `<agentPath>/evals/`. Not needed when `definitions` is provided. */\n  agentPath?: string\n  /** Pre-loaded eval definitions. When provided, `agentPath` is not used for eval discovery. */\n  definitions?: _EvalDefinition[]\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void | Promise<void>\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n  /** Factory for creating span sources. Each eval gets its own instance. Defaults to LocalSpanSource (SSE from dev server). */\n  createSpanSource?: () => SpanSource\n  /** Externally-provided run ID. When set, the runner uses this instead of generating its own. */\n  runId?: string\n  /** Pre-resolved chat integration webhook ID. Skips the getBot() discovery call when provided. */\n  chatWebhookId?: string\n  /** Base URL for the chat service (e.g. https://chat.botpress.dev). Defaults to https://chat.botpress.cloud. */\n  chatBaseUrl?: string\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n", "/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * `expected` defaults to **true**: the overwhelming majority of thrown errors\n * are user/environment conditions, and expected errors are reported as plain\n * `error.expected` analytics events rather than PostHog `$exception`s. Genuine\n * bugs must opt out explicitly with `expected: false` so they reach Error\n * Tracking. There is a single base class \u2014 everything thrown is an `AdkError`\n * or a subclass.\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity) \u2014 reported as `error.expected` events.\n   * False for internal bugs \u2014 reported as PostHog `$exception`s. Defaults to\n   * **true**; set `expected: false` explicitly to flag a genuine bug.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? true\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n\n/**\n * True for an `AdkError` (or subclass) flagged `expected` \u2014 a user/environment\n * condition that should be reported as an `error.expected` analytics event\n * rather than a PostHog `$exception`. Used by the telemetry capture funnel to\n * route errors. Non-`AdkError` throws (and `expected: false` bugs) are not\n * expected and keep flowing to Error Tracking.\n */\nexport function isExpectedError(error: unknown): error is AdkError {\n  return isAdkError(error) && error.expected\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n", "/**\n * Chat client for eval conversations.\n * Drives conversations against a running ADK bot via @botpress/chat.\n *\n * This is a send-only client. Observation (bot responses, tool calls, state)\n * is handled by the SSECollector via trace spans \u2014 not via WebSocket listeners.\n */\n\nimport { Client as BpClient } from '@botpress/client'\nimport type { AuthenticatedClient as AuthedChatClient } from '@botpress/chat'\nimport type { ChatClient, EvalLogger } from './types'\nimport { defaultLogger } from './types'\nimport { EvalRunnerError } from './errors'\n\nconst TRACE_STREAM_PREFLIGHT_TIMEOUT_MS = 10_000\n\n/**\n * A send-only chat session that maintains a single client connection across turns.\n * Messages and events are fire-and-forget; the SSECollector observes results via traces.\n */\nexport class ChatSession {\n  private client: AuthedChatClient | null = null\n  private conversationId: string | null = null\n\n  constructor(\n    private bpClient: BpClient,\n    private botId: string,\n    private _chatWebhookId?: string,\n    private _chatBaseUrl?: string,\n    private _chatClient?: ChatClient\n  ) {}\n\n  async connect() {\n    const webhookId = this._chatWebhookId ?? (await discoverWebhookId(this.bpClient, this.botId))\n\n    // Prefer the host-injected client. The CLI passes its bundled CJS chat client\n    // (working node http adapter); only fall back to a dynamic import when nothing\n    // was injected \u2014 e.g. the cloud runtime, where @botpress/chat resolves to a\n    // real node_modules dep. In a bun-compiled CLI binary the dynamic import\n    // resolves to the browser ESM bundle whose axios http adapter is null\n    // (\"Adapter 'http' is not available in the build\"), so the injected client is\n    // required there.\n    const ChatClientCtor =\n      this._chatClient ?? (await import(/* webpackIgnore: true */ '@botpress/chat' as string)).Client\n\n    this.client = await ChatClientCtor.connect({\n      webhookId,\n      ...(this._chatBaseUrl ? { baseApiUrl: this._chatBaseUrl } : {}),\n    })\n  }\n\n  /** Invariant guard \u2014 calling any session method before connect() is a runner bug. */\n  private assertConnected(): AuthedChatClient {\n    if (!this.client) {\n      throw new EvalRunnerError({\n        code: 'CHAT_NOT_CONNECTED',\n        message: 'ChatSession not connected. Call connect() first.',\n      })\n    }\n    return this.client\n  }\n\n  get userId(): string {\n    return this.assertConnected().user.id\n  }\n\n  async ensureConversation(): Promise<string> {\n    const client = this.assertConnected()\n\n    if (!this.conversationId) {\n      const conv = await client.createConversation({})\n      this.conversationId = conv.conversation.id\n      return conv.conversation.id\n    }\n\n    return this.conversationId\n  }\n\n  /**\n   * Open a fresh conversation under the same user (the user is fixed at\n   * connect()) and make it active. Returns the new conversation id.\n   */\n  async newConversation(): Promise<string> {\n    const client = this.assertConnected()\n    const conv = await client.createConversation({})\n    this.conversationId = conv.conversation.id\n    return conv.conversation.id\n  }\n\n  /**\n   * Send a user message. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendMessage(message: string): Promise<void> {\n    const client = this.assertConnected()\n    const conversationId = await this.ensureConversation()\n    await client.createMessage({ conversationId, payload: { type: 'text', text: message } })\n  }\n\n  async sendEvent(payload: Record<string, unknown>): Promise<void> {\n    const client = this.assertConnected()\n    const conversationId = await this.ensureConversation()\n    await client.createEvent({ payload, conversationId })\n  }\n}\n\n/**\n * Discover the chat integration's webhookId from a bot.\n *\n * Evals talk to the bot through the `chat` integration (via @botpress/chat).\n * If the bot doesn't have that integration installed, evals can't run.\n */\nexport async function discoverWebhookId(client: BpClient, botId: string): Promise<string> {\n  const { bot } = await client.getBot({ id: botId })\n  const integrations = bot.integrations || {}\n\n  const chat = Object.values(integrations).find((int) => (int as Record<string, unknown>).name === 'chat')\n  const webhookId = (chat as Record<string, unknown> | undefined)?.webhookId\n\n  if (!webhookId || typeof webhookId !== 'string') {\n    throw new EvalRunnerError({\n      code: 'CHAT_INTEGRATION_MISSING',\n      message: [\n        'The `chat` integration is not installed on this bot \u2014 evals require it.',\n        '',\n        'To fix:',\n        '  1. Install it:    adk integrations add chat',\n        '  2. Redeploy:      adk dev   (or: adk deploy)',\n      ].join('\\n'),\n      expected: true,\n    })\n  }\n\n  return webhookId\n}\n\n/**\n * Verify the dev server's trace stream can be read before eval turns start.\n *\n * This only checks that `/api/traces/stream` is reachable and emits its initial\n * snapshot. It does not prove the private runtime span-ingest path is healthy.\n */\nexport async function assertTraceStreamReadable(\n  devServerUrl: string,\n  headers: Record<string, string> = {},\n  timeoutMs = TRACE_STREAM_PREFLIGHT_TIMEOUT_MS\n): Promise<void> {\n  const url = `${devServerUrl.replace(/\\/$/, '')}/api/traces/stream?count=1`\n  const abortController = new AbortController()\n  let timedOut = false\n  let reader: ReadableStreamDefaultReader<Uint8Array> | null = null\n\n  const timer = setTimeout(() => {\n    timedOut = true\n    abortController.abort()\n  }, timeoutMs)\n\n  try {\n    const res = await fetch(url, {\n      headers: { ...headers, Accept: 'text/event-stream' },\n      signal: abortController.signal,\n    })\n\n    if (!res.ok) {\n      throw traceStreamPreflightError(`Trace stream pre-flight failed: ${res.status} ${res.statusText}`)\n    }\n\n    if (!res.body) {\n      throw traceStreamPreflightError('Trace stream pre-flight failed: response had no body.')\n    }\n\n    reader = res.body.getReader()\n    const decoder = new TextDecoder()\n    let buffer = ''\n\n    while (true) {\n      const { done, value } = await reader.read()\n      if (done) {\n        throw traceStreamPreflightError('Trace stream pre-flight failed: stream closed before snapshot.')\n      }\n\n      buffer += decoder.decode(value, { stream: true })\n      const parts = buffer.split('\\n\\n')\n      buffer = parts.pop() ?? ''\n\n      for (const part of parts) {\n        if (parseSSEEventName(part) === 'snapshot') {\n          return\n        }\n      }\n    }\n  } catch (err) {\n    if (err instanceof EvalRunnerError) {\n      throw err\n    }\n\n    if (timedOut) {\n      throw traceStreamPreflightError(\n        `Trace stream pre-flight timed out after ${timeoutMs}ms before receiving a snapshot.`\n      )\n    }\n\n    throw traceStreamPreflightError(\n      `Trace stream pre-flight failed: ${err instanceof Error ? err.message : String(err)}`\n    )\n  } finally {\n    clearTimeout(timer)\n    reader?.cancel().catch(() => {})\n  }\n}\n\nfunction parseSSEEventName(block: string): string | null {\n  for (const line of block.split('\\n')) {\n    if (line.startsWith('event:')) {\n      return line.slice('event:'.length).trim()\n    }\n  }\n  return null\n}\n\nfunction traceStreamPreflightError(message: string): EvalRunnerError {\n  return new EvalRunnerError({\n    code: 'SSE_CONNECT_FAILED',\n    message,\n    expected: true,\n    suggestion: 'Make sure the dev server is running (`adk dev`) and trace streaming is available.',\n  })\n}\n\n/**\n * Verify the bot has at least one Conversation bound to `chat.channel`.\n *\n * The `chat` integration can be installed without any handler listening on its\n * channel \u2014 in that case messages sent via @botpress/chat produce no response\n * and the caller times out silently. Fail fast with actionable guidance instead.\n *\n * Uses the dev server's `/api/agent` endpoint, so this is a no-op against\n * production bots (no devServerUrl).\n */\nexport async function assertChatChannelBound(\n  devServerUrl: string,\n  headers: Record<string, string> = {},\n  logger: EvalLogger = defaultLogger\n): Promise<void> {\n  type ConversationSummary = { channel?: string | string[] }\n  let conversations: ConversationSummary[] = []\n  try {\n    const res = await fetch(`${devServerUrl}/api/agent`, { headers })\n    if (!res.ok) return // Dev server unreachable \u2014 skip, let downstream errors surface\n    const agent = (await res.json()) as { conversations?: ConversationSummary[] }\n    conversations = agent.conversations ?? []\n  } catch (err) {\n    // Soft pre-flight only \u2014 a missing dev server is normal (e.g. prod bots),\n    // but log the skip so a *broken* dev server (malformed /api/agent JSON)\n    // isn't invisible while every eval afterwards times out mysteriously.\n    logger?.warn(\n      `Skipping chat.channel binding pre-flight \u2014 could not query ${devServerUrl}/api/agent: ${err instanceof Error ? err.message : String(err)}`\n    )\n    return\n  }\n\n  if (conversations.length === 0) return // Bot has no conversations at all \u2014 different problem\n\n  // Mirrors the runtime's matchesChannel semantics: a handler matches `chat.channel`\n  // if it is the wildcard '*', the literal string 'chat.channel', or an array\n  // containing either.\n  const matchesChatChannel = (ch: string | string[] | undefined): boolean => {\n    if (!ch) return false\n    if (ch === '*' || ch === 'chat.channel') return true\n    return Array.isArray(ch) && (ch.includes('chat.channel') || ch.includes('*'))\n  }\n\n  const bound = conversations.some((c) => matchesChatChannel(c.channel))\n  if (bound) return\n\n  throw new EvalRunnerError({\n    code: 'CHAT_CHANNEL_UNBOUND',\n    message: [\n      'No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.',\n      '',\n      'To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):',\n      \"  channel: 'chat.channel',           // explicit\",\n      \"  channel: '*',                      // wildcard \u2014 matches every channel\",\n      \"  channel: ['chat.channel', ...],    // array form\",\n      '',\n      'The `chat` integration is how evals and `adk chat --single` send messages programmatically.',\n    ].join('\\n'),\n    expected: true,\n  })\n}\n"],
+  "mappings": ";AAiKO,IAAM,gBAA4B;;;ACzIlC,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;AC5CO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;ACd9D,IAAM,oCAAoC;AAMnC,IAAM,cAAN,MAAkB;AAAA,EAIvB,YACU,UACA,OACA,gBACA,cACA,aACR;AALQ;AACA;AACA;AACA;AACA;AAAA,EACP;AAAA,EATK,SAAkC;AAAA,EAClC,iBAAgC;AAAA,EAUxC,MAAM,UAAU;AACd,UAAM,YAAY,KAAK,kBAAmB,MAAM,kBAAkB,KAAK,UAAU,KAAK,KAAK;AAS3F,UAAM,iBACJ,KAAK,gBAAgB,MAAM;AAAA;AAAA,MAAiC;AAAA,IAA0B,GAAG;AAE3F,SAAK,SAAS,MAAM,eAAe,QAAQ;AAAA,MACzC;AAAA,MACA,GAAI,KAAK,eAAe,EAAE,YAAY,KAAK,aAAa,IAAI,CAAC;AAAA,IAC/D,CAAC;AAAA,EACH;AAAA;AAAA,EAGQ,kBAAoC;AAC1C,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AAAA,IACH;AACA,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,gBAAgB,EAAE,KAAK;AAAA,EACrC;AAAA,EAEA,MAAM,qBAAsC;AAC1C,UAAM,SAAS,KAAK,gBAAgB;AAEpC,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,OAAO,MAAM,OAAO,mBAAmB,CAAC,CAAC;AAC/C,WAAK,iBAAiB,KAAK,aAAa;AACxC,aAAO,KAAK,aAAa;AAAA,IAC3B;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,kBAAmC;AACvC,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,OAAO,MAAM,OAAO,mBAAmB,CAAC,CAAC;AAC/C,SAAK,iBAAiB,KAAK,aAAa;AACxC,WAAO,KAAK,aAAa;AAAA,EAC3B;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,YAAY,SAAgC;AAChD,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,OAAO,cAAc,EAAE,gBAAgB,SAAS,EAAE,MAAM,QAAQ,MAAM,QAAQ,EAAE,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,UAAU,SAAiD;AAC/D,UAAM,SAAS,KAAK,gBAAgB;AACpC,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,OAAO,YAAY,EAAE,SAAS,eAAe,CAAC;AAAA,EACtD;AACF;AAQA,eAAsB,kBAAkB,QAAkB,OAAgC;AACxF,QAAM,EAAE,IAAI,IAAI,MAAM,OAAO,OAAO,EAAE,IAAI,MAAM,CAAC;AACjD,QAAM,eAAe,IAAI,gBAAgB,CAAC;AAE1C,QAAM,OAAO,OAAO,OAAO,YAAY,EAAE,KAAK,CAAC,QAAS,IAAgC,SAAS,MAAM;AACvG,QAAM,YAAa,MAA8C;AAEjE,MAAI,CAAC,aAAa,OAAO,cAAc,UAAU;AAC/C,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS;AAAA,QACP;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,MACX,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAQA,eAAsB,0BACpB,cACA,UAAkC,CAAC,GACnC,YAAY,mCACG;AACf,QAAM,MAAM,GAAG,aAAa,QAAQ,OAAO,EAAE,CAAC;AAC9C,QAAM,kBAAkB,IAAI,gBAAgB;AAC5C,MAAI,WAAW;AACf,MAAI,SAAyD;AAE7D,QAAM,QAAQ,WAAW,MAAM;AAC7B,eAAW;AACX,oBAAgB,MAAM;AAAA,EACxB,GAAG,SAAS;AAEZ,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,SAAS,EAAE,GAAG,SAAS,QAAQ,oBAAoB;AAAA,MACnD,QAAQ,gBAAgB;AAAA,IAC1B,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,0BAA0B,mCAAmC,IAAI,MAAM,IAAI,IAAI,UAAU,EAAE;AAAA,IACnG;AAEA,QAAI,CAAC,IAAI,MAAM;AACb,YAAM,0BAA0B,uDAAuD;AAAA,IACzF;AAEA,aAAS,IAAI,KAAK,UAAU;AAC5B,UAAM,UAAU,IAAI,YAAY;AAChC,QAAI,SAAS;AAEb,WAAO,MAAM;AACX,YAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,UAAI,MAAM;AACR,cAAM,0BAA0B,gEAAgE;AAAA,MAClG;AAEA,gBAAU,QAAQ,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;AAChD,YAAM,QAAQ,OAAO,MAAM,MAAM;AACjC,eAAS,MAAM,IAAI,KAAK;AAExB,iBAAW,QAAQ,OAAO;AACxB,YAAI,kBAAkB,IAAI,MAAM,YAAY;AAC1C;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,eAAe,iBAAiB;AAClC,YAAM;AAAA,IACR;AAEA,QAAI,UAAU;AACZ,YAAM;AAAA,QACJ,2CAA2C,SAAS;AAAA,MACtD;AAAA,IACF;AAEA,UAAM;AAAA,MACJ,mCAAmC,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IACrF;AAAA,EACF,UAAE;AACA,iBAAa,KAAK;AAClB,YAAQ,OAAO,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACjC;AACF;AAEA,SAAS,kBAAkB,OAA8B;AACvD,aAAW,QAAQ,MAAM,MAAM,IAAI,GAAG;AACpC,QAAI,KAAK,WAAW,QAAQ,GAAG;AAC7B,aAAO,KAAK,MAAM,SAAS,MAAM,EAAE,KAAK;AAAA,IAC1C;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,0BAA0B,SAAkC;AACnE,SAAO,IAAI,gBAAgB;AAAA,IACzB,MAAM;AAAA,IACN;AAAA,IACA,UAAU;AAAA,IACV,YAAY;AAAA,EACd,CAAC;AACH;AAYA,eAAsB,uBACpB,cACA,UAAkC,CAAC,GACnC,SAAqB,eACN;AAEf,MAAI,gBAAuC,CAAC;AAC5C,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,YAAY,cAAc,EAAE,QAAQ,CAAC;AAChE,QAAI,CAAC,IAAI,GAAI;AACb,UAAM,QAAS,MAAM,IAAI,KAAK;AAC9B,oBAAgB,MAAM,iBAAiB,CAAC;AAAA,EAC1C,SAAS,KAAK;AAIZ,YAAQ;AAAA,MACN,mEAA8D,YAAY,eAAe,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,IAC3I;AACA;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,EAAG;AAKhC,QAAM,qBAAqB,CAAC,OAA+C;AACzE,QAAI,CAAC,GAAI,QAAO;AAChB,QAAI,OAAO,OAAO,OAAO,eAAgB,QAAO;AAChD,WAAO,MAAM,QAAQ,EAAE,MAAM,GAAG,SAAS,cAAc,KAAK,GAAG,SAAS,GAAG;AAAA,EAC7E;AAEA,QAAM,QAAQ,cAAc,KAAK,CAAC,MAAM,mBAAmB,EAAE,OAAO,CAAC;AACrE,MAAI,MAAO;AAEX,QAAM,IAAI,gBAAgB;AAAA,IACxB,MAAM;AAAA,IACN,SAAS;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,IACX,UAAU;AAAA,EACZ,CAAC;AACH;",
   "names": []
 }

package/dist/loader.js

@@ -41,8 +41,9 @@ var AdkError = class extends Error {
   code;
   /**
    * True for user/environment conditions the developer can fix themselves
-   * (validation, auth, connectivity). False for internal bugs. Defaults to
-   * false so unclassified errors surface as bugs until someone triages them.
+   * (validation, auth, connectivity) — reported as `error.expected` events.
+   * False for internal bugs — reported as PostHog `$exception`s. Defaults to
+   * **true**; set `expected: false` explicitly to flag a genuine bug.
    */
   expected;
   /** Structured context. Sanitized before any telemetry leaves the machine. */
@@ -53,7 +54,7 @@ var AdkError = class extends Error {
     super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
     this.name = this.constructor.name;
     this.code = opts.code;
-    this.expected = opts.expected ?? false;
+    this.expected = opts.expected ?? true;
     if (opts.details !== void 0) {
       this.details = opts.details;
     }

package/dist/loader.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/loader.ts", "../src/definition.ts", "../../analytics/src/errors.ts", "../src/errors.ts"],
-  "sourcesContent": ["/**\n * Eval file loader.\n * Loads *.eval.ts files from a directory using dynamic imports.\n * Supports both default and named exports, and multiple evals per file.\n */\n\nimport { readdirSync, existsSync } from 'fs'\nimport { resolve } from 'path'\nimport type { EvalDefinition, EvalFilter } from './types'\nimport { Eval } from './types'\nimport { EvalRunnerError } from './errors'\n\nfunction isEvalDefinition(value: unknown): value is EvalDefinition {\n  return (\n    value !== null &&\n    typeof value === 'object' &&\n    typeof (value as Record<string, unknown>).name === 'string' &&\n    (value as Record<string, unknown>).name !== '' &&\n    Array.isArray((value as Record<string, unknown>).conversation)\n  )\n}\n\nexport async function loadEvalFile(filePath: string): Promise<EvalDefinition[]> {\n  const absPath = resolve(filePath)\n\n  let mod: Record<string, unknown>\n  try {\n    mod = await import(absPath)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    // Guidance stays in the message until the CLI renders `suggestion`\n    // (cli-package remediation) \u2014 report.error/logger.fatal only show message.\n    throw new EvalRunnerError({\n      code: 'EVAL_LOAD_FAILED',\n      message:\n        `Failed to load eval file ${filePath}: ${msg}\\n\\n` +\n        'Make sure your eval file:\\n' +\n        '  - Has no syntax or type errors\\n' +\n        '  - Exports one or more `new Eval({...})` instances\\n' +\n        '  - Has all dependencies installed (`bun install`)',\n      expected: true,\n      cause: err,\n    })\n  }\n\n  const results: EvalDefinition[] = []\n\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === '__esModule') continue\n    if (value instanceof Eval || isEvalDefinition(value)) {\n      results.push(value)\n    }\n  }\n\n  if (results.length === 0) {\n    throw new EvalRunnerError({\n      code: 'EVAL_FILE_EMPTY',\n      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \\`new Eval({...})\\` instances (as default or named exports).`,\n      expected: true,\n    })\n  }\n\n  return results\n}\n\nexport async function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) {\n    return []\n  }\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n  const evals: EvalDefinition[] = []\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    evals.push(...defs)\n  }\n\n  const seen = new Set<string>()\n  for (const e of evals) {\n    if (seen.has(e.name)) {\n      throw new EvalRunnerError({\n        code: 'EVAL_DUPLICATE_NAME',\n        message: `Duplicate eval name \"${e.name}\" found in ${dirPath} \u2014 names must be unique across the evals directory.`,\n        expected: true,\n        details: { name: e.name },\n      })\n    }\n    seen.add(e.name)\n  }\n\n  return evals\n}\n\nexport async function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) return null\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    const found = defs.find((d) => d.name === name)\n    if (found) return found\n  }\n\n  return null\n}\n\nexport function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[] {\n  if (!filter) return evals\n\n  return evals.filter((e) => {\n    if (filter.names && filter.names.length > 0) {\n      if (!filter.names.includes(e.name)) return false\n    }\n    if (filter.tags && filter.tags.length > 0) {\n      if (!e.tags || !filter.tags.some((t) => e.tags!.includes(t))) return false\n    }\n    if (filter.type) {\n      if (e.type !== filter.type) return false\n    }\n    return true\n  })\n}\n", "export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { payload: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n  /**\n   * Run this turn in a NEW conversation under the same user. The prior\n   * transcript is absent, so recall here can only come from user-scoped\n   * persistence, not context. Ignored on the first turn.\n   */\n  newConversation?: boolean\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n", "/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity). False for internal bugs. Defaults to\n   * false so unclassified errors surface as bugs until someone triages them.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? false\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n"],
-  "mappings": ";AAMA,SAAS,aAAa,kBAAkB;AACxC,SAAS,eAAe;;;ACuFjB,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAMT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;ACrGO,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;ACpCO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;AHhB9D,SAAS,iBAAiB,OAAyC;AACjE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAQ,MAAkC,SAAS,YAClD,MAAkC,SAAS,MAC5C,MAAM,QAAS,MAAkC,YAAY;AAEjE;AAEA,eAAsB,aAAa,UAA6C;AAC9E,QAAM,UAAU,QAAQ,QAAQ;AAEhC,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,OAAO;AAAA,EACrB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAG3D,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SACE,4BAA4B,QAAQ,KAAK,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAK9C,UAAU;AAAA,MACV,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,QAAM,UAA4B,CAAC;AAEnC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,QAAQ,aAAc;AAC1B,QAAI,iBAAiB,QAAQ,iBAAiB,KAAK,GAAG;AACpD,cAAQ,KAAK,KAAK;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS,qBAAqB,QAAQ;AAAA,MACtC,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAiB,SAA4C;AACjF,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AACtE,QAAM,QAA0B,CAAC;AAEjC,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,KAAK,GAAG,IAAI;AAAA,EACpB;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,OAAO;AACrB,QAAI,KAAK,IAAI,EAAE,IAAI,GAAG;AACpB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS,wBAAwB,EAAE,IAAI,cAAc,OAAO;AAAA,QAC5D,UAAU;AAAA,QACV,SAAS,EAAE,MAAM,EAAE,KAAK;AAAA,MAC1B,CAAC;AAAA,IACH;AACA,SAAK,IAAI,EAAE,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;AAEA,eAAsB,eAAe,SAAiB,MAA8C;AAClG,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,EAAG,QAAO;AAEhC,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AAEtE,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI;AAC9C,QAAI,MAAO,QAAO;AAAA,EACpB;AAEA,SAAO;AACT;AAEO,SAAS,YAAY,OAAyB,QAAuC;AAC1F,MAAI,CAAC,OAAQ,QAAO;AAEpB,SAAO,MAAM,OAAO,CAAC,MAAM;AACzB,QAAI,OAAO,SAAS,OAAO,MAAM,SAAS,GAAG;AAC3C,UAAI,CAAC,OAAO,MAAM,SAAS,EAAE,IAAI,EAAG,QAAO;AAAA,IAC7C;AACA,QAAI,OAAO,QAAQ,OAAO,KAAK,SAAS,GAAG;AACzC,UAAI,CAAC,EAAE,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,KAAM,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACvE;AACA,QAAI,OAAO,MAAM;AACf,UAAI,EAAE,SAAS,OAAO,KAAM,QAAO;AAAA,IACrC;AACA,WAAO;AAAA,EACT,CAAC;AACH;",
+  "sourcesContent": ["/**\n * Eval file loader.\n * Loads *.eval.ts files from a directory using dynamic imports.\n * Supports both default and named exports, and multiple evals per file.\n */\n\nimport { readdirSync, existsSync } from 'fs'\nimport { resolve } from 'path'\nimport type { EvalDefinition, EvalFilter } from './types'\nimport { Eval } from './types'\nimport { EvalRunnerError } from './errors'\n\nfunction isEvalDefinition(value: unknown): value is EvalDefinition {\n  return (\n    value !== null &&\n    typeof value === 'object' &&\n    typeof (value as Record<string, unknown>).name === 'string' &&\n    (value as Record<string, unknown>).name !== '' &&\n    Array.isArray((value as Record<string, unknown>).conversation)\n  )\n}\n\nexport async function loadEvalFile(filePath: string): Promise<EvalDefinition[]> {\n  const absPath = resolve(filePath)\n\n  let mod: Record<string, unknown>\n  try {\n    mod = await import(absPath)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    // Guidance stays in the message until the CLI renders `suggestion`\n    // (cli-package remediation) \u2014 report.error/logger.fatal only show message.\n    throw new EvalRunnerError({\n      code: 'EVAL_LOAD_FAILED',\n      message:\n        `Failed to load eval file ${filePath}: ${msg}\\n\\n` +\n        'Make sure your eval file:\\n' +\n        '  - Has no syntax or type errors\\n' +\n        '  - Exports one or more `new Eval({...})` instances\\n' +\n        '  - Has all dependencies installed (`bun install`)',\n      expected: true,\n      cause: err,\n    })\n  }\n\n  const results: EvalDefinition[] = []\n\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === '__esModule') continue\n    if (value instanceof Eval || isEvalDefinition(value)) {\n      results.push(value)\n    }\n  }\n\n  if (results.length === 0) {\n    throw new EvalRunnerError({\n      code: 'EVAL_FILE_EMPTY',\n      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \\`new Eval({...})\\` instances (as default or named exports).`,\n      expected: true,\n    })\n  }\n\n  return results\n}\n\nexport async function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) {\n    return []\n  }\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n  const evals: EvalDefinition[] = []\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    evals.push(...defs)\n  }\n\n  const seen = new Set<string>()\n  for (const e of evals) {\n    if (seen.has(e.name)) {\n      throw new EvalRunnerError({\n        code: 'EVAL_DUPLICATE_NAME',\n        message: `Duplicate eval name \"${e.name}\" found in ${dirPath} \u2014 names must be unique across the evals directory.`,\n        expected: true,\n        details: { name: e.name },\n      })\n    }\n    seen.add(e.name)\n  }\n\n  return evals\n}\n\nexport async function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) return null\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    const found = defs.find((d) => d.name === name)\n    if (found) return found\n  }\n\n  return null\n}\n\nexport function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[] {\n  if (!filter) return evals\n\n  return evals.filter((e) => {\n    if (filter.names && filter.names.length > 0) {\n      if (!filter.names.includes(e.name)) return false\n    }\n    if (filter.tags && filter.tags.length > 0) {\n      if (!e.tags || !filter.tags.some((t) => e.tags!.includes(t))) return false\n    }\n    if (filter.type) {\n      if (e.type !== filter.type) return false\n    }\n    return true\n  })\n}\n", "export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { payload: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n  /**\n   * Run this turn in a NEW conversation under the same user. The prior\n   * transcript is absent, so recall here can only come from user-scoped\n   * persistence, not context. Ignored on the first turn.\n   */\n  newConversation?: boolean\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n", "/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * `expected` defaults to **true**: the overwhelming majority of thrown errors\n * are user/environment conditions, and expected errors are reported as plain\n * `error.expected` analytics events rather than PostHog `$exception`s. Genuine\n * bugs must opt out explicitly with `expected: false` so they reach Error\n * Tracking. There is a single base class \u2014 everything thrown is an `AdkError`\n * or a subclass.\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity) \u2014 reported as `error.expected` events.\n   * False for internal bugs \u2014 reported as PostHog `$exception`s. Defaults to\n   * **true**; set `expected: false` explicitly to flag a genuine bug.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? true\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n\n/**\n * True for an `AdkError` (or subclass) flagged `expected` \u2014 a user/environment\n * condition that should be reported as an `error.expected` analytics event\n * rather than a PostHog `$exception`. Used by the telemetry capture funnel to\n * route errors. Non-`AdkError` throws (and `expected: false` bugs) are not\n * expected and keep flowing to Error Tracking.\n */\nexport function isExpectedError(error: unknown): error is AdkError {\n  return isAdkError(error) && error.expected\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n"],
+  "mappings": ";AAMA,SAAS,aAAa,kBAAkB;AACxC,SAAS,eAAe;;;ACuFjB,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAMT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;AC9FO,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;AC5CO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;AHhB9D,SAAS,iBAAiB,OAAyC;AACjE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAQ,MAAkC,SAAS,YAClD,MAAkC,SAAS,MAC5C,MAAM,QAAS,MAAkC,YAAY;AAEjE;AAEA,eAAsB,aAAa,UAA6C;AAC9E,QAAM,UAAU,QAAQ,QAAQ;AAEhC,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,OAAO;AAAA,EACrB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAG3D,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SACE,4BAA4B,QAAQ,KAAK,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAK9C,UAAU;AAAA,MACV,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,QAAM,UAA4B,CAAC;AAEnC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,QAAQ,aAAc;AAC1B,QAAI,iBAAiB,QAAQ,iBAAiB,KAAK,GAAG;AACpD,cAAQ,KAAK,KAAK;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS,qBAAqB,QAAQ;AAAA,MACtC,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAiB,SAA4C;AACjF,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AACtE,QAAM,QAA0B,CAAC;AAEjC,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,KAAK,GAAG,IAAI;AAAA,EACpB;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,OAAO;AACrB,QAAI,KAAK,IAAI,EAAE,IAAI,GAAG;AACpB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS,wBAAwB,EAAE,IAAI,cAAc,OAAO;AAAA,QAC5D,UAAU;AAAA,QACV,SAAS,EAAE,MAAM,EAAE,KAAK;AAAA,MAC1B,CAAC;AAAA,IACH;AACA,SAAK,IAAI,EAAE,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;AAEA,eAAsB,eAAe,SAAiB,MAA8C;AAClG,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,EAAG,QAAO;AAEhC,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AAEtE,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,QAAQ,KAAK,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI;AAC9C,QAAI,MAAO,QAAO;AAAA,EACpB;AAEA,SAAO;AACT;AAEO,SAAS,YAAY,OAAyB,QAAuC;AAC1F,MAAI,CAAC,OAAQ,QAAO;AAEpB,SAAO,MAAM,OAAO,CAAC,MAAM;AACzB,QAAI,OAAO,SAAS,OAAO,MAAM,SAAS,GAAG;AAC3C,UAAI,CAAC,OAAO,MAAM,SAAS,EAAE,IAAI,EAAG,QAAO;AAAA,IAC7C;AACA,QAAI,OAAO,QAAQ,OAAO,KAAK,SAAS,GAAG;AACzC,UAAI,CAAC,EAAE,QAAQ,CAAC,OAAO,KAAK,KAAK,CAAC,MAAM,EAAE,KAAM,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACvE;AACA,QAAI,OAAO,MAAM;AACf,UAAI,EAAE,SAAS,OAAO,KAAM,QAAO;AAAA,IACrC;AACA,WAAO;AAAA,EACT,CAAC;AACH;",
   "names": []
 }

package/dist/runner.js

@@ -43,8 +43,9 @@ var AdkError = class extends Error {
   code;
   /**
    * True for user/environment conditions the developer can fix themselves
-   * (validation, auth, connectivity). False for internal bugs. Defaults to
-   * false so unclassified errors surface as bugs until someone triages them.
+   * (validation, auth, connectivity) — reported as `error.expected` events.
+   * False for internal bugs — reported as PostHog `$exception`s. Defaults to
+   * **true**; set `expected: false` explicitly to flag a genuine bug.
    */
   expected;
   /** Structured context. Sanitized before any telemetry leaves the machine. */
@@ -55,7 +56,7 @@ var AdkError = class extends Error {
     super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
     this.name = this.constructor.name;
     this.code = opts.code;
-    this.expected = opts.expected ?? false;
+    this.expected = opts.expected ?? true;
     if (opts.details !== void 0) {
       this.details = opts.details;
     }