@botpress/evals 1.18.3

package/dist/client.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Chat client for eval conversations.
+ * Drives conversations against a running ADK bot via @botpress/chat.
+ *
+ * This is a send-only client. Observation (bot responses, tool calls, state)
+ * is handled by the SSECollector via trace spans — not via WebSocket listeners.
+ */
+import type { ChatClient } from './types';
+export declare function setDefaultChatClient(chatClient: ChatClient | null): void;
+/**
+ * A send-only chat session that maintains a single client connection across turns.
+ * Messages and events are fire-and-forget; the SSECollector observes results via traces.
+ */
+export declare class ChatSession {
+    private webhookId;
+    private chatClient?;
+    private client;
+    private conversationId;
+    constructor(webhookId: string, chatClient?: ChatClient);
+    connect(): Promise<void>;
+    get userId(): string;
+    ensureConversation(): Promise<string>;
+    /**
+     * Send a user message. Fire-and-forget — observation is via SSECollector.
+     */
+    sendMessage(message: string): Promise<void>;
+    /**
+     * Send an event. Fire-and-forget — observation is via SSECollector.
+     */
+    sendEvent(type: string, payload: Record<string, unknown>): Promise<void>;
+}
+/**
+ * Discover the chat integration's webhookId from a bot.
+ *
+ * Evals talk to the bot through the `chat` integration (via @botpress/chat).
+ * If the bot doesn't have that integration installed, evals can't run.
+ */
+export declare function discoverWebhookId(botId: string, token: string, apiUrl: string): Promise<string>;
+/**
+ * Verify the bot has at least one Conversation bound to `chat.channel`.
+ *
+ * The `chat` integration can be installed without any handler listening on its
+ * channel — in that case messages sent via @botpress/chat produce no response
+ * and the caller times out silently. Fail fast with actionable guidance instead.
+ *
+ * Uses the dev server's `/api/agent` endpoint, so this is a no-op against
+ * production bots (no devServerUrl).
+ */
+export declare function assertChatChannelBound(devServerUrl: string, headers?: Record<string, string>): Promise<void>;
+//# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAwB,MAAM,SAAS,CAAA;AAI/D,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,UAAU,GAAG,IAAI,GAAG,IAAI,CAExE;AAcD;;;GAGG;AACH,qBAAa,WAAW;IAKpB,OAAO,CAAC,SAAS;IACjB,OAAO,CAAC,UAAU,CAAC;IALrB,OAAO,CAAC,MAAM,CAAoC;IAClD,OAAO,CAAC,cAAc,CAAsB;gBAGlC,SAAS,EAAE,MAAM,EACjB,UAAU,CAAC,EAAE,UAAU;IAG3B,OAAO;IAKb,IAAI,MAAM,IAAI,MAAM,CAKnB;IAEK,kBAAkB,IAAI,OAAO,CAAC,MAAM,CAAC;IAc3C;;OAEG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IASjD;;OAEG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAQ/E;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAqBrG;AAED;;;;;;;;;GASG;AACH,wBAAsB,sBAAsB,CAC1C,YAAY,EAAE,MAAM,EACpB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM,GACnC,OAAO,CAAC,IAAI,CAAC,CAsCf"}

package/dist/client.js

@@ -0,0 +1,123 @@
+// src/client.ts
+import { Client as BpClient } from "@botpress/client";
+import { createRequire } from "module";
+var defaultChatClient = null;
+function setDefaultChatClient(chatClient) {
+  defaultChatClient = chatClient;
+}
+function getDefaultChatClient() {
+  if (defaultChatClient) return defaultChatClient;
+  const require2 = createRequire(import.meta.url);
+  const mod = require2("@botpress/chat");
+  const ChatClient = mod.default?.Client ?? mod.Client;
+  if (!ChatClient) {
+    throw new Error("@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.");
+  }
+  return ChatClient;
+}
+var ChatSession = class {
+  constructor(webhookId, chatClient) {
+    this.webhookId = webhookId;
+    this.chatClient = chatClient;
+  }
+  client = null;
+  conversationId = null;
+  async connect() {
+    const ChatClient = this.chatClient ?? getDefaultChatClient();
+    this.client = await ChatClient.connect({ webhookId: this.webhookId });
+  }
+  get userId() {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    return this.client.user.id;
+  }
+  async ensureConversation() {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    if (!this.conversationId) {
+      const conv = await this.client.createConversation({});
+      this.conversationId = conv.conversation.id;
+      return conv.conversation.id;
+    }
+    return this.conversationId;
+  }
+  /**
+   * Send a user message. Fire-and-forget — observation is via SSECollector.
+   */
+  async sendMessage(message) {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    const conversationId = await this.ensureConversation();
+    await this.client.createMessage({ conversationId, payload: { type: "text", text: message } });
+  }
+  /**
+   * Send an event. Fire-and-forget — observation is via SSECollector.
+   */
+  async sendEvent(type, payload) {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    const conversationId = await this.ensureConversation();
+    await this.client.createEvent({ type, payload, conversationId });
+  }
+};
+async function discoverWebhookId(botId, token, apiUrl) {
+  const client = new BpClient({ token, botId, apiUrl });
+  const { bot } = await client.getBot({ id: botId });
+  const integrations = bot.integrations || {};
+  const chat = Object.values(integrations).find((int) => int.name === "chat");
+  const webhookId = chat?.webhookId;
+  if (!webhookId || typeof webhookId !== "string") {
+    throw new Error(
+      [
+        "The `chat` integration is not installed on this bot \u2014 evals require it.",
+        "",
+        "To fix:",
+        "  1. Install it:    adk add chat",
+        "  2. Redeploy:      adk dev   (or: adk deploy)"
+      ].join("\n")
+    );
+  }
+  return webhookId;
+}
+async function assertChatChannelBound(devServerUrl, headers = {}) {
+  let conversations = [];
+  try {
+    const res = await fetch(`${devServerUrl}/api/agent`, { headers });
+    if (!res.ok) return;
+    const agent = await res.json();
+    conversations = agent.conversations ?? [];
+  } catch {
+    return;
+  }
+  if (conversations.length === 0) return;
+  const matchesChatChannel = (ch) => {
+    if (!ch) return false;
+    if (ch === "*" || ch === "chat.channel") return true;
+    return Array.isArray(ch) && (ch.includes("chat.channel") || ch.includes("*"));
+  };
+  const bound = conversations.some((c) => matchesChatChannel(c.channel));
+  if (bound) return;
+  throw new Error(
+    [
+      "No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.",
+      "",
+      "To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):",
+      "  channel: 'chat.channel',           // explicit",
+      "  channel: '*',                      // wildcard \u2014 matches every channel",
+      "  channel: ['chat.channel', ...],    // array form",
+      "",
+      "The `chat` integration is how evals and `adk chat --single` send messages programmatically."
+    ].join("\n")
+  );
+}
+export {
+  ChatSession,
+  assertChatChannelBound,
+  discoverWebhookId,
+  setDefaultChatClient
+};
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/client.ts"],
+  "sourcesContent": ["/**\n * Chat client for eval conversations.\n * Drives conversations against a running ADK bot via @botpress/chat.\n *\n * This is a send-only client. Observation (bot responses, tool calls, state)\n * is handled by the SSECollector via trace spans \u2014 not via WebSocket listeners.\n */\n\nimport { Client as BpClient } from '@botpress/client'\nimport { createRequire } from 'module'\nimport type { ChatClient, ChatClientConnection } from './types'\n\nlet defaultChatClient: ChatClient | null = null\n\nexport function setDefaultChatClient(chatClient: ChatClient | null): void {\n  defaultChatClient = chatClient\n}\n\nfunction getDefaultChatClient(): ChatClient {\n  if (defaultChatClient) return defaultChatClient\n\n  const require = createRequire(import.meta.url)\n  const mod = require('@botpress/chat') as { Client?: ChatClient; default?: { Client?: ChatClient } }\n  const ChatClient = mod.default?.Client ?? mod.Client\n  if (!ChatClient) {\n    throw new Error('@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.')\n  }\n  return ChatClient\n}\n\n/**\n * A send-only chat session that maintains a single client connection across turns.\n * Messages and events are fire-and-forget; the SSECollector observes results via traces.\n */\nexport class ChatSession {\n  private client: ChatClientConnection | null = null\n  private conversationId: string | null = null\n\n  constructor(\n    private webhookId: string,\n    private chatClient?: ChatClient\n  ) {}\n\n  async connect() {\n    const ChatClient = this.chatClient ?? getDefaultChatClient()\n    this.client = await ChatClient.connect({ webhookId: this.webhookId })\n  }\n\n  get userId(): string {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n    return this.client.user.id\n  }\n\n  async ensureConversation(): Promise<string> {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n\n    if (!this.conversationId) {\n      const conv = await this.client.createConversation({})\n      this.conversationId = conv.conversation.id\n      return conv.conversation.id\n    }\n\n    return this.conversationId\n  }\n\n  /**\n   * Send a user message. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendMessage(message: string): Promise<void> {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n\n    const conversationId = await this.ensureConversation()\n    await this.client.createMessage({ conversationId, payload: { type: 'text', text: message } })\n  }\n\n  /**\n   * Send an event. Fire-and-forget \u2014 observation is via SSECollector.\n   */\n  async sendEvent(type: string, payload: Record<string, unknown>): Promise<void> {\n    if (!this.client) {\n      throw new Error('ChatSession not connected. Call connect() first.')\n    }\n\n    const conversationId = await this.ensureConversation()\n    await this.client.createEvent({ type, payload, conversationId })\n  }\n}\n\n/**\n * Discover the chat integration's webhookId from a bot.\n *\n * Evals talk to the bot through the `chat` integration (via @botpress/chat).\n * If the bot doesn't have that integration installed, evals can't run.\n */\nexport async function discoverWebhookId(botId: string, token: string, apiUrl: string): Promise<string> {\n  const client = new BpClient({ token, botId, apiUrl })\n  const { bot } = await client.getBot({ id: botId })\n  const integrations = bot.integrations || {}\n\n  const chat = Object.values(integrations).find((int) => (int as Record<string, unknown>).name === 'chat')\n  const webhookId = (chat as Record<string, unknown> | undefined)?.webhookId\n\n  if (!webhookId || typeof webhookId !== 'string') {\n    throw new Error(\n      [\n        'The `chat` integration is not installed on this bot \u2014 evals require it.',\n        '',\n        'To fix:',\n        '  1. Install it:    adk add chat',\n        '  2. Redeploy:      adk dev   (or: adk deploy)',\n      ].join('\\n')\n    )\n  }\n\n  return webhookId\n}\n\n/**\n * Verify the bot has at least one Conversation bound to `chat.channel`.\n *\n * The `chat` integration can be installed without any handler listening on its\n * channel \u2014 in that case messages sent via @botpress/chat produce no response\n * and the caller times out silently. Fail fast with actionable guidance instead.\n *\n * Uses the dev server's `/api/agent` endpoint, so this is a no-op against\n * production bots (no devServerUrl).\n */\nexport async function assertChatChannelBound(\n  devServerUrl: string,\n  headers: Record<string, string> = {}\n): Promise<void> {\n  type ConversationSummary = { channel?: string | string[] }\n  let conversations: ConversationSummary[] = []\n  try {\n    const res = await fetch(`${devServerUrl}/api/agent`, { headers })\n    if (!res.ok) return // Dev server unreachable \u2014 skip, let downstream errors surface\n    const agent = (await res.json()) as { conversations?: ConversationSummary[] }\n    conversations = agent.conversations ?? []\n  } catch {\n    return // Network error \u2014 skip check\n  }\n\n  if (conversations.length === 0) return // Bot has no conversations at all \u2014 different problem\n\n  // Mirrors the runtime's matchesChannel semantics: a handler matches `chat.channel`\n  // if it is the wildcard '*', the literal string 'chat.channel', or an array\n  // containing either.\n  const matchesChatChannel = (ch: string | string[] | undefined): boolean => {\n    if (!ch) return false\n    if (ch === '*' || ch === 'chat.channel') return true\n    return Array.isArray(ch) && (ch.includes('chat.channel') || ch.includes('*'))\n  }\n\n  const bound = conversations.some((c) => matchesChatChannel(c.channel))\n  if (bound) return\n\n  throw new Error(\n    [\n      'No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.',\n      '',\n      'To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):',\n      \"  channel: 'chat.channel',           // explicit\",\n      \"  channel: '*',                      // wildcard \u2014 matches every channel\",\n      \"  channel: ['chat.channel', ...],    // array form\",\n      '',\n      'The `chat` integration is how evals and `adk chat --single` send messages programmatically.',\n    ].join('\\n')\n  )\n}\n"],
+  "mappings": ";AAQA,SAAS,UAAU,gBAAgB;AACnC,SAAS,qBAAqB;AAG9B,IAAI,oBAAuC;AAEpC,SAAS,qBAAqB,YAAqC;AACxE,sBAAoB;AACtB;AAEA,SAAS,uBAAmC;AAC1C,MAAI,kBAAmB,QAAO;AAE9B,QAAMA,WAAU,cAAc,YAAY,GAAG;AAC7C,QAAM,MAAMA,SAAQ,gBAAgB;AACpC,QAAM,aAAa,IAAI,SAAS,UAAU,IAAI;AAC9C,MAAI,CAAC,YAAY;AACf,UAAM,IAAI,MAAM,6FAA6F;AAAA,EAC/G;AACA,SAAO;AACT;AAMO,IAAM,cAAN,MAAkB;AAAA,EAIvB,YACU,WACA,YACR;AAFQ;AACA;AAAA,EACP;AAAA,EANK,SAAsC;AAAA,EACtC,iBAAgC;AAAA,EAOxC,MAAM,UAAU;AACd,UAAM,aAAa,KAAK,cAAc,qBAAqB;AAC3D,SAAK,SAAS,MAAM,WAAW,QAAQ,EAAE,WAAW,KAAK,UAAU,CAAC;AAAA,EACtE;AAAA,EAEA,IAAI,SAAiB;AACnB,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AACA,WAAO,KAAK,OAAO,KAAK;AAAA,EAC1B;AAAA,EAEA,MAAM,qBAAsC;AAC1C,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AAEA,QAAI,CAAC,KAAK,gBAAgB;AACxB,YAAM,OAAO,MAAM,KAAK,OAAO,mBAAmB,CAAC,CAAC;AACpD,WAAK,iBAAiB,KAAK,aAAa;AACxC,aAAO,KAAK,aAAa;AAAA,IAC3B;AAEA,WAAO,KAAK;AAAA,EACd;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,YAAY,SAAgC;AAChD,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AAEA,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,KAAK,OAAO,cAAc,EAAE,gBAAgB,SAAS,EAAE,MAAM,QAAQ,MAAM,QAAQ,EAAE,CAAC;AAAA,EAC9F;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,UAAU,MAAc,SAAiD;AAC7E,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,kDAAkD;AAAA,IACpE;AAEA,UAAM,iBAAiB,MAAM,KAAK,mBAAmB;AACrD,UAAM,KAAK,OAAO,YAAY,EAAE,MAAM,SAAS,eAAe,CAAC;AAAA,EACjE;AACF;AAQA,eAAsB,kBAAkB,OAAe,OAAe,QAAiC;AACrG,QAAM,SAAS,IAAI,SAAS,EAAE,OAAO,OAAO,OAAO,CAAC;AACpD,QAAM,EAAE,IAAI,IAAI,MAAM,OAAO,OAAO,EAAE,IAAI,MAAM,CAAC;AACjD,QAAM,eAAe,IAAI,gBAAgB,CAAC;AAE1C,QAAM,OAAO,OAAO,OAAO,YAAY,EAAE,KAAK,CAAC,QAAS,IAAgC,SAAS,MAAM;AACvG,QAAM,YAAa,MAA8C;AAEjE,MAAI,CAAC,aAAa,OAAO,cAAc,UAAU;AAC/C,UAAM,IAAI;AAAA,MACR;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAAA,IACb;AAAA,EACF;AAEA,SAAO;AACT;AAYA,eAAsB,uBACpB,cACA,UAAkC,CAAC,GACpB;AAEf,MAAI,gBAAuC,CAAC;AAC5C,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,GAAG,YAAY,cAAc,EAAE,QAAQ,CAAC;AAChE,QAAI,CAAC,IAAI,GAAI;AACb,UAAM,QAAS,MAAM,IAAI,KAAK;AAC9B,oBAAgB,MAAM,iBAAiB,CAAC;AAAA,EAC1C,QAAQ;AACN;AAAA,EACF;AAEA,MAAI,cAAc,WAAW,EAAG;AAKhC,QAAM,qBAAqB,CAAC,OAA+C;AACzE,QAAI,CAAC,GAAI,QAAO;AAChB,QAAI,OAAO,OAAO,OAAO,eAAgB,QAAO;AAChD,WAAO,MAAM,QAAQ,EAAE,MAAM,GAAG,SAAS,cAAc,KAAK,GAAG,SAAS,GAAG;AAAA,EAC7E;AAEA,QAAM,QAAQ,cAAc,KAAK,CAAC,MAAM,mBAAmB,EAAE,OAAO,CAAC;AACrE,MAAI,MAAO;AAEX,QAAM,IAAI;AAAA,IACR;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AAAA,EACb;AACF;",
+  "names": ["require"]
+}

package/dist/definition.d.ts

@@ -0,0 +1,113 @@
+export type MatchOperator = string | {
+    equals: unknown;
+} | {
+    contains: string;
+} | {
+    not_contains: string;
+} | {
+    matches: string;
+} | {
+    in: unknown[];
+} | {
+    exists: boolean;
+} | {
+    gte: number;
+} | {
+    lte: number;
+};
+export type ResponseAssertion = {
+    contains: string;
+} | {
+    not_contains: string;
+} | {
+    matches: string;
+} | {
+    llm_judge: string;
+};
+export type ToolAssertion = {
+    called: string;
+    params?: Record<string, MatchOperator>;
+} | {
+    not_called: string;
+} | {
+    call_order: string[];
+};
+export interface StateAssertion {
+    path: string;
+    equals?: unknown;
+    changed?: boolean;
+}
+export interface WorkflowAssertion {
+    name: string;
+    entered?: boolean;
+    completed?: boolean;
+}
+export type NumericOperator = {
+    equals: number;
+} | {
+    gte: number;
+} | {
+    lte: number;
+};
+export interface TimingAssertion {
+    response_time: NumericOperator;
+}
+export interface TurnAssertions {
+    response?: ResponseAssertion[];
+    tools?: ToolAssertion[];
+    state?: StateAssertion[];
+    workflow?: WorkflowAssertion[];
+    timing?: TimingAssertion[];
+}
+export interface ConversationTurn {
+    user?: string;
+    event?: {
+        type: string;
+        payload?: Record<string, unknown>;
+    };
+    expectSilence?: boolean;
+    assert?: TurnAssertions;
+}
+export interface OutcomeAssertions {
+    state?: StateAssertion[];
+    workflow?: WorkflowAssertion[];
+}
+export interface EvalSetup {
+    state?: {
+        bot?: Record<string, unknown>;
+        user?: Record<string, unknown>;
+        conversation?: Record<string, unknown>;
+    };
+    workflow?: {
+        trigger: string;
+        input?: Record<string, unknown>;
+    };
+}
+export interface EvalDefinition {
+    name: string;
+    description?: string;
+    tags?: string[];
+    type?: 'capability' | 'regression';
+    setup?: EvalSetup;
+    conversation: ConversationTurn[];
+    outcome?: OutcomeAssertions;
+    options?: {
+        idleTimeout?: number;
+        judgePassThreshold?: number;
+    };
+}
+export declare class Eval implements EvalDefinition {
+    readonly name: string;
+    readonly description?: string;
+    readonly tags?: string[];
+    readonly type?: 'capability' | 'regression';
+    readonly setup?: EvalSetup;
+    readonly conversation: ConversationTurn[];
+    readonly outcome?: OutcomeAssertions;
+    readonly options?: {
+        idleTimeout?: number;
+        judgePassThreshold?: number;
+    };
+    constructor(def: EvalDefinition);
+}
+//# sourceMappingURL=definition.d.ts.map

package/dist/definition.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"definition.d.ts","sourceRoot":"","sources":["../src/definition.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,aAAa,GACrB,MAAM,GACN;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GACnB;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GACpB;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,GACxB;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,EAAE,EAAE,OAAO,EAAE,CAAA;CAAE,GACjB;IAAE,MAAM,EAAE,OAAO,CAAA;CAAE,GACnB;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GACf;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,CAAA;AAEnB,MAAM,MAAM,iBAAiB,GACzB;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GACpB;IAAE,YAAY,EAAE,MAAM,CAAA;CAAE,GACxB;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,GACnB;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,CAAA;AAGzB,MAAM,MAAM,aAAa,GACrB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;CAAE,GAC1D;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,GACtB;IAAE,UAAU,EAAE,MAAM,EAAE,CAAA;CAAE,CAAA;AAE5B,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,CAAC,EAAE,OAAO,CAAA;IAChB,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,MAAM,eAAe,GAAG;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,CAAA;AAEpF,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,eAAe,CAAA;CAC/B;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC9B,KAAK,CAAC,EAAE,aAAa,EAAE,CAAA;IACvB,KAAK,CAAC,EAAE,cAAc,EAAE,CAAA;IACxB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC9B,MAAM,CAAC,EAAE,eAAe,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAA;IAC3D,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,MAAM,CAAC,EAAE,cAAc,CAAA;CACxB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAA;IACxB,QAAQ,CAAC,EAAE,iBAAiB,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,CAAC,EAAE;QACN,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC7B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAC9B,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KACvC,CAAA;IACD,QAAQ,CAAC,EAAE;QACT,OAAO,EAAE,MAAM,CAAA;QACf,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAChC,CAAA;CACF;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAClC,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB,YAAY,EAAE,gBAAgB,EAAE,CAAA;IAChC,OAAO,CAAC,EAAE,iBAAiB,CAAA;IAC3B,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;CACF;AAED,qBAAa,IAAK,YAAW,cAAc;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAC3C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,YAAY,EAAE,gBAAgB,EAAE,CAAA;IACzC,QAAQ,CAAC,OAAO,CAAC,EAAE,iBAAiB,CAAA;IACpC,QAAQ,CAAC,OAAO,CAAC,EAAE;QACjB,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;KAC5B,CAAA;gBAEW,GAAG,EAAE,cAAc;CAUhC"}

package/dist/definition.js

@@ -0,0 +1,25 @@
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+export {
+  Eval
+};
+//# sourceMappingURL=definition.js.map

package/dist/definition.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/definition.ts"],
+  "sourcesContent": ["export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n"],
+  "mappings": ";AAuFO,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;",
+  "names": []
+}

package/dist/graders/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Grader barrel exports.
+ */
+export { matchValue, operatorToString } from './match';
+export { gradeResponse } from './response';
+export { gradeLLMJudge, initLLMJudge } from './llm';
+export { gradeTools } from './tools';
+export { gradeState } from './state';
+export { gradeWorkflows } from './workflow';
+export { gradeTiming } from './timing';
+export { gradeOutcome } from './outcome';
+//# sourceMappingURL=index.d.ts.map

package/dist/graders/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/graders/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC1C,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,OAAO,CAAA;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA"}