@botpress/evals 1.18.3

package/dist/types.d.ts

@@ -0,0 +1,184 @@
+/**
+ * Eval type definitions.
+ *
+ * User-facing types live in @botpress/evals so eval authors can update eval
+ * tooling without tying the authoring API to the agent runtime package.
+ */
+export { Eval, type EvalDefinition, type EvalSetup, type ConversationTurn, type TurnAssertions, type OutcomeAssertions, type ResponseAssertion, type ToolAssertion, type StateAssertion, type WorkflowAssertion, type TimingAssertion, type MatchOperator, type NumericOperator, } from './definition';
+export type { Span, SpanStatus } from './trace';
+export interface ChatClientConnection {
+    user: {
+        id: string;
+    };
+    createConversation: (props: Record<string, unknown>) => Promise<{
+        conversation: {
+            id: string;
+        };
+    }>;
+    createMessage: (props: any) => Promise<unknown>;
+    createEvent: (props: any) => Promise<unknown>;
+}
+export interface ChatClient {
+    connect: (props: {
+        webhookId: string;
+    }) => Promise<ChatClientConnection>;
+}
+export interface GraderResult {
+    assertion: string;
+    pass: boolean;
+    expected: string;
+    actual: string;
+}
+export interface ToolCall {
+    name: string;
+    input: Record<string, unknown>;
+    output: string;
+    status: string;
+}
+export interface TurnReport {
+    turnNumber: number;
+    userMessage: string;
+    botResponse: string;
+    assertions: GraderResult[];
+    pass: boolean;
+    /** Time spent waiting for the bot to respond (ms) */
+    botDuration: number;
+    /** Time spent fetching traces + running graders (ms) */
+    evalDuration: number;
+}
+export interface EvalReport {
+    name: string;
+    description?: string;
+    type?: 'capability' | 'regression';
+    tags?: string[];
+    turns: TurnReport[];
+    outcomeAssertions: GraderResult[];
+    pass: boolean;
+    duration: number;
+    error?: string;
+}
+export type EvalProgressEvent = {
+    type: 'suite_start';
+    totalEvals: number;
+} | {
+    type: 'eval_start';
+    evalName: string;
+    index: number;
+    totalTurns: number;
+} | {
+    type: 'turn_start';
+    evalName: string;
+    evalIndex: number;
+    turnNumber: number;
+    totalTurns: number;
+    userMessage: string;
+} | {
+    type: 'turn_complete';
+    evalName: string;
+    evalIndex: number;
+    turnNumber: number;
+    totalTurns: number;
+    turnReport: TurnReport;
+} | {
+    type: 'eval_complete';
+    evalName: string;
+    index: number;
+    report: EvalReport;
+} | {
+    type: 'suite_complete';
+    report: EvalRunReport;
+    error?: string;
+};
+export interface EvalRunReport {
+    id: string;
+    timestamp: string;
+    evals: EvalReport[];
+    passed: number;
+    failed: number;
+    total: number;
+    duration: number;
+    filter?: {
+        names?: string[];
+        tags?: string[];
+        type?: 'capability' | 'regression';
+    };
+    /**
+     * True when the suite was aborted mid-run (e.g. user clicked Stop). The
+     * persisted report is partial — `evals` reflects only the completed evals,
+     * and `total` matches that count, not the original suite size.
+     */
+    aborted?: boolean;
+}
+export interface BotConnection {
+    webhookId: string;
+    botId: string;
+    token: string;
+    apiUrl: string;
+}
+export interface EvalLogger {
+    info: (message: string) => void;
+    warn: (message: string) => void;
+    error: (message: string) => void;
+}
+export interface EvalRunnerConfig {
+    credentials: {
+        token: string;
+        apiUrl: string;
+        botId: string;
+        workspaceId?: string;
+        webhookId?: string;
+    };
+    agentPath: string;
+    /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */
+    devServerUrl?: string;
+    /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */
+    devServerHeaders?: Record<string, string>;
+    /**
+     * Optional host-provided chat client. The CLI passes its bundled chat client
+     * here so the eval engine does not depend on @botpress/adk.
+     */
+    chatClient?: ChatClient;
+    onProgress?: (event: EvalProgressEvent) => void;
+    /** Logger for warnings and errors. Defaults to console. */
+    logger?: EvalLogger;
+    /**
+     * Optional abort signal — when fired, the suite stops starting new evals.
+     * The currently running eval continues to completion (LLM calls in flight
+     * cannot always be cancelled), but no further evals are started.
+     */
+    signal?: AbortSignal;
+    /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */
+    evalOptions?: {
+        idleTimeout?: number;
+        /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */
+        judgePassThreshold?: number;
+        /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */
+        judgeModel?: string;
+    };
+}
+export interface EvalFilter {
+    names?: string[];
+    tags?: string[];
+    type?: 'capability' | 'regression';
+}
+export interface StateMutation {
+    type: string;
+    changedKeys: string[];
+    previous: unknown;
+    current: unknown;
+    swappedToFile: boolean;
+}
+export interface WorkflowSpan {
+    name: string;
+    status: 'running' | 'ok' | 'error';
+    statusFinal?: string;
+}
+export interface TurnData {
+    messages: string[];
+    toolCalls: ToolCall[];
+    stateMutations: StateMutation[];
+    workflowSpans: WorkflowSpan[];
+    handlerDuration: number;
+    handlerStatus: 'ok' | 'error';
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,IAAI,EACJ,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,cAAc,CAAA;AAErB,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE/C,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAA;IACpB,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC;QAAE,YAAY,EAAE;YAAE,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAA;IAGjG,aAAa,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/C,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAC9C;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAA;CACzE;AAID,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,YAAY,EAAE,CAAA;IAC1B,IAAI,EAAE,OAAO,CAAA;IACb,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAA;IACnB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAA;CACrB;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAClC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,iBAAiB,EAAE,YAAY,EAAE,CAAA;IACjC,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAID,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3E;IACE,IAAI,EAAE,YAAY,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;CACpB,GACD;IACE,IAAI,EAAE,eAAe,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,UAAU,CAAA;CACvB,GACD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAIrE,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;QAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;QACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;KACnC,CAAA;IACD;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAID,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACjC;AAID,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;IACD,SAAS,EAAE,MAAM,CAAA;IACjB,oGAAoG;IACpG,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,kGAAkG;IAClG,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC;;;OAGG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAA;IAC/C,2DAA2D;IAC3D,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,4FAA4F;IAC5F,WAAW,CAAC,EAAE;QACZ,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,mIAAmI;QACnI,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,wFAAwF;QACxF,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;CACF;AAID,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;CACnC;AAID,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,OAAO,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;IAChB,aAAa,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;IAClC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,cAAc,EAAE,aAAa,EAAE,CAAA;IAC/B,aAAa,EAAE,YAAY,EAAE,CAAA;IAC7B,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,IAAI,GAAG,OAAO,CAAA;CAC9B"}

package/dist/types.js

@@ -0,0 +1,25 @@
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+export {
+  Eval
+};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/definition.ts"],
+  "sourcesContent": ["export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n"],
+  "mappings": ";AAuFO,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;",
+  "names": []
+}

package/package.json

@@ -0,0 +1,99 @@
+{
+  "name": "@botpress/evals",
+  "version": "1.18.3",
+  "description": "Evaluation definitions and runner for ADK-based Botpress agents",
+  "keywords": [
+    "adk",
+    "agent",
+    "ai",
+    "botpress",
+    "evals",
+    "evaluation"
+  ],
+  "license": "MIT",
+  "author": "Botpress",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/botpress/adk"
+  },
+  "files": [
+    "dist/**/*",
+    "package.json",
+    "README.md"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    },
+    "./client": {
+      "types": "./dist/client.d.ts",
+      "import": "./dist/client.js"
+    },
+    "./definition": {
+      "types": "./dist/definition.d.ts",
+      "import": "./dist/definition.js"
+    },
+    "./graders": {
+      "types": "./dist/graders/index.d.ts",
+      "import": "./dist/graders/index.js"
+    },
+    "./graders/*": {
+      "types": "./dist/graders/*.d.ts",
+      "import": "./dist/graders/*.js"
+    },
+    "./loader": {
+      "types": "./dist/loader.d.ts",
+      "import": "./dist/loader.js"
+    },
+    "./runner": {
+      "types": "./dist/runner.d.ts",
+      "import": "./dist/runner.js"
+    },
+    "./sse-collector": {
+      "types": "./dist/sse-collector.d.ts",
+      "import": "./dist/sse-collector.js"
+    },
+    "./trace": {
+      "types": "./dist/trace.d.ts",
+      "import": "./dist/trace.js"
+    },
+    "./transformer": {
+      "types": "./dist/transformer.d.ts",
+      "import": "./dist/transformer.js"
+    },
+    "./types": {
+      "types": "./dist/types.d.ts",
+      "import": "./dist/types.js"
+    }
+  },
+  "scripts": {
+    "build": "bun build.ts",
+    "clean": "rm -rf dist",
+    "prepublishOnly": "bun run clean && bun run build",
+    "test": "bun test",
+    "check:type": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@botpress/chat": "~0.5.5",
+    "@botpress/client": "~1.42.0",
+    "@botpress/cognitive": "~0.5.4"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.5",
+    "@types/node": "^22.19.3",
+    "esbuild": "^0.25.12",
+    "vitest": "^2.1.8"
+  },
+  "peerDependencies": {
+    "typescript": ">=4.5.0"
+  },
+  "engines": {
+    "bun": ">=1.3.13",
+    "node": ">=22.0.0"
+  },
+  "packageManager": "bun@1.3.13"
+}