@botpress/evals 2.0.1-beta.0 → 2.0.1-beta.1

package/dist/runner.d.ts

@@ -12,6 +12,20 @@ export declare function runEval(evalDef: EvalDefinition, connection: BotConnecti
     devServerUrl?: string;
     idleTimeout?: number;
     judgePassThreshold?: number;
+    /**
+     * Configure the LLM judge inside the runner's OWN graders module. A bundled embedder can hold a
+     * different copy of that module, so initializing it here guarantees the instance grading consults.
+     */
+    judge?: {
+        credentials: {
+            token: string;
+            apiUrl: string;
+            botId: string;
+            workspaceId?: string;
+        };
+        model?: string;
+        failClosed?: boolean;
+    };
     onProgress?: (event: EvalProgressEvent) => void | Promise<void>;
     evalIndex?: number;
     /** Forwarded from EvalRunnerConfig — lets per-turn workflow waits unblock immediately on Stop instead of burning the full idleTimeout each. */

package/dist/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EAGV,aAAa,EAEd,MAAM,SAAS,CAAA;AAGhB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAoHrD,wBAAsB,OAAO,CAC3B,OAAO,EAAE,cAAc,EACvB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,+IAA+I;IAC/I,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,UAAU,CAAC,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAA;IAC3C,MAAM,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACnC,WAAW,CAAC,EAAE,gBAAgB,CAAC,aAAa,CAAC,CAAA;IAC7C,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,MAAM,CAAA;CAChB,GACL,OAAO,CAAC,UAAU,CAAC,CA+UrB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CA2GxG"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EACV,cAAc,EACd,iBAAiB,EACjB,UAAU,EACV,aAAa,EACb,gBAAgB,EAChB,UAAU,EAGV,aAAa,EAEd,MAAM,SAAS,CAAA;AAGhB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AA0HrD,wBAAsB,OAAO,CAC3B,OAAO,EAAE,cAAc,EACvB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE;IACP,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B;;;OAGG;IACH,KAAK,CAAC,EAAE;QACN,WAAW,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,WAAW,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;QACnF,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,UAAU,CAAC,EAAE,OAAO,CAAA;KACrB,CAAA;IACD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,+IAA+I;IAC/I,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC,UAAU,CAAC,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAA;IAC3C,MAAM,CAAC,EAAE,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACnC,WAAW,CAAC,EAAE,gBAAgB,CAAC,aAAa,CAAC,CAAA;IAC7C,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,WAAW,CAAC,EAAE,MAAM,CAAA;CAChB,GACL,OAAO,CAAC,UAAU,CAAC,CAgWrB;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,CA2GxG"}

package/dist/runner.js

@@ -1,3 +1,6 @@
+// src/runner.ts
+import { Client as BpClientCtor } from "@botpress/client";
+
 // src/definition.ts
 var Eval = class {
   name;
@@ -392,10 +395,11 @@ var SSECollector = class _SSECollector {
       throw new Error("waitForTurnComplete aborted");
     }
     return new Promise((resolve2, reject) => {
+      const expected = accept ? [...accept].join(" or ") : "handler.conversation or handler.event";
       const timer = setTimeout(() => {
+        clearStartTimer();
         this._removeWaiter(waiter);
         opts.abortSignal?.removeEventListener("abort", onAbort);
-        const expected = accept ? [...accept].join(" or ") : "handler.conversation or handler.event";
         const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : `no ${expected} span completed`;
         reject(
           new Error(
@@ -403,8 +407,27 @@ var SSECollector = class _SSECollector {
           )
         );
       }, opts.timeout);
+      let startTimer;
+      const clearStartTimer = () => {
+        if (startTimer !== void 0) clearTimeout(startTimer);
+        startTimer = void 0;
+      };
+      if (opts.handlerStartTimeoutMs !== void 0 && opts.handlerStartTimeoutMs < opts.timeout) {
+        startTimer = setTimeout(() => {
+          if (this._hasNewHandlerSpan(accept)) return;
+          clearTimeout(timer);
+          this._removeWaiter(waiter);
+          opts.abortSignal?.removeEventListener("abort", onAbort);
+          reject(
+            new Error(
+              `no ${expected} span started within ${opts.handlerStartTimeoutMs}ms \u2014 no handler is subscribed to this turn's trigger, so the turn can never complete; diagnostics: ${this._formatTurnDiagnostics()}`
+            )
+          );
+        }, opts.handlerStartTimeoutMs);
+      }
       const onAbort = () => {
         clearTimeout(timer);
+        clearStartTimer();
         this._removeWaiter(waiter);
         reject(new Error("waitForTurnComplete aborted"));
       };
@@ -412,6 +435,7 @@ var SSECollector = class _SSECollector {
       const waiter = (_span) => {
         if (this._hasNewCompletedHandler(accept)) {
           clearTimeout(timer);
+          clearStartTimer();
           this._removeWaiter(waiter);
           opts.abortSignal?.removeEventListener("abort", onAbort);
           this._settleAfterHandler(settle).then(resolve2, reject);
@@ -713,6 +737,18 @@ ${value}` : value;
     }
     return false;
   }
+  /**
+   * Whether a turn-handler span has appeared since startTurn() — any status. Proves a handler was
+   * dispatched. Never consumes keys (unlike {@link _hasNewCompletedHandler}).
+   */
+  _hasNewHandlerSpan(acceptSpanNames) {
+    for (const [key, span] of this.spanMap) {
+      if (isTurnHandlerSpan(span) && (!acceptSpanNames || acceptSpanNames.has(span.name)) && !this.seenHandlerKeys.has(key)) {
+        return true;
+      }
+    }
+    return false;
+  }
   /**
    * Check if a handler.workflow span matches the given condition.
    */
@@ -1356,6 +1392,7 @@ import { randomUUID } from "crypto";
 var DEFAULT_IDLE_TIMEOUT = 3e4;
 var DEFAULT_DEV_SERVER_URL = "http://localhost:3001";
 var WORKFLOW_TRIGGER_TIMEOUT_MS = 5 * 60 * 1e3;
+var EVENT_HANDLER_START_TIMEOUT_MS = 3e4;
 var BUILT_IN_STATES = {
   conversation: "state",
   user: "userState",
@@ -1435,6 +1472,14 @@ async function runEval(evalDef, connection, options = {}) {
   const logger = options.logger ?? defaultLogger;
   const idleTimeout = evalDef.options?.idleTimeout ?? options.idleTimeout ?? DEFAULT_IDLE_TIMEOUT;
   const judgePassThreshold = evalDef.options?.judgePassThreshold ?? options.judgePassThreshold;
+  if (options.judge) {
+    const judgeClient = new BpClientCtor(options.judge.credentials);
+    await initLLMJudge(judgeClient, {
+      ...options.judge.model !== void 0 ? { model: options.judge.model } : {},
+      ...options.judge.failClosed !== void 0 ? { failClosed: options.judge.failClosed } : {},
+      logger
+    });
+  }
   const start = Date.now();
   const turns = [];
   let outcomeAssertions = [];
@@ -1526,6 +1571,8 @@ async function runEval(evalDef, connection, options = {}) {
           // can get it absorbed by their transcript saves and silently skipped.
           settleQuietMs: 1500,
           settleMaxMs: 15e3,
+          // Event turns fail fast when nothing is subscribed (see the constant).
+          ...turn.event ? { handlerStartTimeoutMs: EVENT_HANDLER_START_TIMEOUT_MS } : {},
           ...options.signal !== void 0 ? { abortSignal: options.signal } : {}
         });
       } catch (err) {
@@ -1545,8 +1592,13 @@ async function runEval(evalDef, connection, options = {}) {
           });
         }
       }
-      const turnSpans = collector.getTurnSpans();
-      const turnData = transformSpans(turnSpans);
+      let turnData = transformSpans(collector.getTurnSpans());
+      if (turn.event && !turn.expectSilence && turnData.messages.length === 0 && turnData.handlerDuration > 0) {
+        for (let waited = 0; waited < 600 && turnData.messages.length === 0; waited += 150) {
+          await new Promise((resolve2) => setTimeout(resolve2, 150));
+          turnData = transformSpans(collector.getTurnSpans());
+        }
+      }
       const botResponse = turnData.messages.join("\n");
       const botDuration = turnData.handlerDuration;
       const evalStart = Date.now();