@botpress/evals 1.19.0-beta.12 → 1.19.0-beta.14

package/dist/runner.js

@@ -1,6 +1,3 @@
-// src/runner.ts
-import { Client as BpClient2 } from "@botpress/client";
-
 // src/definition.ts
 var Eval = class {
   name;
@@ -26,84 +23,33 @@ var Eval = class {
 // src/types.ts
 var defaultLogger = console;
 
-// src/client.ts
-import { Client as BpClient } from "@botpress/client";
-import { createRequire } from "module";
-
-// ../analytics/src/errors.ts
-var AdkError = class extends Error {
-  /**
-   * Marker for cross-module-instance checks — `instanceof` breaks when a
-   * package is duplicated in the dependency graph.
-   * @internal
-   */
-  static __IS_ADK_BASE_ERROR = true;
-  /**
-   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
-   * pin `Code` to a literal union (`class EvalRunnerError extends
-   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
-   * in switches.
-   */
-  code;
-  /**
-   * True for user/environment conditions the developer can fix themselves
-   * (validation, auth, connectivity). False for internal bugs. Defaults to
-   * false so unclassified errors surface as bugs until someone triages them.
-   */
-  expected;
-  /** Structured context. Sanitized before any telemetry leaves the machine. */
-  details;
-  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
-  suggestion;
-  constructor(opts) {
-    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
-    this.name = this.constructor.name;
-    this.code = opts.code;
-    this.expected = opts.expected ?? false;
-    if (opts.details !== void 0) {
-      this.details = opts.details;
-    }
-    if (opts.suggestion !== void 0) {
-      this.suggestion = opts.suggestion;
-    }
-  }
-};
-function isAdkError(error) {
-  if (error instanceof AdkError) return true;
-  if (!(error instanceof Error)) return false;
-  return error.constructor?.__IS_ADK_BASE_ERROR === true;
-}
-
 // src/errors.ts
+import { AdkError } from "@botpress/analytics";
 var EvalRunnerError = class extends AdkError {
 };
 
 // src/client.ts
 var TRACE_STREAM_PREFLIGHT_TIMEOUT_MS = 1e4;
-var defaultChatClient = null;
-function getDefaultChatClient() {
-  if (defaultChatClient) return defaultChatClient;
-  const require2 = createRequire(import.meta.url);
-  const mod = require2("@botpress/chat");
-  const ChatClient = mod.default?.Client ?? mod.Client;
-  if (!ChatClient) {
-    throw new EvalRunnerError({
-      code: "CHAT_CLIENT_MISSING",
-      message: "@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly."
-    });
-  }
-  return ChatClient;
-}
 var ChatSession = class {
-  constructor(webhookId, chatClient) {
-    this.webhookId = webhookId;
-    this.chatClient = chatClient;
+  constructor(bpClient, botId, _chatWebhookId, _chatBaseUrl, _chatClient) {
+    this.bpClient = bpClient;
+    this.botId = botId;
+    this._chatWebhookId = _chatWebhookId;
+    this._chatBaseUrl = _chatBaseUrl;
+    this._chatClient = _chatClient;
   }
   client = null;
   conversationId = null;
   async connect() {
-    const ChatClient = this.chatClient ?? getDefaultChatClient();
-    this.client = await ChatClient.connect({ webhookId: this.webhookId });
+    const webhookId = this._chatWebhookId ?? await discoverWebhookId(this.bpClient, this.botId);
+    const ChatClientCtor = this._chatClient ?? (await import(
+      /* webpackIgnore: true */
+      "@botpress/chat"
+    )).Client;
+    this.client = await ChatClientCtor.connect({
+      webhookId,
+      ...this._chatBaseUrl ? { baseApiUrl: this._chatBaseUrl } : {}
+    });
   }
   /** Invariant guard — calling any session method before connect() is a runner bug. */
   assertConnected() {
@@ -135,17 +81,13 @@ var ChatSession = class {
     const conversationId = await this.ensureConversation();
     await client.createMessage({ conversationId, payload: { type: "text", text: message } });
   }
-  /**
-   * Send an event. Fire-and-forget — observation is via SSECollector.
-   */
-  async sendEvent(type, payload) {
+  async sendEvent(payload) {
     const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await client.createEvent({ type, payload, conversationId });
+    await client.createEvent({ payload, conversationId });
   }
 };
-async function discoverWebhookId(botId, token, apiUrl) {
-  const client = new BpClient({ token, botId, apiUrl });
+async function discoverWebhookId(client, botId) {
   const { bot } = await client.getBot({ id: botId });
   const integrations = bot.integrations || {};
   const chat = Object.values(integrations).find((int) => int.name === "chat");
@@ -273,13 +215,17 @@ async function assertChatChannelBound(devServerUrl, headers = {}, logger = defau
   });
 }
 
-// src/sse-collector.ts
+// src/spans/sse-collector.ts
 function spanKey(span) {
   return `${span.id.trace}:${span.id.span}`;
 }
 function spanData(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
 }
+var TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
+function isTurnHandlerSpan(span) {
+  return TURN_HANDLER_SPAN_NAMES.has(span.name);
+}
 var SSECollector = class _SSECollector {
   devServerUrl;
   headers;
@@ -341,20 +287,22 @@ var SSECollector = class _SSECollector {
   }
   /**
    * Mark the beginning of a new turn.
-   * Records which handler.conversation span IDs have been seen so far,
-   * so waitForTurnComplete() only resolves on NEW handler completions.
+   * Records which turn-handler span IDs (handler.conversation / handler.event)
+   * have been seen so far, so waitForTurnComplete() only resolves on NEW
+   * handler completions.
    */
   startTurn() {
     this.turnStartKeys = new Set(this.spanMap.keys());
     for (const [key, span] of this.spanMap) {
-      if (span.name === "handler.conversation") {
+      if (isTurnHandlerSpan(span)) {
         this.seenHandlerKeys.add(key);
       }
     }
   }
   static SETTLE_MS = 100;
   /**
-   * Wait for a handler.conversation span to reach terminal status (ok or error).
+   * Wait for a turn-handler span (handler.conversation or handler.event) to
+   * reach terminal status (ok or error).
    * Only considers spans NOT seen before the last startTurn() call.
    *
    * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)
@@ -373,7 +321,7 @@ var SSECollector = class _SSECollector {
       const timer = setTimeout(() => {
         this._removeWaiter(waiter);
         opts.abortSignal?.removeEventListener("abort", onAbort);
-        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation span completed";
+        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation or handler.event span completed";
         reject(
           new Error(
             `waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}; diagnostics: ${this._formatTurnDiagnostics()}`
@@ -621,7 +569,7 @@ ${value}` : value;
     const handlerStatuses = /* @__PURE__ */ new Set();
     for (const span of turnSpans) {
       spanNames.add(span.name);
-      if (span.name === "handler.conversation") {
+      if (isTurnHandlerSpan(span)) {
         handlerStatuses.add(span.status);
       }
     }
@@ -634,7 +582,7 @@ ${value}` : value;
       `totalSpans=${this.spanMap.size}`,
       `turnSpans=${turnSpans.length}`,
       `turnSpanNames=${this._formatSet(spanNames)}`,
-      `handlerConversationStatuses=${this._formatSet(handlerStatuses)}`
+      `turnHandlerStatuses=${this._formatSet(handlerStatuses)}`
     ].join(", ");
   }
   _formatSet(values, limit = 12) {
@@ -670,7 +618,7 @@ ${value}` : value;
     }
   }
   /**
-   * Check if there is a handler.conversation span that:
+   * Check if there is a turn-handler span (handler.conversation / handler.event) that:
    * 1. Was NOT seen before startTurn()
    * 2. Has terminal status (ok or error)
    *
@@ -681,7 +629,7 @@ ${value}` : value;
    */
   _hasNewCompletedHandler() {
     for (const [key, span] of this.spanMap) {
-      if (span.name === "handler.conversation" && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
+      if (isTurnHandlerSpan(span) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
         this.seenHandlerKeys.add(key);
         return true;
       }
@@ -708,6 +656,35 @@ ${value}` : value;
   }
 };
 
+// src/spans/local-span-source.ts
+var LocalSpanSource = class {
+  collector;
+  constructor(devServerUrl, headers = {}) {
+    this.collector = new SSECollector(devServerUrl, headers);
+  }
+  async connect(filter) {
+    await this.collector.connect(filter);
+  }
+  startTurn() {
+    this.collector.startTurn();
+  }
+  async waitForTurnComplete(opts) {
+    await this.collector.waitForTurnComplete(opts);
+  }
+  async waitForWorkflow(name, opts) {
+    await this.collector.waitForWorkflow(name, opts);
+  }
+  getTurnSpans() {
+    return this.collector.getTurnSpans();
+  }
+  getAllSpans() {
+    return this.collector.getAllSpans();
+  }
+  disconnect() {
+    this.collector.disconnect();
+  }
+};
+
 // src/transformer.ts
 function data(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
@@ -795,7 +772,7 @@ function extractWorkflowSpans(spans) {
 }
 function extractHandler(spans) {
   const handler = spans.find(
-    (s) => (s.name === "handler.conversation" || s.name === "handler.workflow") && s.status !== "running"
+    (s) => (s.name === "handler.conversation" || s.name === "handler.event" || s.name === "handler.workflow") && s.status !== "running"
   );
   if (!handler) {
     return { handlerDuration: 0, handlerStatus: "ok" };
@@ -807,55 +784,26 @@ function extractHandler(spans) {
 }
 
 // src/graders/llm.ts
-import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
-var DEFAULT_PASS_THRESHOLD = 3;
-function buildJudgeSystemPrompt(passThreshold) {
-  return `You are an evaluation judge for a chatbot. You will be given:
-- The user's message
-- The bot's response
-- Grading criteria
-
-Your job is to determine if the bot's response meets the criteria.
-
-Respond with a JSON object:
-{
-  "pass": true/false,
-  "score": 1-5,
-  "reason": "brief explanation"
-}
-
-Scoring guide:
-- 5: Fully meets criteria, excellent response
-- 4: Mostly meets criteria, minor issues
-- 3: Partially meets criteria, notable gaps
-- 2: Barely meets criteria, significant issues
-- 1: Does not meet criteria
-
-A score of ${passThreshold} or above is a pass.`;
-}
+import { Zai } from "@botpress/zai";
 var MODEL_ALIASES = ["fast", "best"];
-var _cognitive = null;
+var _zai = null;
 var _availableModels = [];
 var _judgeModel = "fast";
+var _failClosed = true;
 var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
-async function initLLMJudge(credentials, options) {
+async function initLLMJudge(client, options) {
   const log = options?.logger ?? defaultLogger;
   _log = log;
-  const client = new Client({
-    token: credentials.token,
-    apiUrl: credentials.apiUrl,
-    botId: credentials.botId,
-    ...credentials.workspaceId !== void 0 ? { workspaceId: credentials.workspaceId } : {}
-  });
-  _cognitive = new Cognitive({ client, __experimental_beta: true });
+  _failClosed = options?.failClosed ?? true;
+  const cognitive = new Cognitive({ client, __experimental_beta: true });
   try {
-    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys());
+    _availableModels = Array.from((await cognitive.fetchRemoteModels()).keys());
   } catch (err) {
     log.warn(`Failed to fetch available models: ${err.message}. LLM judge will be unavailable.`);
     _availableModels = [];
@@ -866,66 +814,42 @@ async function initLLMJudge(credentials, options) {
   } else if (!_isModelValid(_judgeModel)) {
     log.warn(`Configured LLM judge model "${_judgeModel}" is invalid. Run "adk models" to list available models.`);
   }
+  _zai = new Zai({ client: cognitive, modelId: _judgeModel });
 }
 async function gradeLLMJudge(botResponse, criteria, context) {
-  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
+  const assertion = `llm_judge: "${criteria}"`;
   try {
-    if (!_cognitive) {
+    if (!_zai) {
       _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
-        assertion: `llm_judge: "${criteria}"`,
-        pass: false,
+        assertion,
+        pass: !_failClosed,
+        skipped: true,
         expected: criteria,
         actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
       return {
-        assertion: `llm_judge: "${criteria}"`,
+        assertion,
         pass: false,
         expected: criteria,
         actual: "FAILED \u2014 invalid LLM judge model"
       };
     }
-    const { output } = await _cognitive.generateContent({
-      model: _judgeModel,
-      temperature: 0,
-      responseFormat: "json_object",
-      systemPrompt: buildJudgeSystemPrompt(threshold),
-      messages: [
-        {
-          role: "user",
-          content: `User message: ${context.userMessage}
-
-Bot response: ${botResponse}
-
-Criteria: ${criteria}`
-        }
-      ]
-    });
-    const rawContent = output?.choices?.[0]?.content;
-    const content = typeof rawContent === "string" ? rawContent : null;
-    if (!content) {
-      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
-      return {
-        assertion: `llm_judge: "${criteria}"`,
-        pass: false,
-        expected: criteria,
-        actual: "FAILED \u2014 LLM judge returned empty response"
-      };
-    }
-    const verdict = JSON.parse(content);
+    const { output } = await _zai.check({ userMessage: context.userMessage, botResponse }, criteria).result();
     return {
-      assertion: `llm_judge: "${criteria}"`,
-      pass: verdict.score >= threshold,
+      assertion,
+      pass: output.value,
       expected: criteria,
-      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
+      actual: `${output.value ? "Pass" : "Fail"} \u2014 ${output.explanation}`
     };
   } catch (err) {
     _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
-      assertion: `llm_judge: "${criteria}"`,
-      pass: false,
+      assertion,
+      pass: !_failClosed,
+      skipped: true,
       expected: criteria,
       actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
@@ -1348,6 +1272,7 @@ function filterEvals(evals, filter) {
 }
 
 // src/runner.ts
+import { isAdkError } from "@botpress/analytics";
 import { randomUUID } from "crypto";
 var DEFAULT_IDLE_TIMEOUT = 3e4;
 var DEFAULT_DEV_SERVER_URL = "http://localhost:3001";
@@ -1434,27 +1359,22 @@ async function runEval(evalDef, connection, options = {}) {
   const start = Date.now();
   const turns = [];
   let outcomeAssertions = [];
-  const collector = new SSECollector(devServerUrl, devServerHeaders);
+  const collector = options.spanSource ?? new LocalSpanSource(devServerUrl, devServerHeaders);
   try {
-    const session = new ChatSession(connection.webhookId, options.chatClient);
+    const session = new ChatSession(
+      connection.client,
+      connection.botId,
+      options.chatWebhookId,
+      options.chatBaseUrl,
+      options.chatClient
+    );
     await session.connect();
-    let bpClient = null;
-    const getBpClient = () => {
-      if (!bpClient) {
-        bpClient = new BpClient2({
-          token: connection.token,
-          botId: connection.botId,
-          apiUrl: connection.apiUrl
-        });
-      }
-      return bpClient;
-    };
     let conversationId = "";
     if (evalDef.setup?.state?.conversation || evalDef.setup?.workflow) {
       conversationId = await session.ensureConversation();
     }
     await seedEvalState(
-      getBpClient(),
+      connection.client,
       {
         botId: connection.botId,
         userId: session.userId,
@@ -1463,7 +1383,7 @@ async function runEval(evalDef, connection, options = {}) {
       evalDef.setup
     );
     await triggerEvalWorkflow(
-      getBpClient(),
+      connection.client,
       {
         userId: session.userId,
         ...conversationId ? { conversationId } : {}
@@ -1499,18 +1419,18 @@ async function runEval(evalDef, connection, options = {}) {
       if (!turn.user && !turn.event) {
         throw turnConfigError(`must have either 'user' or 'event'.`);
       }
-      const turnLabelForEvent = turn.event ? `[event: ${turn.event.type}]` : turn.user;
-      options.onProgress?.({
+      const turnLabel = turn.event ? "[event]" : turn.user;
+      await options.onProgress?.({
         type: "turn_start",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnNumber: i + 1,
+        turnIndex: i,
         totalTurns: evalDef.conversation.length,
-        userMessage: turnLabelForEvent
+        userMessage: turnLabel
       });
       collector.startTurn();
       if (turn.event) {
-        await session.sendEvent(turn.event.type, turn.event.payload ?? {});
+        await session.sendEvent(turn.event.payload);
       } else {
         await session.sendMessage(turn.user);
       }
@@ -1540,7 +1460,6 @@ async function runEval(evalDef, connection, options = {}) {
       const turnData = transformSpans(turnSpans);
       const botResponse = turnData.messages.join("\n");
       const botDuration = turnData.handlerDuration;
-      const turnLabel = turn.event ? `[event: ${turn.event.type}]` : turn.user;
       const evalStart = Date.now();
       let assertions = [];
       const noResponse = !turn.expectSilence && turnData.messages.length === 0;
@@ -1595,7 +1514,7 @@ async function runEval(evalDef, connection, options = {}) {
       const turnPass = assertions.every((a) => a.pass);
       const evalDuration = Date.now() - evalStart;
       const turnReport = {
-        turnNumber: i + 1,
+        turnIndex: i,
         userMessage: turnLabel,
         botResponse,
         assertions,
@@ -1604,11 +1523,11 @@ async function runEval(evalDef, connection, options = {}) {
         evalDuration
       };
       turns.push(turnReport);
-      options.onProgress?.({
+      await options.onProgress?.({
         type: "turn_complete",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnNumber: i + 1,
+        turnIndex: i,
         totalTurns: evalDef.conversation.length,
         turnReport
       });
@@ -1655,7 +1574,15 @@ async function runEval(evalDef, connection, options = {}) {
   } catch (err) {
     const adkErr = isAdkError(err) ? err : void 0;
     if (!adkErr || !adkErr.expected) {
-      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${err.message}`);
+      const errMsg = err.message ?? String(err);
+      const errStack = err.stack ?? "";
+      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${errMsg}`);
+      logger.error(`  stack: ${errStack}`);
+      logger.error(`  eval: ${evalDef.name}, turn: ${turns.length}/${evalDef.conversation.length}`);
+      logger.error(`  connection: botId=${connection.botId}`);
+      logger.error(
+        `  spanSource: ${options.spanSource?.constructor?.name ?? "default"}, devServerUrl: ${options.devServerUrl ?? "n/a"}`
+      );
       options.onException?.(err, { source: "evals", eval_name: evalDef.name });
     }
     return {
@@ -1673,9 +1600,15 @@ async function runEval(evalDef, connection, options = {}) {
 }
 async function runEvalSuite(config, filter) {
   const start = Date.now();
-  const runId = randomUUID().replace(/-/g, "").slice(0, 26);
-  const evalsDir = `${config.agentPath}/evals`;
-  const allEvals = await loadEvalsFromDir(evalsDir);
+  const runId = config.runId ?? randomUUID().replace(/-/g, "").slice(0, 26);
+  let allEvals;
+  if (config.definitions) {
+    allEvals = config.definitions;
+  } else if (config.agentPath) {
+    allEvals = await loadEvalsFromDir(`${config.agentPath}/evals`);
+  } else {
+    throw new Error("Either `definitions` or `agentPath` must be provided in EvalRunnerConfig");
+  }
   const evals = filterEvals(allEvals, filter);
   if (evals.length === 0) {
     return {
@@ -1689,38 +1622,26 @@ async function runEvalSuite(config, filter) {
       ...filter !== void 0 ? { filter } : {}
     };
   }
-  await initLLMJudge(
-    {
-      token: config.credentials.token,
-      apiUrl: config.credentials.apiUrl,
-      botId: config.credentials.botId,
-      ...config.credentials.workspaceId !== void 0 ? { workspaceId: config.credentials.workspaceId } : {}
-    },
-    {
-      ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
-      ...config.logger !== void 0 ? { logger: config.logger } : {}
-    }
-  );
-  let webhookId = config.credentials.webhookId;
-  if (!webhookId) {
-    webhookId = await discoverWebhookId(config.credentials.botId, config.credentials.token, config.credentials.apiUrl);
-  }
+  await initLLMJudge(config.client, {
+    ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
+    ...config.logger !== void 0 ? { logger: config.logger } : {}
+  });
   const connection = {
-    webhookId,
-    botId: config.credentials.botId,
-    token: config.credentials.token,
-    apiUrl: config.credentials.apiUrl
+    client: config.client,
+    botId: config.botId
   };
   const devServerUrl = config.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = config.devServerHeaders ?? {};
-  await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
-  await assertTraceStreamReadable(devServerUrl, devServerHeaders);
+  if (!config.createSpanSource) {
+    await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
+    await assertTraceStreamReadable(devServerUrl, devServerHeaders);
+  }
   const reports = [];
-  config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
+  await config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
   for (let i = 0; i < evals.length; i++) {
     if (config.signal?.aborted) break;
     const evalDef = evals[i];
-    config.onProgress?.({
+    await config.onProgress?.({
       type: "eval_start",
       evalName: evalDef.name,
       index: i,
@@ -1736,10 +1657,13 @@ async function runEvalSuite(config, filter) {
       ...config.signal !== void 0 ? { signal: config.signal } : {},
       ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {},
       ...config.logger !== void 0 ? { logger: config.logger } : {},
-      ...config.onException !== void 0 ? { onException: config.onException } : {}
+      ...config.onException !== void 0 ? { onException: config.onException } : {},
+      ...config.createSpanSource !== void 0 ? { spanSource: config.createSpanSource() } : {},
+      ...config.chatWebhookId !== void 0 ? { chatWebhookId: config.chatWebhookId } : {},
+      ...config.chatBaseUrl !== void 0 ? { chatBaseUrl: config.chatBaseUrl } : {}
     });
     reports.push(report);
-    config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
+    await config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
   }
   const runReport = {
     id: runId,
@@ -1754,7 +1678,7 @@ async function runEvalSuite(config, filter) {
   if (config.signal?.aborted) {
     runReport.aborted = true;
   }
-  config.onProgress?.({ type: "suite_complete", report: runReport });
+  await config.onProgress?.({ type: "suite_complete", report: runReport });
   return runReport;
 }
 export {