@botpress/evals 1.19.0-beta.14 → 1.19.0-beta.16

package/dist/runner.js

@@ -1,3 +1,6 @@
+// src/runner.ts
+import { Client as BpClient2 } from "@botpress/client";
+
 // src/definition.ts
 var Eval = class {
   name;
@@ -23,33 +26,84 @@ var Eval = class {
 // src/types.ts
 var defaultLogger = console;
 
+// src/client.ts
+import { Client as BpClient } from "@botpress/client";
+import { createRequire } from "module";
+
+// ../analytics/src/errors.ts
+var AdkError = class extends Error {
+  /**
+   * Marker for cross-module-instance checks — `instanceof` breaks when a
+   * package is duplicated in the dependency graph.
+   * @internal
+   */
+  static __IS_ADK_BASE_ERROR = true;
+  /**
+   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
+   * pin `Code` to a literal union (`class EvalRunnerError extends
+   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
+   * in switches.
+   */
+  code;
+  /**
+   * True for user/environment conditions the developer can fix themselves
+   * (validation, auth, connectivity). False for internal bugs. Defaults to
+   * false so unclassified errors surface as bugs until someone triages them.
+   */
+  expected;
+  /** Structured context. Sanitized before any telemetry leaves the machine. */
+  details;
+  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
+  suggestion;
+  constructor(opts) {
+    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
+    this.name = this.constructor.name;
+    this.code = opts.code;
+    this.expected = opts.expected ?? false;
+    if (opts.details !== void 0) {
+      this.details = opts.details;
+    }
+    if (opts.suggestion !== void 0) {
+      this.suggestion = opts.suggestion;
+    }
+  }
+};
+function isAdkError(error) {
+  if (error instanceof AdkError) return true;
+  if (!(error instanceof Error)) return false;
+  return error.constructor?.__IS_ADK_BASE_ERROR === true;
+}
+
 // src/errors.ts
-import { AdkError } from "@botpress/analytics";
 var EvalRunnerError = class extends AdkError {
 };
 
 // src/client.ts
 var TRACE_STREAM_PREFLIGHT_TIMEOUT_MS = 1e4;
+var defaultChatClient = null;
+function getDefaultChatClient() {
+  if (defaultChatClient) return defaultChatClient;
+  const require2 = createRequire(import.meta.url);
+  const mod = require2("@botpress/chat");
+  const ChatClient = mod.default?.Client ?? mod.Client;
+  if (!ChatClient) {
+    throw new EvalRunnerError({
+      code: "CHAT_CLIENT_MISSING",
+      message: "@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly."
+    });
+  }
+  return ChatClient;
+}
 var ChatSession = class {
-  constructor(bpClient, botId, _chatWebhookId, _chatBaseUrl, _chatClient) {
-    this.bpClient = bpClient;
-    this.botId = botId;
-    this._chatWebhookId = _chatWebhookId;
-    this._chatBaseUrl = _chatBaseUrl;
-    this._chatClient = _chatClient;
+  constructor(webhookId, chatClient) {
+    this.webhookId = webhookId;
+    this.chatClient = chatClient;
   }
   client = null;
   conversationId = null;
   async connect() {
-    const webhookId = this._chatWebhookId ?? await discoverWebhookId(this.bpClient, this.botId);
-    const ChatClientCtor = this._chatClient ?? (await import(
-      /* webpackIgnore: true */
-      "@botpress/chat"
-    )).Client;
-    this.client = await ChatClientCtor.connect({
-      webhookId,
-      ...this._chatBaseUrl ? { baseApiUrl: this._chatBaseUrl } : {}
-    });
+    const ChatClient = this.chatClient ?? getDefaultChatClient();
+    this.client = await ChatClient.connect({ webhookId: this.webhookId });
   }
   /** Invariant guard — calling any session method before connect() is a runner bug. */
   assertConnected() {
@@ -81,13 +135,17 @@ var ChatSession = class {
     const conversationId = await this.ensureConversation();
     await client.createMessage({ conversationId, payload: { type: "text", text: message } });
   }
-  async sendEvent(payload) {
+  /**
+   * Send an event. Fire-and-forget — observation is via SSECollector.
+   */
+  async sendEvent(type, payload) {
     const client = this.assertConnected();
     const conversationId = await this.ensureConversation();
-    await client.createEvent({ payload, conversationId });
+    await client.createEvent({ type, payload, conversationId });
   }
 };
-async function discoverWebhookId(client, botId) {
+async function discoverWebhookId(botId, token, apiUrl) {
+  const client = new BpClient({ token, botId, apiUrl });
   const { bot } = await client.getBot({ id: botId });
   const integrations = bot.integrations || {};
   const chat = Object.values(integrations).find((int) => int.name === "chat");
@@ -215,17 +273,13 @@ async function assertChatChannelBound(devServerUrl, headers = {}, logger = defau
   });
 }
 
-// src/spans/sse-collector.ts
+// src/sse-collector.ts
 function spanKey(span) {
   return `${span.id.trace}:${span.id.span}`;
 }
 function spanData(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
 }
-var TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
-function isTurnHandlerSpan(span) {
-  return TURN_HANDLER_SPAN_NAMES.has(span.name);
-}
 var SSECollector = class _SSECollector {
   devServerUrl;
   headers;
@@ -287,22 +341,20 @@ var SSECollector = class _SSECollector {
   }
   /**
    * Mark the beginning of a new turn.
-   * Records which turn-handler span IDs (handler.conversation / handler.event)
-   * have been seen so far, so waitForTurnComplete() only resolves on NEW
-   * handler completions.
+   * Records which handler.conversation span IDs have been seen so far,
+   * so waitForTurnComplete() only resolves on NEW handler completions.
    */
   startTurn() {
     this.turnStartKeys = new Set(this.spanMap.keys());
     for (const [key, span] of this.spanMap) {
-      if (isTurnHandlerSpan(span)) {
+      if (span.name === "handler.conversation") {
         this.seenHandlerKeys.add(key);
       }
     }
   }
   static SETTLE_MS = 100;
   /**
-   * Wait for a turn-handler span (handler.conversation or handler.event) to
-   * reach terminal status (ok or error).
+   * Wait for a handler.conversation span to reach terminal status (ok or error).
    * Only considers spans NOT seen before the last startTurn() call.
    *
    * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)
@@ -321,7 +373,7 @@ var SSECollector = class _SSECollector {
       const timer = setTimeout(() => {
         this._removeWaiter(waiter);
         opts.abortSignal?.removeEventListener("abort", onAbort);
-        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation or handler.event span completed";
+        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation span completed";
         reject(
           new Error(
             `waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}; diagnostics: ${this._formatTurnDiagnostics()}`
@@ -569,7 +621,7 @@ ${value}` : value;
     const handlerStatuses = /* @__PURE__ */ new Set();
     for (const span of turnSpans) {
       spanNames.add(span.name);
-      if (isTurnHandlerSpan(span)) {
+      if (span.name === "handler.conversation") {
         handlerStatuses.add(span.status);
       }
     }
@@ -582,7 +634,7 @@ ${value}` : value;
       `totalSpans=${this.spanMap.size}`,
       `turnSpans=${turnSpans.length}`,
       `turnSpanNames=${this._formatSet(spanNames)}`,
-      `turnHandlerStatuses=${this._formatSet(handlerStatuses)}`
+      `handlerConversationStatuses=${this._formatSet(handlerStatuses)}`
     ].join(", ");
   }
   _formatSet(values, limit = 12) {
@@ -618,7 +670,7 @@ ${value}` : value;
     }
   }
   /**
-   * Check if there is a turn-handler span (handler.conversation / handler.event) that:
+   * Check if there is a handler.conversation span that:
    * 1. Was NOT seen before startTurn()
    * 2. Has terminal status (ok or error)
    *
@@ -629,7 +681,7 @@ ${value}` : value;
    */
   _hasNewCompletedHandler() {
     for (const [key, span] of this.spanMap) {
-      if (isTurnHandlerSpan(span) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
+      if (span.name === "handler.conversation" && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
         this.seenHandlerKeys.add(key);
         return true;
       }
@@ -656,35 +708,6 @@ ${value}` : value;
   }
 };
 
-// src/spans/local-span-source.ts
-var LocalSpanSource = class {
-  collector;
-  constructor(devServerUrl, headers = {}) {
-    this.collector = new SSECollector(devServerUrl, headers);
-  }
-  async connect(filter) {
-    await this.collector.connect(filter);
-  }
-  startTurn() {
-    this.collector.startTurn();
-  }
-  async waitForTurnComplete(opts) {
-    await this.collector.waitForTurnComplete(opts);
-  }
-  async waitForWorkflow(name, opts) {
-    await this.collector.waitForWorkflow(name, opts);
-  }
-  getTurnSpans() {
-    return this.collector.getTurnSpans();
-  }
-  getAllSpans() {
-    return this.collector.getAllSpans();
-  }
-  disconnect() {
-    this.collector.disconnect();
-  }
-};
-
 // src/transformer.ts
 function data(span) {
   return span.data && typeof span.data === "object" ? span.data : {};
@@ -772,7 +795,7 @@ function extractWorkflowSpans(spans) {
 }
 function extractHandler(spans) {
   const handler = spans.find(
-    (s) => (s.name === "handler.conversation" || s.name === "handler.event" || s.name === "handler.workflow") && s.status !== "running"
+    (s) => (s.name === "handler.conversation" || s.name === "handler.workflow") && s.status !== "running"
   );
   if (!handler) {
     return { handlerDuration: 0, handlerStatus: "ok" };
@@ -784,26 +807,55 @@ function extractHandler(spans) {
 }
 
 // src/graders/llm.ts
+import { Client } from "@botpress/client";
 import { Cognitive } from "@botpress/cognitive";
-import { Zai } from "@botpress/zai";
+var DEFAULT_PASS_THRESHOLD = 3;
+function buildJudgeSystemPrompt(passThreshold) {
+  return `You are an evaluation judge for a chatbot. You will be given:
+- The user's message
+- The bot's response
+- Grading criteria
+
+Your job is to determine if the bot's response meets the criteria.
+
+Respond with a JSON object:
+{
+  "pass": true/false,
+  "score": 1-5,
+  "reason": "brief explanation"
+}
+
+Scoring guide:
+- 5: Fully meets criteria, excellent response
+- 4: Mostly meets criteria, minor issues
+- 3: Partially meets criteria, notable gaps
+- 2: Barely meets criteria, significant issues
+- 1: Does not meet criteria
+
+A score of ${passThreshold} or above is a pass.`;
+}
 var MODEL_ALIASES = ["fast", "best"];
-var _zai = null;
+var _cognitive = null;
 var _availableModels = [];
 var _judgeModel = "fast";
-var _failClosed = true;
 var _log = defaultLogger;
 function _isModelValid(model) {
   if (MODEL_ALIASES.includes(model)) return true;
   if (_availableModels.length === 0) return false;
   return _availableModels.includes(model);
 }
-async function initLLMJudge(client, options) {
+async function initLLMJudge(credentials, options) {
   const log = options?.logger ?? defaultLogger;
   _log = log;
-  _failClosed = options?.failClosed ?? true;
-  const cognitive = new Cognitive({ client, __experimental_beta: true });
+  const client = new Client({
+    token: credentials.token,
+    apiUrl: credentials.apiUrl,
+    botId: credentials.botId,
+    ...credentials.workspaceId !== void 0 ? { workspaceId: credentials.workspaceId } : {}
+  });
+  _cognitive = new Cognitive({ client, __experimental_beta: true });
   try {
-    _availableModels = Array.from((await cognitive.fetchRemoteModels()).keys());
+    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys());
   } catch (err) {
     log.warn(`Failed to fetch available models: ${err.message}. LLM judge will be unavailable.`);
     _availableModels = [];
@@ -814,42 +866,66 @@ async function initLLMJudge(client, options) {
   } else if (!_isModelValid(_judgeModel)) {
     log.warn(`Configured LLM judge model "${_judgeModel}" is invalid. Run "adk models" to list available models.`);
   }
-  _zai = new Zai({ client: cognitive, modelId: _judgeModel });
 }
 async function gradeLLMJudge(botResponse, criteria, context) {
-  const assertion = `llm_judge: "${criteria}"`;
+  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
   try {
-    if (!_zai) {
+    if (!_cognitive) {
       _log.warn(`LLM judge unavailable for "${criteria}": no credentials configured (initLLMJudge not called)`);
       return {
-        assertion,
-        pass: !_failClosed,
-        skipped: true,
+        assertion: `llm_judge: "${criteria}"`,
+        pass: false,
         expected: criteria,
         actual: "FAILED \u2014 LLM judge unavailable: no credentials configured"
       };
     }
     if (!_isModelValid(_judgeModel)) {
       return {
-        assertion,
+        assertion: `llm_judge: "${criteria}"`,
         pass: false,
         expected: criteria,
         actual: "FAILED \u2014 invalid LLM judge model"
       };
     }
-    const { output } = await _zai.check({ userMessage: context.userMessage, botResponse }, criteria).result();
+    const { output } = await _cognitive.generateContent({
+      model: _judgeModel,
+      temperature: 0,
+      responseFormat: "json_object",
+      systemPrompt: buildJudgeSystemPrompt(threshold),
+      messages: [
+        {
+          role: "user",
+          content: `User message: ${context.userMessage}
+
+Bot response: ${botResponse}
+
+Criteria: ${criteria}`
+        }
+      ]
+    });
+    const rawContent = output?.choices?.[0]?.content;
+    const content = typeof rawContent === "string" ? rawContent : null;
+    if (!content) {
+      _log.warn(`LLM judge returned an empty response for "${criteria}"`);
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: false,
+        expected: criteria,
+        actual: "FAILED \u2014 LLM judge returned empty response"
+      };
+    }
+    const verdict = JSON.parse(content);
     return {
-      assertion,
-      pass: output.value,
+      assertion: `llm_judge: "${criteria}"`,
+      pass: verdict.score >= threshold,
       expected: criteria,
-      actual: `${output.value ? "Pass" : "Fail"} \u2014 ${output.explanation}`
+      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
     };
   } catch (err) {
     _log.warn(`LLM judge failed for "${criteria}": ${err.message}`);
     return {
-      assertion,
-      pass: !_failClosed,
-      skipped: true,
+      assertion: `llm_judge: "${criteria}"`,
+      pass: false,
       expected: criteria,
       actual: `FAILED \u2014 LLM judge unavailable: ${err.message}`
     };
@@ -1272,7 +1348,6 @@ function filterEvals(evals, filter) {
 }
 
 // src/runner.ts
-import { isAdkError } from "@botpress/analytics";
 import { randomUUID } from "crypto";
 var DEFAULT_IDLE_TIMEOUT = 3e4;
 var DEFAULT_DEV_SERVER_URL = "http://localhost:3001";
@@ -1359,22 +1434,27 @@ async function runEval(evalDef, connection, options = {}) {
   const start = Date.now();
   const turns = [];
   let outcomeAssertions = [];
-  const collector = options.spanSource ?? new LocalSpanSource(devServerUrl, devServerHeaders);
+  const collector = new SSECollector(devServerUrl, devServerHeaders);
   try {
-    const session = new ChatSession(
-      connection.client,
-      connection.botId,
-      options.chatWebhookId,
-      options.chatBaseUrl,
-      options.chatClient
-    );
+    const session = new ChatSession(connection.webhookId, options.chatClient);
     await session.connect();
+    let bpClient = null;
+    const getBpClient = () => {
+      if (!bpClient) {
+        bpClient = new BpClient2({
+          token: connection.token,
+          botId: connection.botId,
+          apiUrl: connection.apiUrl
+        });
+      }
+      return bpClient;
+    };
     let conversationId = "";
     if (evalDef.setup?.state?.conversation || evalDef.setup?.workflow) {
       conversationId = await session.ensureConversation();
     }
     await seedEvalState(
-      connection.client,
+      getBpClient(),
       {
         botId: connection.botId,
         userId: session.userId,
@@ -1383,7 +1463,7 @@ async function runEval(evalDef, connection, options = {}) {
       evalDef.setup
     );
     await triggerEvalWorkflow(
-      connection.client,
+      getBpClient(),
       {
         userId: session.userId,
         ...conversationId ? { conversationId } : {}
@@ -1419,18 +1499,18 @@ async function runEval(evalDef, connection, options = {}) {
       if (!turn.user && !turn.event) {
         throw turnConfigError(`must have either 'user' or 'event'.`);
       }
-      const turnLabel = turn.event ? "[event]" : turn.user;
-      await options.onProgress?.({
+      const turnLabelForEvent = turn.event ? `[event: ${turn.event.type}]` : turn.user;
+      options.onProgress?.({
         type: "turn_start",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnIndex: i,
+        turnNumber: i + 1,
         totalTurns: evalDef.conversation.length,
-        userMessage: turnLabel
+        userMessage: turnLabelForEvent
       });
       collector.startTurn();
       if (turn.event) {
-        await session.sendEvent(turn.event.payload);
+        await session.sendEvent(turn.event.type, turn.event.payload ?? {});
       } else {
         await session.sendMessage(turn.user);
       }
@@ -1460,6 +1540,7 @@ async function runEval(evalDef, connection, options = {}) {
       const turnData = transformSpans(turnSpans);
       const botResponse = turnData.messages.join("\n");
       const botDuration = turnData.handlerDuration;
+      const turnLabel = turn.event ? `[event: ${turn.event.type}]` : turn.user;
       const evalStart = Date.now();
       let assertions = [];
       const noResponse = !turn.expectSilence && turnData.messages.length === 0;
@@ -1514,7 +1595,7 @@ async function runEval(evalDef, connection, options = {}) {
       const turnPass = assertions.every((a) => a.pass);
       const evalDuration = Date.now() - evalStart;
       const turnReport = {
-        turnIndex: i,
+        turnNumber: i + 1,
         userMessage: turnLabel,
         botResponse,
         assertions,
@@ -1523,11 +1604,11 @@ async function runEval(evalDef, connection, options = {}) {
         evalDuration
       };
       turns.push(turnReport);
-      await options.onProgress?.({
+      options.onProgress?.({
         type: "turn_complete",
         evalName: evalDef.name,
         evalIndex: options.evalIndex ?? 0,
-        turnIndex: i,
+        turnNumber: i + 1,
         totalTurns: evalDef.conversation.length,
         turnReport
       });
@@ -1574,15 +1655,7 @@ async function runEval(evalDef, connection, options = {}) {
   } catch (err) {
     const adkErr = isAdkError(err) ? err : void 0;
     if (!adkErr || !adkErr.expected) {
-      const errMsg = err.message ?? String(err);
-      const errStack = err.stack ?? "";
-      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${errMsg}`);
-      logger.error(`  stack: ${errStack}`);
-      logger.error(`  eval: ${evalDef.name}, turn: ${turns.length}/${evalDef.conversation.length}`);
-      logger.error(`  connection: botId=${connection.botId}`);
-      logger.error(
-        `  spanSource: ${options.spanSource?.constructor?.name ?? "default"}, devServerUrl: ${options.devServerUrl ?? "n/a"}`
-      );
+      logger.error(`Eval "${evalDef.name}" failed unexpectedly: ${err.message}`);
       options.onException?.(err, { source: "evals", eval_name: evalDef.name });
     }
     return {
@@ -1600,15 +1673,9 @@ async function runEval(evalDef, connection, options = {}) {
 }
 async function runEvalSuite(config, filter) {
   const start = Date.now();
-  const runId = config.runId ?? randomUUID().replace(/-/g, "").slice(0, 26);
-  let allEvals;
-  if (config.definitions) {
-    allEvals = config.definitions;
-  } else if (config.agentPath) {
-    allEvals = await loadEvalsFromDir(`${config.agentPath}/evals`);
-  } else {
-    throw new Error("Either `definitions` or `agentPath` must be provided in EvalRunnerConfig");
-  }
+  const runId = randomUUID().replace(/-/g, "").slice(0, 26);
+  const evalsDir = `${config.agentPath}/evals`;
+  const allEvals = await loadEvalsFromDir(evalsDir);
   const evals = filterEvals(allEvals, filter);
   if (evals.length === 0) {
     return {
@@ -1622,26 +1689,38 @@ async function runEvalSuite(config, filter) {
       ...filter !== void 0 ? { filter } : {}
     };
   }
-  await initLLMJudge(config.client, {
-    ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
-    ...config.logger !== void 0 ? { logger: config.logger } : {}
-  });
+  await initLLMJudge(
+    {
+      token: config.credentials.token,
+      apiUrl: config.credentials.apiUrl,
+      botId: config.credentials.botId,
+      ...config.credentials.workspaceId !== void 0 ? { workspaceId: config.credentials.workspaceId } : {}
+    },
+    {
+      ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
+      ...config.logger !== void 0 ? { logger: config.logger } : {}
+    }
+  );
+  let webhookId = config.credentials.webhookId;
+  if (!webhookId) {
+    webhookId = await discoverWebhookId(config.credentials.botId, config.credentials.token, config.credentials.apiUrl);
+  }
   const connection = {
-    client: config.client,
-    botId: config.botId
+    webhookId,
+    botId: config.credentials.botId,
+    token: config.credentials.token,
+    apiUrl: config.credentials.apiUrl
   };
   const devServerUrl = config.devServerUrl || DEFAULT_DEV_SERVER_URL;
   const devServerHeaders = config.devServerHeaders ?? {};
-  if (!config.createSpanSource) {
-    await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
-    await assertTraceStreamReadable(devServerUrl, devServerHeaders);
-  }
+  await assertChatChannelBound(devServerUrl, devServerHeaders, config.logger);
+  await assertTraceStreamReadable(devServerUrl, devServerHeaders);
   const reports = [];
-  await config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
+  config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
   for (let i = 0; i < evals.length; i++) {
     if (config.signal?.aborted) break;
     const evalDef = evals[i];
-    await config.onProgress?.({
+    config.onProgress?.({
       type: "eval_start",
       evalName: evalDef.name,
       index: i,
@@ -1657,13 +1736,10 @@ async function runEvalSuite(config, filter) {
       ...config.signal !== void 0 ? { signal: config.signal } : {},
       ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {},
       ...config.logger !== void 0 ? { logger: config.logger } : {},
-      ...config.onException !== void 0 ? { onException: config.onException } : {},
-      ...config.createSpanSource !== void 0 ? { spanSource: config.createSpanSource() } : {},
-      ...config.chatWebhookId !== void 0 ? { chatWebhookId: config.chatWebhookId } : {},
-      ...config.chatBaseUrl !== void 0 ? { chatBaseUrl: config.chatBaseUrl } : {}
+      ...config.onException !== void 0 ? { onException: config.onException } : {}
     });
     reports.push(report);
-    await config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
+    config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
   }
   const runReport = {
     id: runId,
@@ -1678,7 +1754,7 @@ async function runEvalSuite(config, filter) {
   if (config.signal?.aborted) {
     runReport.aborted = true;
   }
-  await config.onProgress?.({ type: "suite_complete", report: runReport });
+  config.onProgress?.({ type: "suite_complete", report: runReport });
   return runReport;
 }
 export {