@botpress/evals 1.18.3

package/dist/runner.js

@@ -0,0 +1,1536 @@
+// src/runner.ts
+import { Client as BpClient2 } from "@botpress/client";
+
+// src/client.ts
+import { Client as BpClient } from "@botpress/client";
+import { createRequire } from "module";
+var defaultChatClient = null;
+function getDefaultChatClient() {
+  if (defaultChatClient) return defaultChatClient;
+  const require2 = createRequire(import.meta.url);
+  const mod = require2("@botpress/chat");
+  const ChatClient = mod.default?.Client ?? mod.Client;
+  if (!ChatClient) {
+    throw new Error("@botpress/chat did not expose a Client export. Pass EvalRunnerConfig.chatClient explicitly.");
+  }
+  return ChatClient;
+}
+var ChatSession = class {
+  constructor(webhookId, chatClient) {
+    this.webhookId = webhookId;
+    this.chatClient = chatClient;
+  }
+  client = null;
+  conversationId = null;
+  async connect() {
+    const ChatClient = this.chatClient ?? getDefaultChatClient();
+    this.client = await ChatClient.connect({ webhookId: this.webhookId });
+  }
+  get userId() {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    return this.client.user.id;
+  }
+  async ensureConversation() {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    if (!this.conversationId) {
+      const conv = await this.client.createConversation({});
+      this.conversationId = conv.conversation.id;
+      return conv.conversation.id;
+    }
+    return this.conversationId;
+  }
+  /**
+   * Send a user message. Fire-and-forget — observation is via SSECollector.
+   */
+  async sendMessage(message) {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    const conversationId = await this.ensureConversation();
+    await this.client.createMessage({ conversationId, payload: { type: "text", text: message } });
+  }
+  /**
+   * Send an event. Fire-and-forget — observation is via SSECollector.
+   */
+  async sendEvent(type, payload) {
+    if (!this.client) {
+      throw new Error("ChatSession not connected. Call connect() first.");
+    }
+    const conversationId = await this.ensureConversation();
+    await this.client.createEvent({ type, payload, conversationId });
+  }
+};
+async function discoverWebhookId(botId, token, apiUrl) {
+  const client = new BpClient({ token, botId, apiUrl });
+  const { bot } = await client.getBot({ id: botId });
+  const integrations = bot.integrations || {};
+  const chat = Object.values(integrations).find((int) => int.name === "chat");
+  const webhookId = chat?.webhookId;
+  if (!webhookId || typeof webhookId !== "string") {
+    throw new Error(
+      [
+        "The `chat` integration is not installed on this bot \u2014 evals require it.",
+        "",
+        "To fix:",
+        "  1. Install it:    adk add chat",
+        "  2. Redeploy:      adk dev   (or: adk deploy)"
+      ].join("\n")
+    );
+  }
+  return webhookId;
+}
+async function assertChatChannelBound(devServerUrl, headers = {}) {
+  let conversations = [];
+  try {
+    const res = await fetch(`${devServerUrl}/api/agent`, { headers });
+    if (!res.ok) return;
+    const agent = await res.json();
+    conversations = agent.conversations ?? [];
+  } catch {
+    return;
+  }
+  if (conversations.length === 0) return;
+  const matchesChatChannel = (ch) => {
+    if (!ch) return false;
+    if (ch === "*" || ch === "chat.channel") return true;
+    return Array.isArray(ch) && (ch.includes("chat.channel") || ch.includes("*"));
+  };
+  const bound = conversations.some((c) => matchesChatChannel(c.channel));
+  if (bound) return;
+  throw new Error(
+    [
+      "No Conversation is bound to `chat.channel` \u2014 messages via `@botpress/chat` will time out silently.",
+      "",
+      "To fix, bind a Conversation in `src/conversations/` to `chat.channel` (or `*`):",
+      "  channel: 'chat.channel',           // explicit",
+      "  channel: '*',                      // wildcard \u2014 matches every channel",
+      "  channel: ['chat.channel', ...],    // array form",
+      "",
+      "The `chat` integration is how evals and `adk chat --single` send messages programmatically."
+    ].join("\n")
+  );
+}
+
+// src/sse-collector.ts
+function spanKey(span) {
+  return `${span.id.trace}:${span.id.span}`;
+}
+function spanData(span) {
+  return span.data && typeof span.data === "object" ? span.data : {};
+}
+var SSECollector = class _SSECollector {
+  devServerUrl;
+  headers;
+  /** All spans accumulated since connect(), keyed by spanKey for dedup. */
+  spanMap = /* @__PURE__ */ new Map();
+  /** Index into spanMap insertion order marking where the current turn started. */
+  turnStartKeys = /* @__PURE__ */ new Set();
+  /** Handler span IDs seen before the current turn (for startTurn filtering). */
+  seenHandlerKeys = /* @__PURE__ */ new Set();
+  /** Callbacks waiting for span updates. */
+  waiters = [];
+  /** AbortController for the active SSE connection. */
+  abortController = null;
+  /** Whether the background loop is running. */
+  connected = false;
+  /** Error from the SSE stream if it died unexpectedly. */
+  streamError = null;
+  /** Set while a settle phase is in flight; called by disconnect() to short-circuit. */
+  _cancelSettle = null;
+  constructor(devServerUrl, headers = {}) {
+    this.devServerUrl = devServerUrl;
+    this.headers = headers;
+  }
+  /**
+   * Open a streaming SSE connection filtered by conversationId or workflowId.
+   * Starts a background loop that parses SSE events and accumulates spans.
+   */
+  async connect(filter) {
+    const attributeName = "conversationId" in filter ? "conversationId" : "workflowId";
+    const attributeValue = "conversationId" in filter ? filter.conversationId : filter.workflowId;
+    const url = `${this.devServerUrl}/api/traces/stream?attributeName=${encodeURIComponent(attributeName)}&attributeValue=${encodeURIComponent(attributeValue)}`;
+    this.abortController = new AbortController();
+    const response = await fetch(url, {
+      headers: { ...this.headers, Accept: "text/event-stream" },
+      signal: this.abortController.signal
+    });
+    if (!response.ok) {
+      throw new Error(`SSE connection failed: ${response.status} ${response.statusText}`);
+    }
+    if (!response.body) {
+      throw new Error("SSE response has no body");
+    }
+    this.connected = true;
+    this._consumeStream(response.body).catch((err) => {
+      if (this.connected) {
+        this.streamError = err instanceof Error ? err : new Error(String(err));
+      }
+    });
+    await this._waitForSnapshot();
+  }
+  /**
+   * Mark the beginning of a new turn.
+   * Records which handler.conversation span IDs have been seen so far,
+   * so waitForTurnComplete() only resolves on NEW handler completions.
+   */
+  startTurn() {
+    this.turnStartKeys = new Set(this.spanMap.keys());
+    for (const [key, span] of this.spanMap) {
+      if (span.name === "handler.conversation") {
+        this.seenHandlerKeys.add(key);
+      }
+    }
+  }
+  static SETTLE_MS = 100;
+  /**
+   * Wait for a handler.conversation span to reach terminal status (ok or error).
+   * Only considers spans NOT seen before the last startTurn() call.
+   *
+   * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)
+   * to ensure all child spans (tool calls, messages, state saves) have arrived.
+   * The runtime's HttpSpanExporter fires POSTs without awaiting them, so child
+   * spans may arrive slightly after the handler close event.
+   */
+  async waitForTurnComplete(opts) {
+    if (this._hasNewCompletedHandler()) {
+      return this._settleAfterHandler();
+    }
+    if (opts.abortSignal?.aborted) {
+      throw new Error("waitForTurnComplete aborted");
+    }
+    return new Promise((resolve2, reject) => {
+      const timer = setTimeout(() => {
+        this._removeWaiter(waiter);
+        opts.abortSignal?.removeEventListener("abort", onAbort);
+        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "no handler.conversation span completed";
+        reject(new Error(`waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}`));
+      }, opts.timeout);
+      const onAbort = () => {
+        clearTimeout(timer);
+        this._removeWaiter(waiter);
+        reject(new Error("waitForTurnComplete aborted"));
+      };
+      opts.abortSignal?.addEventListener("abort", onAbort, { once: true });
+      const waiter = (_span) => {
+        if (this._hasNewCompletedHandler()) {
+          clearTimeout(timer);
+          this._removeWaiter(waiter);
+          opts.abortSignal?.removeEventListener("abort", onAbort);
+          this._settleAfterHandler().then(resolve2, reject);
+        }
+      };
+      this.waiters.push(waiter);
+    });
+  }
+  static MAX_SETTLE_MS = 2e3;
+  /**
+   * After the handler span closes, wait for spans to stop arriving.
+   * Resolves when no new spans arrive for SETTLE_MS, or after MAX_SETTLE_MS hard cap.
+   *
+   * Exposes a cancellation hook via _cancelSettle so disconnect() can short-
+   * circuit an in-flight settle phase and avoid leaking timers for up to 2s.
+   */
+  _settleAfterHandler() {
+    return new Promise((resolve2) => {
+      let resolved = false;
+      const done = () => {
+        if (resolved) return;
+        resolved = true;
+        clearTimeout(hardCap);
+        clearTimeout(settleTimer);
+        this._removeWaiter(settleWaiter);
+        this._cancelSettle = null;
+        resolve2();
+      };
+      const hardCap = setTimeout(done, _SSECollector.MAX_SETTLE_MS);
+      let settleTimer = setTimeout(done, _SSECollector.SETTLE_MS);
+      const settleWaiter = () => {
+        if (resolved) return;
+        clearTimeout(settleTimer);
+        settleTimer = setTimeout(done, _SSECollector.SETTLE_MS);
+      };
+      this.waiters.push(settleWaiter);
+      this._cancelSettle = done;
+    });
+  }
+  /**
+   * Wait for a handler.workflow span with matching workflow name.
+   * - signal 'entered': resolves when the span appears (any status)
+   * - signal 'completed': resolves when the span has status !== 'running'
+   */
+  async waitForWorkflow(name, opts) {
+    if (this._matchesWorkflowCondition(name, opts.signal)) {
+      return;
+    }
+    if (opts.abortSignal?.aborted) {
+      throw new Error(`waitForWorkflow("${name}", { signal: "${opts.signal}" }) aborted`);
+    }
+    return new Promise((resolve2, reject) => {
+      const timer = setTimeout(() => {
+        this._removeWaiter(waiter);
+        opts.abortSignal?.removeEventListener("abort", onAbort);
+        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "span not found";
+        reject(
+          new Error(
+            `waitForWorkflow("${name}", { signal: "${opts.signal}" }) timed out after ${opts.timeout}ms \u2014 ${reason}`
+          )
+        );
+      }, opts.timeout);
+      const onAbort = () => {
+        clearTimeout(timer);
+        this._removeWaiter(waiter);
+        reject(new Error(`waitForWorkflow("${name}", { signal: "${opts.signal}" }) aborted`));
+      };
+      opts.abortSignal?.addEventListener("abort", onAbort, { once: true });
+      const waiter = (_span) => {
+        if (this._matchesWorkflowCondition(name, opts.signal)) {
+          clearTimeout(timer);
+          this._removeWaiter(waiter);
+          opts.abortSignal?.removeEventListener("abort", onAbort);
+          resolve2();
+        }
+      };
+      this.waiters.push(waiter);
+    });
+  }
+  /**
+   * Return spans accumulated since the last startTurn() call.
+   */
+  getTurnSpans() {
+    const result = [];
+    for (const [key, span] of this.spanMap) {
+      if (!this.turnStartKeys.has(key)) {
+        result.push(span);
+      }
+    }
+    return result;
+  }
+  /**
+   * Return all spans accumulated since connect().
+   */
+  getAllSpans() {
+    return Array.from(this.spanMap.values());
+  }
+  /**
+   * Close the SSE connection and clean up.
+   */
+  disconnect() {
+    this.connected = false;
+    this.abortController?.abort();
+    this.abortController = null;
+    this._cancelSettle?.();
+    this._cancelSettle = null;
+    this.waiters = [];
+    this.snapshotResolvers = [];
+  }
+  // ── Private ──────────────────────────────────────────────────
+  /** Whether we have received the initial snapshot. */
+  snapshotReceived = false;
+  snapshotResolvers = [];
+  static SNAPSHOT_TIMEOUT_MS = 1e4;
+  /**
+   * Returns a promise that resolves once the snapshot event has been processed,
+   * or rejects after a timeout to prevent connect() from hanging indefinitely.
+   */
+  _waitForSnapshot() {
+    if (this.snapshotReceived) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve2, reject) => {
+      let settled = false;
+      const resolver = () => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        resolve2();
+      };
+      const timer = setTimeout(() => {
+        if (settled) return;
+        settled = true;
+        const idx = this.snapshotResolvers.indexOf(resolver);
+        if (idx !== -1) this.snapshotResolvers.splice(idx, 1);
+        reject(
+          new Error(
+            `SSE snapshot not received within ${_SSECollector.SNAPSHOT_TIMEOUT_MS}ms \u2014 is the dev server healthy?`
+          )
+        );
+      }, _SSECollector.SNAPSHOT_TIMEOUT_MS);
+      this.snapshotResolvers.push(resolver);
+    });
+  }
+  /**
+   * Consume the ReadableStream body, parse SSE lines, and dispatch events.
+   */
+  async _consumeStream(body) {
+    const decoder = new TextDecoder();
+    let buffer = "";
+    const reader = body.getReader();
+    try {
+      while (this.connected) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buffer += decoder.decode(value, { stream: true });
+        const parts = buffer.split("\n\n");
+        buffer = parts.pop() ?? "";
+        for (const part of parts) {
+          this._parseSSEBlock(part);
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+  /**
+   * Parse a single SSE event block (text between \n\n boundaries).
+   * Extracts event name and data payload.
+   */
+  _parseSSEBlock(block) {
+    const lines = block.split("\n");
+    let eventName = "";
+    let dataLine = "";
+    for (const line of lines) {
+      if (line.startsWith("event:")) {
+        eventName = line.slice("event:".length).trim();
+      } else if (line.startsWith("data:")) {
+        const value = line.slice("data:".length).trim();
+        dataLine = dataLine ? `${dataLine}
+${value}` : value;
+      }
+    }
+    if (!eventName || !dataLine) {
+      if (!this.snapshotReceived && block.includes("retry:")) {
+      }
+      return;
+    }
+    try {
+      const payload = JSON.parse(dataLine);
+      this._handleEvent(eventName, payload);
+    } catch {
+    }
+  }
+  /**
+   * Dispatch a parsed SSE event to the appropriate handler.
+   */
+  _handleEvent(eventName, payload) {
+    if (eventName === "snapshot") {
+      const data2 = payload;
+      const spans = Array.isArray(data2.spans) ? data2.spans : [];
+      for (const span of spans) {
+        this._upsertSpan(span);
+      }
+      this.snapshotReceived = true;
+      for (const resolve2 of this.snapshotResolvers) {
+        resolve2();
+      }
+      this.snapshotResolvers = [];
+    } else if (eventName === "update") {
+      const data2 = payload;
+      if (data2.span) {
+        this._upsertSpan(data2.span);
+        this._notifyWaiters(data2.span);
+      }
+    }
+  }
+  /**
+   * Insert or replace a span in the map (dedup by trace+span ID).
+   */
+  _upsertSpan(span) {
+    this.spanMap.set(spanKey(span), span);
+  }
+  /**
+   * Notify all waiters that a new span was received.
+   */
+  _notifyWaiters(span) {
+    const current = [...this.waiters];
+    for (const waiter of current) {
+      waiter(span);
+    }
+  }
+  /**
+   * Remove a waiter callback from the list.
+   */
+  _removeWaiter(fn) {
+    const idx = this.waiters.indexOf(fn);
+    if (idx !== -1) {
+      this.waiters.splice(idx, 1);
+    }
+  }
+  /**
+   * Check if there is a handler.conversation span that:
+   * 1. Was NOT seen before startTurn()
+   * 2. Has terminal status (ok or error)
+   *
+   * Marks the matched key as consumed so the same handler span cannot
+   * satisfy both the current turn's settle phase AND the next turn's
+   * waitForTurnComplete check (which would otherwise resolve instantly
+   * against stale data instead of waiting for the new turn's handler).
+   */
+  _hasNewCompletedHandler() {
+    for (const [key, span] of this.spanMap) {
+      if (span.name === "handler.conversation" && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
+        this.seenHandlerKeys.add(key);
+        return true;
+      }
+    }
+    return false;
+  }
+  /**
+   * Check if a handler.workflow span matches the given condition.
+   */
+  _matchesWorkflowCondition(name, signal) {
+    for (const span of this.spanMap.values()) {
+      if (span.name !== "handler.workflow") continue;
+      const d = spanData(span);
+      const wfName = d["workflow.name"] || d["workflowName"];
+      if (wfName !== name) continue;
+      if (signal === "entered") {
+        return true;
+      }
+      if (signal === "completed") {
+        return span.status === "ok" || span.status === "error";
+      }
+    }
+    return false;
+  }
+};
+
+// src/transformer.ts
+function data(span) {
+  return span.data && typeof span.data === "object" ? span.data : {};
+}
+function tryParseJson(value) {
+  if (value == null) return void 0;
+  if (typeof value !== "string") return value;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return value;
+  }
+}
+function parseJsonOrFallback(value, fallback) {
+  if (typeof value !== "string") return fallback;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return fallback;
+  }
+}
+function transformSpans(spans) {
+  return {
+    messages: extractMessages(spans),
+    toolCalls: extractToolCalls(spans),
+    stateMutations: extractStateMutations(spans),
+    workflowSpans: extractWorkflowSpans(spans),
+    ...extractHandler(spans)
+  };
+}
+function extractMessages(spans) {
+  return spans.filter((s) => s.name === "chat.sendMessage" && data(s)["message.preview"]).sort((a, b) => a.timing.startedAt - b.timing.startedAt).map((s) => String(data(s)["message.preview"]));
+}
+function extractToolCalls(spans) {
+  const toolSpans = spans.filter(
+    (s) => s.name === "autonomous.tool" && s.status !== "running" && data(s)["autonomous.tool.name"]
+  );
+  const seen = /* @__PURE__ */ new Set();
+  const unique = toolSpans.filter((s) => {
+    if (seen.has(s.id.span)) return false;
+    seen.add(s.id.span);
+    return true;
+  });
+  return unique.sort((a, b) => (a.timing.endedAt ?? Infinity) - (b.timing.endedAt ?? Infinity)).map((s) => {
+    const d = data(s);
+    const input = parseJsonOrFallback(d["autonomous.tool.input"], {});
+    return {
+      name: String(d["autonomous.tool.name"]),
+      input,
+      output: String(d["autonomous.tool.output"] ?? ""),
+      status: String(d["autonomous.tool.status"] ?? "unknown")
+    };
+  });
+}
+function extractStateMutations(spans) {
+  return spans.filter((s) => {
+    if (s.name !== "state.save") return false;
+    const d = data(s);
+    const changedKeys = d["state.changed_keys"];
+    return Array.isArray(changedKeys) && changedKeys.length > 0;
+  }).map((s) => {
+    const d = data(s);
+    return {
+      type: String(d["type"] ?? "unknown"),
+      changedKeys: d["state.changed_keys"] ?? [],
+      previous: tryParseJson(d["state.previous_value"]),
+      current: tryParseJson(d["state.value"]),
+      swappedToFile: d["swapped_to_file"] === true
+    };
+  });
+}
+function extractWorkflowSpans(spans) {
+  return spans.filter((s) => s.name === "handler.workflow").map((s) => {
+    const d = data(s);
+    const result = {
+      name: String(d["workflow.name"] || d["workflowName"] || ""),
+      status: s.status
+    };
+    const statusFinal = d["workflow.status.final"];
+    if (statusFinal) {
+      result.statusFinal = String(statusFinal);
+    }
+    return result;
+  });
+}
+function extractHandler(spans) {
+  const handler = spans.find(
+    (s) => (s.name === "handler.conversation" || s.name === "handler.workflow") && s.status !== "running"
+  );
+  if (!handler) {
+    return { handlerDuration: 0, handlerStatus: "ok" };
+  }
+  return {
+    handlerDuration: handler.timing.duration ?? 0,
+    handlerStatus: handler.status === "error" ? "error" : "ok"
+  };
+}
+
+// src/graders/llm.ts
+import { Client } from "@botpress/client";
+import { Cognitive } from "@botpress/cognitive";
+var DEFAULT_PASS_THRESHOLD = 3;
+function buildJudgeSystemPrompt(passThreshold) {
+  return `You are an evaluation judge for a chatbot. You will be given:
+- The user's message
+- The bot's response
+- Grading criteria
+
+Your job is to determine if the bot's response meets the criteria.
+
+Respond with a JSON object:
+{
+  "pass": true/false,
+  "score": 1-5,
+  "reason": "brief explanation"
+}
+
+Scoring guide:
+- 5: Fully meets criteria, excellent response
+- 4: Mostly meets criteria, minor issues
+- 3: Partially meets criteria, notable gaps
+- 2: Barely meets criteria, significant issues
+- 1: Does not meet criteria
+
+A score of ${passThreshold} or above is a pass.`;
+}
+var MODEL_ALIASES = ["fast", "best"];
+var _cognitive = null;
+var _availableModels = [];
+var _judgeModel = "fast";
+function _isModelValid(model) {
+  if (MODEL_ALIASES.includes(model)) return true;
+  if (_availableModels.length === 0) return false;
+  return _availableModels.includes(model);
+}
+async function initLLMJudge(credentials, options) {
+  const log = options?.logger ?? console;
+  const client = new Client({
+    token: credentials.token,
+    apiUrl: credentials.apiUrl,
+    botId: credentials.botId,
+    ...credentials.workspaceId !== void 0 ? { workspaceId: credentials.workspaceId } : {}
+  });
+  _cognitive = new Cognitive({ client, __experimental_beta: true });
+  try {
+    _availableModels = Array.from((await _cognitive.fetchRemoteModels()).keys());
+  } catch (err) {
+    log.warn(`Failed to fetch available models: ${err.message}. LLM judge will be unavailable.`);
+    _availableModels = [];
+  }
+  _judgeModel = options?.model ?? "fast";
+  if (_availableModels.length === 0) {
+    log.warn("No available Cognitive models: cannot validate judge model.");
+  } else if (!_isModelValid(_judgeModel)) {
+    log.warn(`Configured LLM judge model "${_judgeModel}" is invalid. Run "adk models" to list available models.`);
+  }
+}
+async function gradeLLMJudge(botResponse, criteria, context) {
+  const threshold = Math.max(1, Math.min(5, context.passThreshold ?? DEFAULT_PASS_THRESHOLD));
+  try {
+    if (!_cognitive) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: true,
+        expected: criteria,
+        actual: "SKIPPED \u2014 LLM judge unavailable: no credentials configured"
+      };
+    }
+    if (!_isModelValid(_judgeModel)) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: false,
+        expected: criteria,
+        actual: "FAILED \u2014 invalid LLM judge model"
+      };
+    }
+    const { output } = await _cognitive.generateContent({
+      model: _judgeModel,
+      temperature: 0,
+      responseFormat: "json_object",
+      systemPrompt: buildJudgeSystemPrompt(threshold),
+      messages: [
+        {
+          role: "user",
+          content: `User message: ${context.userMessage}
+
+Bot response: ${botResponse}
+
+Criteria: ${criteria}`
+        }
+      ]
+    });
+    const rawContent = output?.choices?.[0]?.content;
+    const content = typeof rawContent === "string" ? rawContent : null;
+    if (!content) {
+      return {
+        assertion: `llm_judge: "${criteria}"`,
+        pass: true,
+        expected: criteria,
+        actual: "SKIPPED \u2014 LLM judge returned empty response"
+      };
+    }
+    const verdict = JSON.parse(content);
+    return {
+      assertion: `llm_judge: "${criteria}"`,
+      pass: verdict.score >= threshold,
+      expected: criteria,
+      actual: `Score ${verdict.score}/5 (threshold: ${threshold}) \u2014 ${verdict.reason}`
+    };
+  } catch (err) {
+    return {
+      assertion: `llm_judge: "${criteria}"`,
+      pass: true,
+      expected: criteria,
+      actual: `SKIPPED \u2014 LLM judge unavailable: ${err.message}`
+    };
+  }
+}
+
+// src/graders/response.ts
+async function gradeResponse(botResponse, assertions, context) {
+  const results = [];
+  for (const assertion of assertions) {
+    if ("contains" in assertion) {
+      const pass = botResponse.toLowerCase().includes(assertion.contains.toLowerCase());
+      results.push({
+        assertion: `contains "${assertion.contains}"`,
+        pass,
+        expected: `Response contains "${assertion.contains}"`,
+        actual: pass ? `Found in response` : `Not found in response`
+      });
+      continue;
+    }
+    if ("not_contains" in assertion) {
+      const pass = !botResponse.toLowerCase().includes(assertion.not_contains.toLowerCase());
+      results.push({
+        assertion: `not_contains "${assertion.not_contains}"`,
+        pass,
+        expected: `Response does not contain "${assertion.not_contains}"`,
+        actual: pass ? `Not found in response` : `Found in response`
+      });
+      continue;
+    }
+    if ("matches" in assertion) {
+      const regex = new RegExp(assertion.matches, "i");
+      const pass = regex.test(botResponse);
+      results.push({
+        assertion: `matches ${assertion.matches}`,
+        pass,
+        expected: `Response matches /${assertion.matches}/`,
+        actual: pass ? `Matched` : `No match`
+      });
+      continue;
+    }
+    if ("llm_judge" in assertion) {
+      const result = await gradeLLMJudge(botResponse, assertion.llm_judge, {
+        userMessage: context.userMessage,
+        ...context.judgePassThreshold !== void 0 ? { passThreshold: context.judgePassThreshold } : {}
+      });
+      results.push(result);
+      continue;
+    }
+    results.push({
+      assertion: "unknown",
+      pass: false,
+      expected: "known assertion type",
+      actual: `Unknown assertion: ${JSON.stringify(assertion)}`
+    });
+  }
+  return results;
+}
+
+// src/graders/match.ts
+function matchValue(operator, actual) {
+  if (typeof operator === "string") {
+    return String(actual) === operator;
+  }
+  if ("equals" in operator) {
+    return actual === operator.equals;
+  }
+  if ("contains" in operator) {
+    return String(actual).toLowerCase().includes(operator.contains.toLowerCase());
+  }
+  if ("not_contains" in operator) {
+    return !String(actual).toLowerCase().includes(operator.not_contains.toLowerCase());
+  }
+  if ("matches" in operator) {
+    return new RegExp(operator.matches, "i").test(String(actual));
+  }
+  if ("in" in operator) {
+    return operator.in.includes(actual);
+  }
+  if ("exists" in operator) {
+    return operator.exists ? actual !== void 0 && actual !== null : actual === void 0 || actual === null;
+  }
+  if ("gte" in operator) {
+    return Number(actual) >= operator.gte;
+  }
+  if ("lte" in operator) {
+    return Number(actual) <= operator.lte;
+  }
+  return false;
+}
+function operatorToString(operator) {
+  if (typeof operator === "string") return `equals "${operator}"`;
+  if ("equals" in operator) return `equals ${JSON.stringify(operator.equals)}`;
+  if ("contains" in operator) return `contains "${operator.contains}"`;
+  if ("not_contains" in operator) return `not_contains "${operator.not_contains}"`;
+  if ("matches" in operator) return `matches /${operator.matches}/`;
+  if ("in" in operator) return `in [${operator.in.map((v) => JSON.stringify(v)).join(", ")}]`;
+  if ("exists" in operator) return operator.exists ? "exists" : "does not exist";
+  if ("gte" in operator) return `>= ${operator.gte}`;
+  if ("lte" in operator) return `<= ${operator.lte}`;
+  return JSON.stringify(operator);
+}
+
+// src/graders/tools.ts
+function gradeTools(toolCalls, assertions) {
+  return assertions.map((assertion) => {
+    if ("called" in assertion && !("not_called" in assertion) && !("call_order" in assertion)) {
+      const matches = toolCalls.filter((tc) => tc.name === assertion.called);
+      const wasCalled = matches.length > 0;
+      if (!wasCalled) {
+        return {
+          assertion: `tool called: ${assertion.called}`,
+          pass: false,
+          expected: `${assertion.called} was called`,
+          actual: `Not called. Tools called: [${toolCalls.map((tc) => tc.name).join(", ") || "none"}]`
+        };
+      }
+      if (assertion.params) {
+        const paramResults = [];
+        for (const [key, operator] of Object.entries(assertion.params)) {
+          const anyMatch = matches.some((tc) => matchValue(operator, tc.input[key]));
+          paramResults.push({
+            key,
+            pass: anyMatch,
+            detail: anyMatch ? `matched` : `expected ${key} ${operatorToString(operator)}, got ${JSON.stringify(matches.map((tc) => tc.input[key]))}`
+          });
+        }
+        const allParamsPass = paramResults.every((p) => p.pass);
+        const failedParams = paramResults.filter((p) => !p.pass);
+        return {
+          assertion: `tool called: ${assertion.called} with params`,
+          pass: allParamsPass,
+          expected: `${assertion.called} called with ${Object.entries(assertion.params).map(([k, v]) => `${k} ${operatorToString(v)}`).join(", ")}`,
+          actual: allParamsPass ? `Matched` : failedParams.map((p) => p.detail).join("; ")
+        };
+      }
+      return {
+        assertion: `tool called: ${assertion.called}`,
+        pass: true,
+        expected: `${assertion.called} was called`,
+        actual: `Called ${matches.length} time(s)`
+      };
+    }
+    if ("not_called" in assertion) {
+      const wasCalled = toolCalls.some((tc) => tc.name === assertion.not_called);
+      return {
+        assertion: `tool not_called: ${assertion.not_called}`,
+        pass: !wasCalled,
+        expected: `${assertion.not_called} was NOT called`,
+        actual: wasCalled ? `Was called` : `Not called`
+      };
+    }
+    if ("call_order" in assertion) {
+      const calledNames = toolCalls.map((tc) => tc.name);
+      const expectedOrder = assertion.call_order;
+      let cursor = 0;
+      for (const name of calledNames) {
+        if (cursor < expectedOrder.length && name === expectedOrder[cursor]) {
+          cursor++;
+        }
+      }
+      const inOrder = cursor === expectedOrder.length;
+      return {
+        assertion: `call_order: [${expectedOrder.join(" \u2192 ")}]`,
+        pass: inOrder,
+        expected: `Tools called in order: [${expectedOrder.join(" \u2192 ")}]`,
+        actual: `Actual order: [${calledNames.join(" \u2192 ") || "none"}]`
+      };
+    }
+    return {
+      assertion: "unknown tool assertion",
+      pass: false,
+      expected: "known assertion type",
+      actual: `Unknown: ${JSON.stringify(assertion)}`
+    };
+  });
+}
+
+// src/graders/state.ts
+function parseStatePath(path) {
+  const dot = path.indexOf(".");
+  if (dot === -1) {
+    throw new Error(`Invalid state path "${path}" \u2014 expected "type.field" format`);
+  }
+  const type = path.slice(0, dot);
+  const field = path.slice(dot + 1);
+  if (type !== "bot" && type !== "user" && type !== "conversation") {
+    throw new Error(`Unknown state type "${type}" in path "${path}" \u2014 expected bot, user, or conversation`);
+  }
+  return { type, field };
+}
+function gradeState(mutations, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const parsed = parseStatePath(assertion.path);
+    const matchingMutations = mutations.filter((m) => m.type === parsed.type);
+    if (assertion.equals !== void 0) {
+      if (matchingMutations.length === 0) {
+        results.push({
+          assertion: `state: ${assertion.path} equals`,
+          pass: false,
+          expected: JSON.stringify(assertion.equals),
+          actual: `No state.save span found for "${parsed.type}" \u2014 the bot did not mutate this state during the turn. If this state was seeded via setup.state, assert on the tool/action that reads it, or use a conversation turn that triggers a state write.`
+        });
+        continue;
+      }
+      const lastMutation = matchingMutations[matchingMutations.length - 1];
+      if (lastMutation.swappedToFile) {
+        results.push({
+          assertion: `state: ${assertion.path} equals`,
+          pass: false,
+          expected: JSON.stringify(assertion.equals),
+          actual: "State was too large for trace-based assertion (swapped to file)"
+        });
+        continue;
+      }
+      const currentState = lastMutation.current;
+      const actualValue = currentState ? currentState[parsed.field] : void 0;
+      const pass = deepEqual(actualValue, assertion.equals);
+      results.push({
+        assertion: `state: ${assertion.path} equals`,
+        pass,
+        expected: JSON.stringify(assertion.equals),
+        actual: JSON.stringify(actualValue)
+      });
+    }
+    if (assertion.changed !== void 0) {
+      if (matchingMutations.length === 0) {
+        results.push({
+          assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+          pass: !assertion.changed,
+          expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+          actual: 'No state mutation found for type "' + parsed.type + '"'
+        });
+        continue;
+      }
+      const hasSwapped = matchingMutations.some((m) => m.swappedToFile);
+      if (hasSwapped) {
+        results.push({
+          assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+          pass: false,
+          expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+          actual: "State was too large for trace-based assertion (swapped to file)"
+        });
+        continue;
+      }
+      const fieldChanged = matchingMutations.some((m) => m.changedKeys.includes(parsed.field));
+      const pass = assertion.changed ? fieldChanged : !fieldChanged;
+      results.push({
+        assertion: `state: ${assertion.path} ${assertion.changed ? "changed" : "unchanged"}`,
+        pass,
+        expected: assertion.changed ? `Field "${parsed.field}" changed in a "${parsed.type}" state mutation` : `Field "${parsed.field}" unchanged`,
+        actual: fieldChanged ? `Field "${parsed.field}" was changed` : `Field "${parsed.field}" was not changed`
+      });
+    }
+  }
+  return results;
+}
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (a === null || b === null) return false;
+  if (typeof a !== typeof b) return false;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    return a.every((v, i) => deepEqual(v, b[i]));
+  }
+  if (Array.isArray(a) !== Array.isArray(b)) return false;
+  if (typeof a === "object" && typeof b === "object") {
+    const keysA = Object.keys(a);
+    const keysB = Object.keys(b);
+    if (keysA.length !== keysB.length) return false;
+    return keysA.every((k) => deepEqual(a[k], b[k]));
+  }
+  return false;
+}
+
+// src/graders/workflow.ts
+function gradeWorkflows(workflows, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const matching = workflows.filter((wf) => wf.name === assertion.name);
+    if (assertion.entered !== void 0) {
+      const wasEntered = matching.length > 0;
+      const pass = assertion.entered ? wasEntered : !wasEntered;
+      results.push({
+        assertion: `workflow: ${assertion.name} ${assertion.entered ? "entered" : "not entered"}`,
+        pass,
+        expected: assertion.entered ? `Workflow "${assertion.name}" was entered` : `Workflow "${assertion.name}" was not entered`,
+        actual: wasEntered ? `Found ${matching.length} workflow span(s)` : `No workflow spans found`
+      });
+    }
+    if (assertion.completed !== void 0) {
+      const didComplete = matching.some(
+        (wf) => wf.statusFinal === "completed" || wf.status === "ok" || wf.status === "error"
+      );
+      const pass = assertion.completed ? didComplete : !didComplete;
+      results.push({
+        assertion: `workflow: ${assertion.name} ${assertion.completed ? "completed" : "not completed"}`,
+        pass,
+        expected: assertion.completed ? `Workflow "${assertion.name}" completed` : `Workflow "${assertion.name}" did not complete`,
+        actual: didComplete ? `Completed` : `Not completed`
+      });
+    }
+  }
+  return results;
+}
+
+// src/graders/timing.ts
+function gradeTiming(botDuration, assertions) {
+  const results = [];
+  for (const assertion of assertions) {
+    const pass = matchValue(assertion.response_time, botDuration);
+    const expected = operatorToString(assertion.response_time);
+    results.push({
+      assertion: `response_time ${expected}`,
+      pass,
+      expected: `Response time ${expected}`,
+      actual: `${botDuration}ms`
+    });
+  }
+  return results;
+}
+
+// src/graders/outcome.ts
+function gradeOutcome(mutations, evalDef, workflowSpans) {
+  const outcome = evalDef.outcome;
+  if (!outcome) return [];
+  const results = [];
+  if (outcome.state && outcome.state.length > 0) {
+    const stateResults = gradeState(mutations, outcome.state);
+    results.push(...stateResults);
+  }
+  if (outcome.workflow && outcome.workflow.length > 0) {
+    const workflowResults = gradeWorkflows(workflowSpans, outcome.workflow);
+    results.push(...workflowResults);
+  }
+  return results;
+}
+
+// src/loader.ts
+import { readdirSync, existsSync } from "fs";
+import { resolve } from "path";
+
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// src/loader.ts
+function isEvalDefinition(value) {
+  return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
+}
+async function loadEvalFile(filePath) {
+  const absPath = resolve(filePath);
+  let mod;
+  try {
+    mod = await import(absPath);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `Failed to load eval file ${filePath}: ${msg}
+
+Make sure your eval file:
+  - Has no syntax or type errors
+  - Exports one or more \`new Eval({...})\` instances
+  - Has all dependencies installed (\`bun install\`)`,
+      { cause: err }
+    );
+  }
+  const results = [];
+  for (const [key, value] of Object.entries(mod)) {
+    if (key === "__esModule") continue;
+    if (value instanceof Eval || isEvalDefinition(value)) {
+      results.push(value);
+    }
+  }
+  if (results.length === 0) {
+    throw new Error(
+      `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`
+    );
+  }
+  return results;
+}
+async function loadEvalsFromDir(dirPath) {
+  const absDir = resolve(dirPath);
+  if (!existsSync(absDir)) {
+    return [];
+  }
+  const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
+  const evals = [];
+  for (const f of files) {
+    const defs = await loadEvalFile(`${absDir}/${f}`);
+    evals.push(...defs);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const e of evals) {
+    if (seen.has(e.name)) {
+      throw new Error(`Duplicate eval name "${e.name}" found in ${dirPath}`);
+    }
+    seen.add(e.name);
+  }
+  return evals;
+}
+function filterEvals(evals, filter) {
+  if (!filter) return evals;
+  return evals.filter((e) => {
+    if (filter.names && filter.names.length > 0) {
+      if (!filter.names.includes(e.name)) return false;
+    }
+    if (filter.tags && filter.tags.length > 0) {
+      if (!e.tags || !filter.tags.some((t) => e.tags.includes(t))) return false;
+    }
+    if (filter.type) {
+      if (e.type !== filter.type) return false;
+    }
+    return true;
+  });
+}
+
+// src/runner.ts
+import { randomUUID } from "crypto";
+var DEFAULT_IDLE_TIMEOUT = 3e4;
+var DEFAULT_DEV_SERVER_URL = "http://localhost:3001";
+var WORKFLOW_TRIGGER_TIMEOUT_MS = 5 * 60 * 1e3;
+var BUILT_IN_STATES = {
+  conversation: "state",
+  user: "userState",
+  bot: "botState"
+};
+function buildStatePayload(value) {
+  return {
+    value,
+    location: { type: "state" }
+  };
+}
+async function seedEvalState(client, ctx, setup) {
+  const state = setup?.state;
+  if (!state) return;
+  const writes = [];
+  if (state.bot) {
+    writes.push(
+      client.setState({
+        type: "bot",
+        id: ctx.botId,
+        name: BUILT_IN_STATES.bot,
+        payload: buildStatePayload(state.bot)
+      })
+    );
+  }
+  if (state.user) {
+    writes.push(
+      client.setState({
+        type: "user",
+        id: ctx.userId,
+        name: BUILT_IN_STATES.user,
+        payload: buildStatePayload(state.user)
+      })
+    );
+  }
+  if (state.conversation) {
+    if (!ctx.conversationId) {
+      throw new Error("Cannot seed conversation state before a conversation is created.");
+    }
+    writes.push(
+      client.setState({
+        type: "conversation",
+        id: ctx.conversationId,
+        name: BUILT_IN_STATES.conversation,
+        payload: buildStatePayload(state.conversation)
+      })
+    );
+  }
+  await Promise.all(writes);
+}
+async function triggerEvalWorkflow(client, ctx, setup) {
+  const workflow = setup?.workflow;
+  if (!workflow) return;
+  await client.createWorkflow({
+    name: workflow.trigger,
+    input: workflow.input ?? {},
+    status: "pending",
+    userId: ctx.userId,
+    timeoutAt: new Date(Date.now() + WORKFLOW_TRIGGER_TIMEOUT_MS).toISOString(),
+    ...ctx.conversationId !== void 0 ? { conversationId: ctx.conversationId } : {}
+  });
+}
+function evalMetadata(evalDef) {
+  return {
+    name: evalDef.name,
+    ...evalDef.description !== void 0 ? { description: evalDef.description } : {},
+    ...evalDef.type !== void 0 ? { type: evalDef.type } : {},
+    ...evalDef.tags !== void 0 ? { tags: evalDef.tags } : {}
+  };
+}
+async function runEval(evalDef, connection, options = {}) {
+  const devServerUrl = options.devServerUrl || DEFAULT_DEV_SERVER_URL;
+  const devServerHeaders = options.devServerHeaders ?? {};
+  const idleTimeout = evalDef.options?.idleTimeout ?? options.idleTimeout ?? DEFAULT_IDLE_TIMEOUT;
+  const judgePassThreshold = evalDef.options?.judgePassThreshold ?? options.judgePassThreshold;
+  const start = Date.now();
+  const turns = [];
+  let outcomeAssertions = [];
+  const collector = new SSECollector(devServerUrl, devServerHeaders);
+  try {
+    const session = new ChatSession(connection.webhookId, options.chatClient);
+    await session.connect();
+    let bpClient = null;
+    const getBpClient = () => {
+      if (!bpClient) {
+        bpClient = new BpClient2({
+          token: connection.token,
+          botId: connection.botId,
+          apiUrl: connection.apiUrl
+        });
+      }
+      return bpClient;
+    };
+    let conversationId = "";
+    if (evalDef.setup?.state?.conversation || evalDef.setup?.workflow) {
+      conversationId = await session.ensureConversation();
+    }
+    await seedEvalState(
+      getBpClient(),
+      {
+        botId: connection.botId,
+        userId: session.userId,
+        ...conversationId ? { conversationId } : {}
+      },
+      evalDef.setup
+    );
+    await triggerEvalWorkflow(
+      getBpClient(),
+      {
+        userId: session.userId,
+        ...conversationId ? { conversationId } : {}
+      },
+      evalDef.setup
+    );
+    if (!conversationId && evalDef.conversation.length > 0) {
+      conversationId = await session.ensureConversation();
+    }
+    if (conversationId) {
+      await collector.connect({ conversationId });
+    } else if (evalDef.conversation.length > 0) {
+      throw new Error("Cannot run conversation eval without a conversationId \u2014 no chat session or setup created one.");
+    }
+    for (let i = 0; i < evalDef.conversation.length; i++) {
+      if (options.signal?.aborted) break;
+      const turn = evalDef.conversation[i];
+      if (turn.expectSilence && turn.assert?.response) {
+        throw new Error(`Turn ${i + 1}: 'expectSilence' and 'assert.response' are mutually exclusive.`);
+      }
+      if (turn.user && turn.event) {
+        throw new Error(`Turn ${i + 1}: 'user' and 'event' are mutually exclusive.`);
+      }
+      if (!turn.user && !turn.event) {
+        throw new Error(`Turn ${i + 1}: must have either 'user' or 'event'.`);
+      }
+      const turnLabelForEvent = turn.event ? `[event: ${turn.event.type}]` : turn.user;
+      options.onProgress?.({
+        type: "turn_start",
+        evalName: evalDef.name,
+        evalIndex: options.evalIndex ?? 0,
+        turnNumber: i + 1,
+        totalTurns: evalDef.conversation.length,
+        userMessage: turnLabelForEvent
+      });
+      collector.startTurn();
+      if (turn.event) {
+        await session.sendEvent(turn.event.type, turn.event.payload ?? {});
+      } else {
+        await session.sendMessage(turn.user);
+      }
+      try {
+        await collector.waitForTurnComplete({
+          timeout: idleTimeout,
+          ...options.signal !== void 0 ? { abortSignal: options.signal } : {}
+        });
+      } catch (err) {
+        if (options.signal?.aborted) break;
+        throw err;
+      }
+      if (turn.assert?.workflow) {
+        for (const wfAssert of turn.assert.workflow) {
+          const expectsPositive = wfAssert.entered !== false && wfAssert.completed !== false;
+          if (!expectsPositive) continue;
+          const wfSignal = wfAssert.completed ? "completed" : "entered";
+          await collector.waitForWorkflow(wfAssert.name, {
+            signal: wfSignal,
+            timeout: idleTimeout,
+            ...options.signal !== void 0 ? { abortSignal: options.signal } : {}
+          }).catch(() => {
+          });
+        }
+      }
+      const turnSpans = collector.getTurnSpans();
+      const turnData = transformSpans(turnSpans);
+      const botResponse = turnData.messages.join("\n");
+      const botDuration = turnData.handlerDuration;
+      const turnLabel = turn.event ? `[event: ${turn.event.type}]` : turn.user;
+      if (!turn.expectSilence && turnData.messages.length === 0) {
+        throw new Error(`Turn ${i + 1}: bot produced no response.`);
+      }
+      const evalStart = Date.now();
+      let assertions = [];
+      if (turn.expectSilence) {
+        const wasSilent = turnData.messages.length === 0;
+        assertions.push({
+          assertion: "no_response",
+          pass: wasSilent,
+          expected: "No response",
+          actual: wasSilent ? "No response" : `Bot responded: "${botResponse}"`
+        });
+      }
+      if (turn.assert?.response) {
+        assertions = await gradeResponse(botResponse, turn.assert.response, {
+          userMessage: turnLabel,
+          ...judgePassThreshold !== void 0 ? { judgePassThreshold } : {}
+        });
+      }
+      if (turn.assert?.tools) {
+        const toolResults = gradeTools(turnData.toolCalls, turn.assert.tools);
+        assertions = [...assertions, ...toolResults];
+      }
+      if (turn.assert?.state) {
+        try {
+          const stateResults = gradeState(turnData.stateMutations, turn.assert.state);
+          assertions.push(...stateResults);
+        } catch (err) {
+          assertions.push({
+            assertion: "state",
+            pass: false,
+            expected: "State assertions executed",
+            actual: `Error: ${err.message}`
+          });
+        }
+      }
+      if (turn.assert?.workflow) {
+        const workflowResults = gradeWorkflows(turnData.workflowSpans, turn.assert.workflow);
+        assertions.push(...workflowResults);
+      }
+      if (turn.assert?.timing) {
+        const timingResults = gradeTiming(botDuration, turn.assert.timing);
+        assertions.push(...timingResults);
+      }
+      const turnPass = assertions.every((a) => a.pass);
+      const evalDuration = Date.now() - evalStart;
+      const turnReport = {
+        turnNumber: i + 1,
+        userMessage: turnLabel,
+        botResponse,
+        assertions,
+        pass: turnPass,
+        botDuration,
+        evalDuration
+      };
+      turns.push(turnReport);
+      options.onProgress?.({
+        type: "turn_complete",
+        evalName: evalDef.name,
+        evalIndex: options.evalIndex ?? 0,
+        turnNumber: i + 1,
+        totalTurns: evalDef.conversation.length,
+        turnReport
+      });
+    }
+    if (evalDef.outcome && !options.signal?.aborted) {
+      if (evalDef.outcome.workflow) {
+        for (const wfAssert of evalDef.outcome.workflow) {
+          const expectsPositive = wfAssert.entered !== false && wfAssert.completed !== false;
+          if (!expectsPositive) continue;
+          const wfSignal = wfAssert.completed ? "completed" : "entered";
+          await collector.waitForWorkflow(wfAssert.name, {
+            signal: wfSignal,
+            timeout: idleTimeout,
+            ...options.signal !== void 0 ? { abortSignal: options.signal } : {}
+          }).catch(() => {
+          });
+        }
+      }
+      try {
+        const allTurnData = transformSpans(collector.getAllSpans());
+        outcomeAssertions = gradeOutcome(allTurnData.stateMutations, evalDef, allTurnData.workflowSpans);
+      } catch (err) {
+        outcomeAssertions = [
+          {
+            assertion: "outcome",
+            pass: false,
+            expected: "Outcome assertions executed",
+            actual: `Error: ${err.message}`
+          }
+        ];
+      }
+    }
+    const aborted = options.signal?.aborted === true;
+    const turnsPass = turns.every((t) => t.pass);
+    const outcomePass = outcomeAssertions.every((a) => a.pass);
+    return {
+      ...evalMetadata(evalDef),
+      turns,
+      outcomeAssertions,
+      pass: !aborted && turnsPass && outcomePass,
+      duration: Date.now() - start,
+      ...aborted ? { error: "Eval aborted" } : {}
+    };
+  } catch (err) {
+    return {
+      ...evalMetadata(evalDef),
+      turns,
+      outcomeAssertions,
+      pass: false,
+      duration: Date.now() - start,
+      error: err.message
+    };
+  } finally {
+    collector.disconnect();
+  }
+}
+async function runEvalSuite(config, filter) {
+  const start = Date.now();
+  const runId = randomUUID().replace(/-/g, "").slice(0, 26);
+  const evalsDir = `${config.agentPath}/evals`;
+  const allEvals = await loadEvalsFromDir(evalsDir);
+  const evals = filterEvals(allEvals, filter);
+  if (evals.length === 0) {
+    return {
+      id: runId,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      evals: [],
+      passed: 0,
+      failed: 0,
+      total: 0,
+      duration: 0,
+      ...filter !== void 0 ? { filter } : {}
+    };
+  }
+  await initLLMJudge(
+    {
+      token: config.credentials.token,
+      apiUrl: config.credentials.apiUrl,
+      botId: config.credentials.botId,
+      ...config.credentials.workspaceId !== void 0 ? { workspaceId: config.credentials.workspaceId } : {}
+    },
+    {
+      ...config.evalOptions?.judgeModel !== void 0 ? { model: config.evalOptions.judgeModel } : {},
+      ...config.logger !== void 0 ? { logger: config.logger } : {}
+    }
+  );
+  let webhookId = config.credentials.webhookId;
+  if (!webhookId) {
+    webhookId = await discoverWebhookId(config.credentials.botId, config.credentials.token, config.credentials.apiUrl);
+  }
+  const connection = {
+    webhookId,
+    botId: config.credentials.botId,
+    token: config.credentials.token,
+    apiUrl: config.credentials.apiUrl
+  };
+  const devServerUrl = config.devServerUrl || DEFAULT_DEV_SERVER_URL;
+  const devServerHeaders = config.devServerHeaders ?? {};
+  await assertChatChannelBound(devServerUrl, devServerHeaders);
+  const reports = [];
+  config.onProgress?.({ type: "suite_start", totalEvals: evals.length });
+  for (let i = 0; i < evals.length; i++) {
+    if (config.signal?.aborted) break;
+    const evalDef = evals[i];
+    config.onProgress?.({
+      type: "eval_start",
+      evalName: evalDef.name,
+      index: i,
+      totalTurns: evalDef.conversation.length
+    });
+    const report = await runEval(evalDef, connection, {
+      devServerUrl,
+      devServerHeaders,
+      evalIndex: i,
+      ...config.evalOptions?.idleTimeout !== void 0 ? { idleTimeout: config.evalOptions.idleTimeout } : {},
+      ...config.evalOptions?.judgePassThreshold !== void 0 ? { judgePassThreshold: config.evalOptions.judgePassThreshold } : {},
+      ...config.onProgress !== void 0 ? { onProgress: config.onProgress } : {},
+      ...config.signal !== void 0 ? { signal: config.signal } : {},
+      ...config.chatClient !== void 0 ? { chatClient: config.chatClient } : {}
+    });
+    reports.push(report);
+    config.onProgress?.({ type: "eval_complete", evalName: evalDef.name, index: i, report });
+  }
+  const runReport = {
+    id: runId,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    evals: reports,
+    passed: reports.filter((r) => r.pass).length,
+    failed: reports.filter((r) => !r.pass).length,
+    total: reports.length,
+    duration: Date.now() - start,
+    ...filter !== void 0 ? { filter } : {}
+  };
+  if (config.signal?.aborted) {
+    runReport.aborted = true;
+  }
+  config.onProgress?.({ type: "suite_complete", report: runReport });
+  return runReport;
+}
+export {
+  runEval,
+  runEvalSuite
+};
+//# sourceMappingURL=runner.js.map