@botpress/evals 1.19.0-beta.6 → 1.19.0-beta.9

package/dist/sse-collector.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sse-collector.d.ts","sourceRoot":"","sources":["../src/sse-collector.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEnC,KAAK,SAAS,GAAG;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAA;AAEpE,UAAU,WAAW;IACnB,OAAO,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B;AAED,UAAU,mBAAoB,SAAQ,WAAW;IAC/C,MAAM,EAAE,SAAS,GAAG,WAAW,CAAA;CAChC;AAgBD,qBAAa,YAAY;IACvB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,OAAO,CAAwB;IAEvC,yEAAyE;IACzE,OAAO,CAAC,OAAO,CAA0B;IAEzC,iFAAiF;IACjF,OAAO,CAAC,aAAa,CAAoB;IAEzC,+EAA+E;IAC/E,OAAO,CAAC,eAAe,CAAoB;IAE3C,0CAA0C;IAC1C,OAAO,CAAC,OAAO,CAAkC;IAEjD,qDAAqD;IACrD,OAAO,CAAC,eAAe,CAA+B;IAEtD,8CAA8C;IAC9C,OAAO,CAAC,SAAS,CAAQ;IAEzB,yDAAyD;IACzD,OAAO,CAAC,WAAW,CAAqB;IAExC,sFAAsF;IACtF,OAAO,CAAC,aAAa,CAA4B;gBAErC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM;IAKtE;;;OAGG;IACG,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAoC/C;;;;OAIG;IACH,SAAS,IAAI,IAAI;IAYjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEvC;;;;;;;;OAQG;IACG,mBAAmB,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAuC3D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAQ;IAE7C;;;;;;OAMG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;;;OAIG;IACG,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyC7E;;OAEG;IACH,YAAY,IAAI,IAAI,EAAE;IAUtB;;OAEG;IACH,WAAW,IAAI,IAAI,EAAE;IAIrB;;OAEG;IACH,UAAU,IAAI,IAAI;IAgBlB,qDAAqD;IACrD,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,iBAAiB,CAAwB;IAEjD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAEpD;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IA2BxB;;OAEG;YACW,cAAc;IA2B5B;;;OAGG;IACH,OAAO,CAAC,cAAc;IA+BtB;;OAEG;IACH,OAAO,CAAC,YAAY;IAsBpB;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,aAAa;IAOrB;;;;;;;;;OASG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;CAmBlC"}
+{"version":3,"file":"sse-collector.d.ts","sourceRoot":"","sources":["../src/sse-collector.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAGnC,KAAK,SAAS,GAAG;IAAE,cAAc,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,UAAU,EAAE,MAAM,CAAA;CAAE,CAAA;AAEpE,UAAU,WAAW;IACnB,OAAO,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,WAAW,CAAC,EAAE,WAAW,CAAA;CAC1B;AAED,UAAU,mBAAoB,SAAQ,WAAW;IAC/C,MAAM,EAAE,SAAS,GAAG,WAAW,CAAA;CAChC;AAgBD,qBAAa,YAAY;IACvB,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,OAAO,CAAwB;IAEvC,yEAAyE;IACzE,OAAO,CAAC,OAAO,CAA0B;IAEzC,iFAAiF;IACjF,OAAO,CAAC,aAAa,CAAoB;IAEzC,+EAA+E;IAC/E,OAAO,CAAC,eAAe,CAAoB;IAE3C,0CAA0C;IAC1C,OAAO,CAAC,OAAO,CAAkC;IAEjD,qDAAqD;IACrD,OAAO,CAAC,eAAe,CAA+B;IAEtD,8CAA8C;IAC9C,OAAO,CAAC,SAAS,CAAQ;IAEzB,yDAAyD;IACzD,OAAO,CAAC,WAAW,CAAqB;IAExC,sFAAsF;IACtF,OAAO,CAAC,aAAa,CAA4B;gBAErC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAM;IAKtE;;;OAGG;IACG,OAAO,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;IAyC/C;;;;OAIG;IACH,SAAS,IAAI,IAAI;IAYjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAM;IAEvC;;;;;;;;OAQG;IACG,mBAAmB,CAAC,IAAI,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAuC3D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAQ;IAE7C;;;;;;OAMG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;;;OAIG;IACG,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAyC7E;;OAEG;IACH,YAAY,IAAI,IAAI,EAAE;IAUtB;;OAEG;IACH,WAAW,IAAI,IAAI,EAAE;IAIrB;;OAEG;IACH,UAAU,IAAI,IAAI;IAgBlB,qDAAqD;IACrD,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,iBAAiB,CAAwB;IAEjD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAEpD;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IA2BxB;;OAEG;YACW,cAAc;IA2B5B;;;OAGG;IACH,OAAO,CAAC,cAAc;IA+BtB;;OAEG;IACH,OAAO,CAAC,YAAY;IAsBpB;;OAEG;IACH,OAAO,CAAC,WAAW;IAInB;;OAEG;IACH,OAAO,CAAC,cAAc;IAQtB;;OAEG;IACH,OAAO,CAAC,aAAa;IAOrB;;;;;;;;;OASG;IACH,OAAO,CAAC,uBAAuB;IAc/B;;OAEG;IACH,OAAO,CAAC,yBAAyB;CAmBlC"}

package/dist/sse-collector.js

@@ -1,3 +1,46 @@
+// ../analytics/src/errors.ts
+var AdkError = class extends Error {
+  /**
+   * Marker for cross-module-instance checks — `instanceof` breaks when a
+   * package is duplicated in the dependency graph.
+   * @internal
+   */
+  static __IS_ADK_BASE_ERROR = true;
+  /**
+   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
+   * pin `Code` to a literal union (`class EvalRunnerError extends
+   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
+   * in switches.
+   */
+  code;
+  /**
+   * True for user/environment conditions the developer can fix themselves
+   * (validation, auth, connectivity). False for internal bugs. Defaults to
+   * false so unclassified errors surface as bugs until someone triages them.
+   */
+  expected;
+  /** Structured context. Sanitized before any telemetry leaves the machine. */
+  details;
+  /** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
+  suggestion;
+  constructor(opts) {
+    super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
+    this.name = this.constructor.name;
+    this.code = opts.code;
+    this.expected = opts.expected ?? false;
+    if (opts.details !== void 0) {
+      this.details = opts.details;
+    }
+    if (opts.suggestion !== void 0) {
+      this.suggestion = opts.suggestion;
+    }
+  }
+};
+
+// src/errors.ts
+var EvalRunnerError = class extends AdkError {
+};
+
 // src/sse-collector.ts
 function spanKey(span) {
   return `${span.id.trace}:${span.id.span}`;
@@ -42,10 +85,15 @@ var SSECollector = class _SSECollector {
       signal: this.abortController.signal
     });
     if (!response.ok) {
-      throw new Error(`SSE connection failed: ${response.status} ${response.statusText}`);
+      throw new EvalRunnerError({
+        code: "SSE_CONNECT_FAILED",
+        message: `SSE connection failed: ${response.status} ${response.statusText}`,
+        expected: true,
+        suggestion: "Make sure the dev server is running (`adk dev`) and reachable."
+      });
     }
     if (!response.body) {
-      throw new Error("SSE response has no body");
+      throw new EvalRunnerError({ code: "SSE_NO_BODY", message: "SSE response has no body" });
     }
     this.connected = true;
     this._consumeStream(response.body).catch((err) => {

package/dist/sse-collector.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../src/sse-collector.ts"],
-  "sourcesContent": ["/**\n * SSECollector \u2014 connects to the ADK dev server's SSE trace stream,\n * accumulates spans, and provides promise-based wait methods for\n * turn completion and workflow signals.\n *\n * Uses fetch() + async iteration with manual SSE line parsing\n * (Bun does not have native EventSource).\n */\n\nimport type { Span } from './trace'\n\ntype SSEFilter = { conversationId: string } | { workflowId: string }\n\ninterface WaitOptions {\n  timeout: number\n  /**\n   * Optional abort signal \u2014 when fired, the wait rejects immediately rather\n   * than burning the full timeout. Used by the runner so Stop unblocks\n   * pending workflow assertions (otherwise a single negative-result wait\n   * can hold the suite hostage for `idleTimeout` per assertion).\n   */\n  abortSignal?: AbortSignal\n}\n\ninterface WorkflowWaitOptions extends WaitOptions {\n  signal: 'entered' | 'completed'\n}\n\n/**\n * Unique key for span deduplication.\n */\nfunction spanKey(span: Span): string {\n  return `${span.id.trace}:${span.id.span}`\n}\n\n/**\n * Type-safe accessor for span.data fields.\n */\nfunction spanData(span: Span): Record<string, unknown> {\n  return span.data && typeof span.data === 'object' ? (span.data as Record<string, unknown>) : {}\n}\n\nexport class SSECollector {\n  private devServerUrl: string\n  private headers: Record<string, string>\n\n  /** All spans accumulated since connect(), keyed by spanKey for dedup. */\n  private spanMap = new Map<string, Span>()\n\n  /** Index into spanMap insertion order marking where the current turn started. */\n  private turnStartKeys = new Set<string>()\n\n  /** Handler span IDs seen before the current turn (for startTurn filtering). */\n  private seenHandlerKeys = new Set<string>()\n\n  /** Callbacks waiting for span updates. */\n  private waiters: Array<(span: Span) => void> = []\n\n  /** AbortController for the active SSE connection. */\n  private abortController: AbortController | null = null\n\n  /** Whether the background loop is running. */\n  private connected = false\n\n  /** Error from the SSE stream if it died unexpectedly. */\n  private streamError: Error | null = null\n\n  /** Set while a settle phase is in flight; called by disconnect() to short-circuit. */\n  private _cancelSettle: (() => void) | null = null\n\n  constructor(devServerUrl: string, headers: Record<string, string> = {}) {\n    this.devServerUrl = devServerUrl\n    this.headers = headers\n  }\n\n  /**\n   * Open a streaming SSE connection filtered by conversationId or workflowId.\n   * Starts a background loop that parses SSE events and accumulates spans.\n   */\n  async connect(filter: SSEFilter): Promise<void> {\n    const attributeName = 'conversationId' in filter ? 'conversationId' : 'workflowId'\n    const attributeValue = 'conversationId' in filter ? filter.conversationId : filter.workflowId\n\n    const url = `${this.devServerUrl}/api/traces/stream?attributeName=${encodeURIComponent(attributeName)}&attributeValue=${encodeURIComponent(attributeValue)}`\n\n    this.abortController = new AbortController()\n\n    const response = await fetch(url, {\n      headers: { ...this.headers, Accept: 'text/event-stream' },\n      signal: this.abortController.signal,\n    })\n\n    if (!response.ok) {\n      throw new Error(`SSE connection failed: ${response.status} ${response.statusText}`)\n    }\n\n    if (!response.body) {\n      throw new Error('SSE response has no body')\n    }\n\n    this.connected = true\n\n    // Start background parsing loop\n    this._consumeStream(response.body).catch((err) => {\n      if (this.connected) {\n        this.streamError = err instanceof Error ? err : new Error(String(err))\n      }\n    })\n\n    // Wait for the initial snapshot to arrive before returning.\n    // This ensures that connect() resolves only after the snapshot\n    // has been processed so callers can immediately check spans.\n    await this._waitForSnapshot()\n  }\n\n  /**\n   * Mark the beginning of a new turn.\n   * Records which handler.conversation span IDs have been seen so far,\n   * so waitForTurnComplete() only resolves on NEW handler completions.\n   */\n  startTurn(): void {\n    // Snapshot the current set of span keys as the turn boundary\n    this.turnStartKeys = new Set(this.spanMap.keys())\n\n    // Record all handler.conversation spans seen so far\n    for (const [key, span] of this.spanMap) {\n      if (span.name === 'handler.conversation') {\n        this.seenHandlerKeys.add(key)\n      }\n    }\n  }\n\n  private static readonly SETTLE_MS = 100\n\n  /**\n   * Wait for a handler.conversation span to reach terminal status (ok or error).\n   * Only considers spans NOT seen before the last startTurn() call.\n   *\n   * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)\n   * to ensure all child spans (tool calls, messages, state saves) have arrived.\n   * The runtime's HttpSpanExporter fires POSTs without awaiting them, so child\n   * spans may arrive slightly after the handler close event.\n   */\n  async waitForTurnComplete(opts: WaitOptions): Promise<void> {\n    // Check if a completed handler already exists (e.g. from snapshot)\n    if (this._hasNewCompletedHandler()) {\n      return this._settleAfterHandler()\n    }\n    if (opts.abortSignal?.aborted) {\n      throw new Error('waitForTurnComplete aborted')\n    }\n\n    return new Promise<void>((resolve, reject) => {\n      const timer = setTimeout(() => {\n        this._removeWaiter(waiter)\n        opts.abortSignal?.removeEventListener('abort', onAbort)\n        const reason = this.streamError\n          ? `SSE stream died: ${this.streamError.message}`\n          : 'no handler.conversation span completed'\n        reject(new Error(`waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}`))\n      }, opts.timeout)\n\n      const onAbort = () => {\n        clearTimeout(timer)\n        this._removeWaiter(waiter)\n        reject(new Error('waitForTurnComplete aborted'))\n      }\n      opts.abortSignal?.addEventListener('abort', onAbort, { once: true })\n\n      const waiter = (_span: Span) => {\n        if (this._hasNewCompletedHandler()) {\n          clearTimeout(timer)\n          this._removeWaiter(waiter)\n          opts.abortSignal?.removeEventListener('abort', onAbort)\n          this._settleAfterHandler().then(resolve, reject)\n        }\n      }\n\n      this.waiters.push(waiter)\n    })\n  }\n\n  private static readonly MAX_SETTLE_MS = 2_000\n\n  /**\n   * After the handler span closes, wait for spans to stop arriving.\n   * Resolves when no new spans arrive for SETTLE_MS, or after MAX_SETTLE_MS hard cap.\n   *\n   * Exposes a cancellation hook via _cancelSettle so disconnect() can short-\n   * circuit an in-flight settle phase and avoid leaking timers for up to 2s.\n   */\n  private _settleAfterHandler(): Promise<void> {\n    return new Promise<void>((resolve) => {\n      let resolved = false\n      const done = () => {\n        if (resolved) return\n        resolved = true\n        clearTimeout(hardCap)\n        clearTimeout(settleTimer)\n        this._removeWaiter(settleWaiter)\n        this._cancelSettle = null\n        resolve()\n      }\n\n      const hardCap = setTimeout(done, SSECollector.MAX_SETTLE_MS)\n\n      let settleTimer = setTimeout(done, SSECollector.SETTLE_MS)\n\n      const settleWaiter = () => {\n        if (resolved) return\n        clearTimeout(settleTimer)\n        settleTimer = setTimeout(done, SSECollector.SETTLE_MS)\n      }\n\n      this.waiters.push(settleWaiter)\n      this._cancelSettle = done\n    })\n  }\n\n  /**\n   * Wait for a handler.workflow span with matching workflow name.\n   * - signal 'entered': resolves when the span appears (any status)\n   * - signal 'completed': resolves when the span has status !== 'running'\n   */\n  async waitForWorkflow(name: string, opts: WorkflowWaitOptions): Promise<void> {\n    // Check if condition is already met\n    if (this._matchesWorkflowCondition(name, opts.signal)) {\n      return\n    }\n    if (opts.abortSignal?.aborted) {\n      throw new Error(`waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) aborted`)\n    }\n\n    return new Promise<void>((resolve, reject) => {\n      const timer = setTimeout(() => {\n        this._removeWaiter(waiter)\n        opts.abortSignal?.removeEventListener('abort', onAbort)\n        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : 'span not found'\n        reject(\n          new Error(\n            `waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) timed out after ${opts.timeout}ms \u2014 ${reason}`\n          )\n        )\n      }, opts.timeout)\n\n      const onAbort = () => {\n        clearTimeout(timer)\n        this._removeWaiter(waiter)\n        reject(new Error(`waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) aborted`))\n      }\n      opts.abortSignal?.addEventListener('abort', onAbort, { once: true })\n\n      const waiter = (_span: Span) => {\n        if (this._matchesWorkflowCondition(name, opts.signal)) {\n          clearTimeout(timer)\n          this._removeWaiter(waiter)\n          opts.abortSignal?.removeEventListener('abort', onAbort)\n          resolve()\n        }\n      }\n\n      this.waiters.push(waiter)\n    })\n  }\n\n  /**\n   * Return spans accumulated since the last startTurn() call.\n   */\n  getTurnSpans(): Span[] {\n    const result: Span[] = []\n    for (const [key, span] of this.spanMap) {\n      if (!this.turnStartKeys.has(key)) {\n        result.push(span)\n      }\n    }\n    return result\n  }\n\n  /**\n   * Return all spans accumulated since connect().\n   */\n  getAllSpans(): Span[] {\n    return Array.from(this.spanMap.values())\n  }\n\n  /**\n   * Close the SSE connection and clean up.\n   */\n  disconnect(): void {\n    this.connected = false\n    this.abortController?.abort()\n    this.abortController = null\n    // Short-circuit an in-flight settle phase so its hardCap / settleTimer\n    // don't fire after disconnect.\n    this._cancelSettle?.()\n    this._cancelSettle = null\n    // Waiters hold setTimeout references that will reject on their own;\n    // clearing the array just prevents further notifications.\n    this.waiters = []\n    this.snapshotResolvers = []\n  }\n\n  // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  /** Whether we have received the initial snapshot. */\n  private snapshotReceived = false\n  private snapshotResolvers: Array<() => void> = []\n\n  private static readonly SNAPSHOT_TIMEOUT_MS = 10_000\n\n  /**\n   * Returns a promise that resolves once the snapshot event has been processed,\n   * or rejects after a timeout to prevent connect() from hanging indefinitely.\n   */\n  private _waitForSnapshot(): Promise<void> {\n    if (this.snapshotReceived) {\n      return Promise.resolve()\n    }\n    return new Promise<void>((resolve, reject) => {\n      let settled = false\n      const resolver = () => {\n        if (settled) return\n        settled = true\n        clearTimeout(timer)\n        resolve()\n      }\n      const timer = setTimeout(() => {\n        if (settled) return\n        settled = true\n        const idx = this.snapshotResolvers.indexOf(resolver)\n        if (idx !== -1) this.snapshotResolvers.splice(idx, 1)\n        reject(\n          new Error(\n            `SSE snapshot not received within ${SSECollector.SNAPSHOT_TIMEOUT_MS}ms \u2014 is the dev server healthy?`\n          )\n        )\n      }, SSECollector.SNAPSHOT_TIMEOUT_MS)\n      this.snapshotResolvers.push(resolver)\n    })\n  }\n\n  /**\n   * Consume the ReadableStream body, parse SSE lines, and dispatch events.\n   */\n  private async _consumeStream(body: ReadableStream<Uint8Array>): Promise<void> {\n    const decoder = new TextDecoder()\n    let buffer = ''\n\n    const reader = body.getReader()\n\n    try {\n      while (this.connected) {\n        const { done, value } = await reader.read()\n        if (done) break\n\n        buffer += decoder.decode(value, { stream: true })\n\n        // Split on double-newline boundaries to get complete SSE events\n        const parts = buffer.split('\\n\\n')\n        // The last part may be incomplete \u2014 keep it in the buffer\n        buffer = parts.pop() ?? ''\n\n        for (const part of parts) {\n          this._parseSSEBlock(part)\n        }\n      }\n    } finally {\n      reader.releaseLock()\n    }\n  }\n\n  /**\n   * Parse a single SSE event block (text between \\n\\n boundaries).\n   * Extracts event name and data payload.\n   */\n  private _parseSSEBlock(block: string): void {\n    const lines = block.split('\\n')\n    let eventName = ''\n    let dataLine = ''\n\n    for (const line of lines) {\n      if (line.startsWith('event:')) {\n        eventName = line.slice('event:'.length).trim()\n      } else if (line.startsWith('data:')) {\n        const value = line.slice('data:'.length).trim()\n        dataLine = dataLine ? `${dataLine}\\n${value}` : value\n      }\n      // id: and retry: lines are ignored\n    }\n\n    if (!eventName || !dataLine) {\n      // Could be a retry: line or empty block \u2014 skip\n      if (!this.snapshotReceived && block.includes('retry:')) {\n        // The retry line comes before the snapshot; don't resolve yet\n      }\n      return\n    }\n\n    try {\n      const payload = JSON.parse(dataLine)\n      this._handleEvent(eventName, payload)\n    } catch {\n      // Malformed JSON \u2014 skip\n    }\n  }\n\n  /**\n   * Dispatch a parsed SSE event to the appropriate handler.\n   */\n  private _handleEvent(eventName: string, payload: unknown): void {\n    if (eventName === 'snapshot') {\n      const data = payload as { spans: Span[] }\n      const spans = Array.isArray(data.spans) ? data.spans : []\n      for (const span of spans) {\n        this._upsertSpan(span)\n      }\n      this.snapshotReceived = true\n      for (const resolve of this.snapshotResolvers) {\n        resolve()\n      }\n      this.snapshotResolvers = []\n    } else if (eventName === 'update') {\n      const data = payload as { span: Span }\n      if (data.span) {\n        this._upsertSpan(data.span)\n        this._notifyWaiters(data.span)\n      }\n    }\n    // keepalive events are intentionally ignored\n  }\n\n  /**\n   * Insert or replace a span in the map (dedup by trace+span ID).\n   */\n  private _upsertSpan(span: Span): void {\n    this.spanMap.set(spanKey(span), span)\n  }\n\n  /**\n   * Notify all waiters that a new span was received.\n   */\n  private _notifyWaiters(span: Span): void {\n    // Copy the array since waiters may remove themselves during iteration\n    const current = [...this.waiters]\n    for (const waiter of current) {\n      waiter(span)\n    }\n  }\n\n  /**\n   * Remove a waiter callback from the list.\n   */\n  private _removeWaiter(fn: (span: Span) => void): void {\n    const idx = this.waiters.indexOf(fn)\n    if (idx !== -1) {\n      this.waiters.splice(idx, 1)\n    }\n  }\n\n  /**\n   * Check if there is a handler.conversation span that:\n   * 1. Was NOT seen before startTurn()\n   * 2. Has terminal status (ok or error)\n   *\n   * Marks the matched key as consumed so the same handler span cannot\n   * satisfy both the current turn's settle phase AND the next turn's\n   * waitForTurnComplete check (which would otherwise resolve instantly\n   * against stale data instead of waiting for the new turn's handler).\n   */\n  private _hasNewCompletedHandler(): boolean {\n    for (const [key, span] of this.spanMap) {\n      if (\n        span.name === 'handler.conversation' &&\n        !this.seenHandlerKeys.has(key) &&\n        (span.status === 'ok' || span.status === 'error')\n      ) {\n        this.seenHandlerKeys.add(key)\n        return true\n      }\n    }\n    return false\n  }\n\n  /**\n   * Check if a handler.workflow span matches the given condition.\n   */\n  private _matchesWorkflowCondition(name: string, signal: 'entered' | 'completed'): boolean {\n    for (const span of this.spanMap.values()) {\n      if (span.name !== 'handler.workflow') continue\n\n      const d = spanData(span)\n      const wfName = d['workflow.name'] || d['workflowName']\n      if (wfName !== name) continue\n\n      if (signal === 'entered') {\n        // Any status means the workflow was entered\n        return true\n      }\n\n      if (signal === 'completed') {\n        return span.status === 'ok' || span.status === 'error'\n      }\n    }\n    return false\n  }\n}\n"],
-  "mappings": ";AA+BA,SAAS,QAAQ,MAAoB;AACnC,SAAO,GAAG,KAAK,GAAG,KAAK,IAAI,KAAK,GAAG,IAAI;AACzC;AAKA,SAAS,SAAS,MAAqC;AACrD,SAAO,KAAK,QAAQ,OAAO,KAAK,SAAS,WAAY,KAAK,OAAmC,CAAC;AAChG;AAEO,IAAM,eAAN,MAAM,cAAa;AAAA,EAChB;AAAA,EACA;AAAA;AAAA,EAGA,UAAU,oBAAI,IAAkB;AAAA;AAAA,EAGhC,gBAAgB,oBAAI,IAAY;AAAA;AAAA,EAGhC,kBAAkB,oBAAI,IAAY;AAAA;AAAA,EAGlC,UAAuC,CAAC;AAAA;AAAA,EAGxC,kBAA0C;AAAA;AAAA,EAG1C,YAAY;AAAA;AAAA,EAGZ,cAA4B;AAAA;AAAA,EAG5B,gBAAqC;AAAA,EAE7C,YAAY,cAAsB,UAAkC,CAAC,GAAG;AACtE,SAAK,eAAe;AACpB,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,QAAQ,QAAkC;AAC9C,UAAM,gBAAgB,oBAAoB,SAAS,mBAAmB;AACtE,UAAM,iBAAiB,oBAAoB,SAAS,OAAO,iBAAiB,OAAO;AAEnF,UAAM,MAAM,GAAG,KAAK,YAAY,oCAAoC,mBAAmB,aAAa,CAAC,mBAAmB,mBAAmB,cAAc,CAAC;AAE1J,SAAK,kBAAkB,IAAI,gBAAgB;AAE3C,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,SAAS,EAAE,GAAG,KAAK,SAAS,QAAQ,oBAAoB;AAAA,MACxD,QAAQ,KAAK,gBAAgB;AAAA,IAC/B,CAAC;AAED,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,IAAI,MAAM,0BAA0B,SAAS,MAAM,IAAI,SAAS,UAAU,EAAE;AAAA,IACpF;AAEA,QAAI,CAAC,SAAS,MAAM;AAClB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,SAAK,YAAY;AAGjB,SAAK,eAAe,SAAS,IAAI,EAAE,MAAM,CAAC,QAAQ;AAChD,UAAI,KAAK,WAAW;AAClB,aAAK,cAAc,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAAA,MACvE;AAAA,IACF,CAAC;AAKD,UAAM,KAAK,iBAAiB;AAAA,EAC9B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,YAAkB;AAEhB,SAAK,gBAAgB,IAAI,IAAI,KAAK,QAAQ,KAAK,CAAC;AAGhD,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UAAI,KAAK,SAAS,wBAAwB;AACxC,aAAK,gBAAgB,IAAI,GAAG;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAwB,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWpC,MAAM,oBAAoB,MAAkC;AAE1D,QAAI,KAAK,wBAAwB,GAAG;AAClC,aAAO,KAAK,oBAAoB;AAAA,IAClC;AACA,QAAI,KAAK,aAAa,SAAS;AAC7B,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,QAAQ,WAAW,MAAM;AAC7B,aAAK,cAAc,MAAM;AACzB,aAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,cAAM,SAAS,KAAK,cAChB,oBAAoB,KAAK,YAAY,OAAO,KAC5C;AACJ,eAAO,IAAI,MAAM,uCAAuC,KAAK,OAAO,aAAQ,MAAM,EAAE,CAAC;AAAA,MACvF,GAAG,KAAK,OAAO;AAEf,YAAM,UAAU,MAAM;AACpB,qBAAa,KAAK;AAClB,aAAK,cAAc,MAAM;AACzB,eAAO,IAAI,MAAM,6BAA6B,CAAC;AAAA,MACjD;AACA,WAAK,aAAa,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAEnE,YAAM,SAAS,CAAC,UAAgB;AAC9B,YAAI,KAAK,wBAAwB,GAAG;AAClC,uBAAa,KAAK;AAClB,eAAK,cAAc,MAAM;AACzB,eAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,eAAK,oBAAoB,EAAE,KAAK,SAAS,MAAM;AAAA,QACjD;AAAA,MACF;AAEA,WAAK,QAAQ,KAAK,MAAM;AAAA,IAC1B,CAAC;AAAA,EACH;AAAA,EAEA,OAAwB,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAShC,sBAAqC;AAC3C,WAAO,IAAI,QAAc,CAAC,YAAY;AACpC,UAAI,WAAW;AACf,YAAM,OAAO,MAAM;AACjB,YAAI,SAAU;AACd,mBAAW;AACX,qBAAa,OAAO;AACpB,qBAAa,WAAW;AACxB,aAAK,cAAc,YAAY;AAC/B,aAAK,gBAAgB;AACrB,gBAAQ;AAAA,MACV;AAEA,YAAM,UAAU,WAAW,MAAM,cAAa,aAAa;AAE3D,UAAI,cAAc,WAAW,MAAM,cAAa,SAAS;AAEzD,YAAM,eAAe,MAAM;AACzB,YAAI,SAAU;AACd,qBAAa,WAAW;AACxB,sBAAc,WAAW,MAAM,cAAa,SAAS;AAAA,MACvD;AAEA,WAAK,QAAQ,KAAK,YAAY;AAC9B,WAAK,gBAAgB;AAAA,IACvB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,gBAAgB,MAAc,MAA0C;AAE5E,QAAI,KAAK,0BAA0B,MAAM,KAAK,MAAM,GAAG;AACrD;AAAA,IACF;AACA,QAAI,KAAK,aAAa,SAAS;AAC7B,YAAM,IAAI,MAAM,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,cAAc;AAAA,IACpF;AAEA,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,QAAQ,WAAW,MAAM;AAC7B,aAAK,cAAc,MAAM;AACzB,aAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,cAAM,SAAS,KAAK,cAAc,oBAAoB,KAAK,YAAY,OAAO,KAAK;AACnF;AAAA,UACE,IAAI;AAAA,YACF,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,wBAAwB,KAAK,OAAO,aAAQ,MAAM;AAAA,UACxG;AAAA,QACF;AAAA,MACF,GAAG,KAAK,OAAO;AAEf,YAAM,UAAU,MAAM;AACpB,qBAAa,KAAK;AAClB,aAAK,cAAc,MAAM;AACzB,eAAO,IAAI,MAAM,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,cAAc,CAAC;AAAA,MACtF;AACA,WAAK,aAAa,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAEnE,YAAM,SAAS,CAAC,UAAgB;AAC9B,YAAI,KAAK,0BAA0B,MAAM,KAAK,MAAM,GAAG;AACrD,uBAAa,KAAK;AAClB,eAAK,cAAc,MAAM;AACzB,eAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,kBAAQ;AAAA,QACV;AAAA,MACF;AAEA,WAAK,QAAQ,KAAK,MAAM;AAAA,IAC1B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,eAAuB;AACrB,UAAM,SAAiB,CAAC;AACxB,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UAAI,CAAC,KAAK,cAAc,IAAI,GAAG,GAAG;AAChC,eAAO,KAAK,IAAI;AAAA,MAClB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,cAAsB;AACpB,WAAO,MAAM,KAAK,KAAK,QAAQ,OAAO,CAAC;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA,EAKA,aAAmB;AACjB,SAAK,YAAY;AACjB,SAAK,iBAAiB,MAAM;AAC5B,SAAK,kBAAkB;AAGvB,SAAK,gBAAgB;AACrB,SAAK,gBAAgB;AAGrB,SAAK,UAAU,CAAC;AAChB,SAAK,oBAAoB,CAAC;AAAA,EAC5B;AAAA;AAAA;AAAA,EAKQ,mBAAmB;AAAA,EACnB,oBAAuC,CAAC;AAAA,EAEhD,OAAwB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMtC,mBAAkC;AACxC,QAAI,KAAK,kBAAkB;AACzB,aAAO,QAAQ,QAAQ;AAAA,IACzB;AACA,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,UAAI,UAAU;AACd,YAAM,WAAW,MAAM;AACrB,YAAI,QAAS;AACb,kBAAU;AACV,qBAAa,KAAK;AAClB,gBAAQ;AAAA,MACV;AACA,YAAM,QAAQ,WAAW,MAAM;AAC7B,YAAI,QAAS;AACb,kBAAU;AACV,cAAM,MAAM,KAAK,kBAAkB,QAAQ,QAAQ;AACnD,YAAI,QAAQ,GAAI,MAAK,kBAAkB,OAAO,KAAK,CAAC;AACpD;AAAA,UACE,IAAI;AAAA,YACF,oCAAoC,cAAa,mBAAmB;AAAA,UACtE;AAAA,QACF;AAAA,MACF,GAAG,cAAa,mBAAmB;AACnC,WAAK,kBAAkB,KAAK,QAAQ;AAAA,IACtC,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAe,MAAiD;AAC5E,UAAM,UAAU,IAAI,YAAY;AAChC,QAAI,SAAS;AAEb,UAAM,SAAS,KAAK,UAAU;AAE9B,QAAI;AACF,aAAO,KAAK,WAAW;AACrB,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AAEV,kBAAU,QAAQ,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;AAGhD,cAAM,QAAQ,OAAO,MAAM,MAAM;AAEjC,iBAAS,MAAM,IAAI,KAAK;AAExB,mBAAW,QAAQ,OAAO;AACxB,eAAK,eAAe,IAAI;AAAA,QAC1B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,eAAe,OAAqB;AAC1C,UAAM,QAAQ,MAAM,MAAM,IAAI;AAC9B,QAAI,YAAY;AAChB,QAAI,WAAW;AAEf,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,WAAW,QAAQ,GAAG;AAC7B,oBAAY,KAAK,MAAM,SAAS,MAAM,EAAE,KAAK;AAAA,MAC/C,WAAW,KAAK,WAAW,OAAO,GAAG;AACnC,cAAM,QAAQ,KAAK,MAAM,QAAQ,MAAM,EAAE,KAAK;AAC9C,mBAAW,WAAW,GAAG,QAAQ;AAAA,EAAK,KAAK,KAAK;AAAA,MAClD;AAAA,IAEF;AAEA,QAAI,CAAC,aAAa,CAAC,UAAU;AAE3B,UAAI,CAAC,KAAK,oBAAoB,MAAM,SAAS,QAAQ,GAAG;AAAA,MAExD;AACA;AAAA,IACF;AAEA,QAAI;AACF,YAAM,UAAU,KAAK,MAAM,QAAQ;AACnC,WAAK,aAAa,WAAW,OAAO;AAAA,IACtC,QAAQ;AAAA,IAER;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,aAAa,WAAmB,SAAwB;AAC9D,QAAI,cAAc,YAAY;AAC5B,YAAM,OAAO;AACb,YAAM,QAAQ,MAAM,QAAQ,KAAK,KAAK,IAAI,KAAK,QAAQ,CAAC;AACxD,iBAAW,QAAQ,OAAO;AACxB,aAAK,YAAY,IAAI;AAAA,MACvB;AACA,WAAK,mBAAmB;AACxB,iBAAW,WAAW,KAAK,mBAAmB;AAC5C,gBAAQ;AAAA,MACV;AACA,WAAK,oBAAoB,CAAC;AAAA,IAC5B,WAAW,cAAc,UAAU;AACjC,YAAM,OAAO;AACb,UAAI,KAAK,MAAM;AACb,aAAK,YAAY,KAAK,IAAI;AAC1B,aAAK,eAAe,KAAK,IAAI;AAAA,MAC/B;AAAA,IACF;AAAA,EAEF;AAAA;AAAA;AAAA;AAAA,EAKQ,YAAY,MAAkB;AACpC,SAAK,QAAQ,IAAI,QAAQ,IAAI,GAAG,IAAI;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAe,MAAkB;AAEvC,UAAM,UAAU,CAAC,GAAG,KAAK,OAAO;AAChC,eAAW,UAAU,SAAS;AAC5B,aAAO,IAAI;AAAA,IACb;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,cAAc,IAAgC;AACpD,UAAM,MAAM,KAAK,QAAQ,QAAQ,EAAE;AACnC,QAAI,QAAQ,IAAI;AACd,WAAK,QAAQ,OAAO,KAAK,CAAC;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYQ,0BAAmC;AACzC,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UACE,KAAK,SAAS,0BACd,CAAC,KAAK,gBAAgB,IAAI,GAAG,MAC5B,KAAK,WAAW,QAAQ,KAAK,WAAW,UACzC;AACA,aAAK,gBAAgB,IAAI,GAAG;AAC5B,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,0BAA0B,MAAc,QAA0C;AACxF,eAAW,QAAQ,KAAK,QAAQ,OAAO,GAAG;AACxC,UAAI,KAAK,SAAS,mBAAoB;AAEtC,YAAM,IAAI,SAAS,IAAI;AACvB,YAAM,SAAS,EAAE,eAAe,KAAK,EAAE,cAAc;AACrD,UAAI,WAAW,KAAM;AAErB,UAAI,WAAW,WAAW;AAExB,eAAO;AAAA,MACT;AAEA,UAAI,WAAW,aAAa;AAC1B,eAAO,KAAK,WAAW,QAAQ,KAAK,WAAW;AAAA,MACjD;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;",
+  "sources": ["../../analytics/src/errors.ts", "../src/errors.ts", "../src/sse-collector.ts"],
+  "sourcesContent": ["/**\n * Shared base error for ADK packages. Lives in `@botpress/analytics` because\n * it's the dependency-light package every surface already shares (cli, adk,\n * ui) \u2014 so all typed errors can genuinely inherit one base instead of\n * mirroring its shape.\n *\n * The policy lives in docs/ERROR-HANDLING.md. The short version: every error\n * we *intend* to throw should carry a stable `code` (so PostHog error\n * tracking can slice issues by code) and an `expected` flag separating\n * user/environment conditions (bad input, not logged in, network down \u2014 render\n * a friendly message, low alert priority) from internal bugs (invariant\n * violations \u2014 full stack, page someone).\n *\n * The runtime package is out of scope and keeps its own `Errors.AbstractError`\n * hierarchy. The adk typed families (`DependencyError`, `Agent0*Error`) extend\n * this base; new code should start here.\n */\nexport class AdkError<Code extends string = string> extends Error {\n  /**\n   * Marker for cross-module-instance checks \u2014 `instanceof` breaks when a\n   * package is duplicated in the dependency graph.\n   * @internal\n   */\n  public static readonly __IS_ADK_BASE_ERROR = true\n\n  /**\n   * Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses\n   * pin `Code` to a literal union (`class EvalRunnerError extends\n   * AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows\n   * in switches.\n   */\n  readonly code: Code\n  /**\n   * True for user/environment conditions the developer can fix themselves\n   * (validation, auth, connectivity). False for internal bugs. Defaults to\n   * false so unclassified errors surface as bugs until someone triages them.\n   */\n  readonly expected: boolean\n  /** Structured context. Sanitized before any telemetry leaves the machine. */\n  readonly details?: Record<string, unknown>\n  /** Actionable next step shown to the user, e.g. \"Run 'adk login' first.\" */\n  readonly suggestion?: string\n\n  constructor(opts: {\n    code: Code\n    message: string\n    expected?: boolean\n    details?: Record<string, unknown>\n    suggestion?: string\n    cause?: unknown\n  }) {\n    super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined)\n    this.name = this.constructor.name\n    this.code = opts.code\n    this.expected = opts.expected ?? false\n    // Conditional assignment keeps the optional fields absent-not-undefined,\n    // which downstream tsconfigs with exactOptionalPropertyTypes require.\n    if (opts.details !== undefined) {\n      this.details = opts.details\n    }\n    if (opts.suggestion !== undefined) {\n      this.suggestion = opts.suggestion\n    }\n  }\n}\n\n/**\n * Duck-typed guard that survives duplicated module instances in the\n * dependency graph, where `instanceof AdkError` returns false for an error\n * constructed from another copy of this module. No manual prototype walking\n * needed: static properties are inherited through the constructor chain, so\n * a subclass's constructor resolves the marker in one lookup.\n */\nexport function isAdkError(error: unknown): error is AdkError {\n  if (error instanceof AdkError) return true\n  if (!(error instanceof Error)) return false\n  return (error.constructor as unknown as Record<string, unknown> | undefined)?.__IS_ADK_BASE_ERROR === true\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n", "/**\n * SSECollector \u2014 connects to the ADK dev server's SSE trace stream,\n * accumulates spans, and provides promise-based wait methods for\n * turn completion and workflow signals.\n *\n * Uses fetch() + async iteration with manual SSE line parsing\n * (Bun does not have native EventSource).\n */\n\nimport type { Span } from './trace'\nimport { EvalRunnerError } from './errors'\n\ntype SSEFilter = { conversationId: string } | { workflowId: string }\n\ninterface WaitOptions {\n  timeout: number\n  /**\n   * Optional abort signal \u2014 when fired, the wait rejects immediately rather\n   * than burning the full timeout. Used by the runner so Stop unblocks\n   * pending workflow assertions (otherwise a single negative-result wait\n   * can hold the suite hostage for `idleTimeout` per assertion).\n   */\n  abortSignal?: AbortSignal\n}\n\ninterface WorkflowWaitOptions extends WaitOptions {\n  signal: 'entered' | 'completed'\n}\n\n/**\n * Unique key for span deduplication.\n */\nfunction spanKey(span: Span): string {\n  return `${span.id.trace}:${span.id.span}`\n}\n\n/**\n * Type-safe accessor for span.data fields.\n */\nfunction spanData(span: Span): Record<string, unknown> {\n  return span.data && typeof span.data === 'object' ? (span.data as Record<string, unknown>) : {}\n}\n\nexport class SSECollector {\n  private devServerUrl: string\n  private headers: Record<string, string>\n\n  /** All spans accumulated since connect(), keyed by spanKey for dedup. */\n  private spanMap = new Map<string, Span>()\n\n  /** Index into spanMap insertion order marking where the current turn started. */\n  private turnStartKeys = new Set<string>()\n\n  /** Handler span IDs seen before the current turn (for startTurn filtering). */\n  private seenHandlerKeys = new Set<string>()\n\n  /** Callbacks waiting for span updates. */\n  private waiters: Array<(span: Span) => void> = []\n\n  /** AbortController for the active SSE connection. */\n  private abortController: AbortController | null = null\n\n  /** Whether the background loop is running. */\n  private connected = false\n\n  /** Error from the SSE stream if it died unexpectedly. */\n  private streamError: Error | null = null\n\n  /** Set while a settle phase is in flight; called by disconnect() to short-circuit. */\n  private _cancelSettle: (() => void) | null = null\n\n  constructor(devServerUrl: string, headers: Record<string, string> = {}) {\n    this.devServerUrl = devServerUrl\n    this.headers = headers\n  }\n\n  /**\n   * Open a streaming SSE connection filtered by conversationId or workflowId.\n   * Starts a background loop that parses SSE events and accumulates spans.\n   */\n  async connect(filter: SSEFilter): Promise<void> {\n    const attributeName = 'conversationId' in filter ? 'conversationId' : 'workflowId'\n    const attributeValue = 'conversationId' in filter ? filter.conversationId : filter.workflowId\n\n    const url = `${this.devServerUrl}/api/traces/stream?attributeName=${encodeURIComponent(attributeName)}&attributeValue=${encodeURIComponent(attributeValue)}`\n\n    this.abortController = new AbortController()\n\n    const response = await fetch(url, {\n      headers: { ...this.headers, Accept: 'text/event-stream' },\n      signal: this.abortController.signal,\n    })\n\n    if (!response.ok) {\n      throw new EvalRunnerError({\n        code: 'SSE_CONNECT_FAILED',\n        message: `SSE connection failed: ${response.status} ${response.statusText}`,\n        expected: true,\n        suggestion: 'Make sure the dev server is running (`adk dev`) and reachable.',\n      })\n    }\n\n    if (!response.body) {\n      throw new EvalRunnerError({ code: 'SSE_NO_BODY', message: 'SSE response has no body' })\n    }\n\n    this.connected = true\n\n    // Start background parsing loop\n    this._consumeStream(response.body).catch((err) => {\n      if (this.connected) {\n        this.streamError = err instanceof Error ? err : new Error(String(err))\n      }\n    })\n\n    // Wait for the initial snapshot to arrive before returning.\n    // This ensures that connect() resolves only after the snapshot\n    // has been processed so callers can immediately check spans.\n    await this._waitForSnapshot()\n  }\n\n  /**\n   * Mark the beginning of a new turn.\n   * Records which handler.conversation span IDs have been seen so far,\n   * so waitForTurnComplete() only resolves on NEW handler completions.\n   */\n  startTurn(): void {\n    // Snapshot the current set of span keys as the turn boundary\n    this.turnStartKeys = new Set(this.spanMap.keys())\n\n    // Record all handler.conversation spans seen so far\n    for (const [key, span] of this.spanMap) {\n      if (span.name === 'handler.conversation') {\n        this.seenHandlerKeys.add(key)\n      }\n    }\n  }\n\n  private static readonly SETTLE_MS = 100\n\n  /**\n   * Wait for a handler.conversation span to reach terminal status (ok or error).\n   * Only considers spans NOT seen before the last startTurn() call.\n   *\n   * After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)\n   * to ensure all child spans (tool calls, messages, state saves) have arrived.\n   * The runtime's HttpSpanExporter fires POSTs without awaiting them, so child\n   * spans may arrive slightly after the handler close event.\n   */\n  async waitForTurnComplete(opts: WaitOptions): Promise<void> {\n    // Check if a completed handler already exists (e.g. from snapshot)\n    if (this._hasNewCompletedHandler()) {\n      return this._settleAfterHandler()\n    }\n    if (opts.abortSignal?.aborted) {\n      throw new Error('waitForTurnComplete aborted')\n    }\n\n    return new Promise<void>((resolve, reject) => {\n      const timer = setTimeout(() => {\n        this._removeWaiter(waiter)\n        opts.abortSignal?.removeEventListener('abort', onAbort)\n        const reason = this.streamError\n          ? `SSE stream died: ${this.streamError.message}`\n          : 'no handler.conversation span completed'\n        reject(new Error(`waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}`))\n      }, opts.timeout)\n\n      const onAbort = () => {\n        clearTimeout(timer)\n        this._removeWaiter(waiter)\n        reject(new Error('waitForTurnComplete aborted'))\n      }\n      opts.abortSignal?.addEventListener('abort', onAbort, { once: true })\n\n      const waiter = (_span: Span) => {\n        if (this._hasNewCompletedHandler()) {\n          clearTimeout(timer)\n          this._removeWaiter(waiter)\n          opts.abortSignal?.removeEventListener('abort', onAbort)\n          this._settleAfterHandler().then(resolve, reject)\n        }\n      }\n\n      this.waiters.push(waiter)\n    })\n  }\n\n  private static readonly MAX_SETTLE_MS = 2_000\n\n  /**\n   * After the handler span closes, wait for spans to stop arriving.\n   * Resolves when no new spans arrive for SETTLE_MS, or after MAX_SETTLE_MS hard cap.\n   *\n   * Exposes a cancellation hook via _cancelSettle so disconnect() can short-\n   * circuit an in-flight settle phase and avoid leaking timers for up to 2s.\n   */\n  private _settleAfterHandler(): Promise<void> {\n    return new Promise<void>((resolve) => {\n      let resolved = false\n      const done = () => {\n        if (resolved) return\n        resolved = true\n        clearTimeout(hardCap)\n        clearTimeout(settleTimer)\n        this._removeWaiter(settleWaiter)\n        this._cancelSettle = null\n        resolve()\n      }\n\n      const hardCap = setTimeout(done, SSECollector.MAX_SETTLE_MS)\n\n      let settleTimer = setTimeout(done, SSECollector.SETTLE_MS)\n\n      const settleWaiter = () => {\n        if (resolved) return\n        clearTimeout(settleTimer)\n        settleTimer = setTimeout(done, SSECollector.SETTLE_MS)\n      }\n\n      this.waiters.push(settleWaiter)\n      this._cancelSettle = done\n    })\n  }\n\n  /**\n   * Wait for a handler.workflow span with matching workflow name.\n   * - signal 'entered': resolves when the span appears (any status)\n   * - signal 'completed': resolves when the span has status !== 'running'\n   */\n  async waitForWorkflow(name: string, opts: WorkflowWaitOptions): Promise<void> {\n    // Check if condition is already met\n    if (this._matchesWorkflowCondition(name, opts.signal)) {\n      return\n    }\n    if (opts.abortSignal?.aborted) {\n      throw new Error(`waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) aborted`)\n    }\n\n    return new Promise<void>((resolve, reject) => {\n      const timer = setTimeout(() => {\n        this._removeWaiter(waiter)\n        opts.abortSignal?.removeEventListener('abort', onAbort)\n        const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : 'span not found'\n        reject(\n          new Error(\n            `waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) timed out after ${opts.timeout}ms \u2014 ${reason}`\n          )\n        )\n      }, opts.timeout)\n\n      const onAbort = () => {\n        clearTimeout(timer)\n        this._removeWaiter(waiter)\n        reject(new Error(`waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) aborted`))\n      }\n      opts.abortSignal?.addEventListener('abort', onAbort, { once: true })\n\n      const waiter = (_span: Span) => {\n        if (this._matchesWorkflowCondition(name, opts.signal)) {\n          clearTimeout(timer)\n          this._removeWaiter(waiter)\n          opts.abortSignal?.removeEventListener('abort', onAbort)\n          resolve()\n        }\n      }\n\n      this.waiters.push(waiter)\n    })\n  }\n\n  /**\n   * Return spans accumulated since the last startTurn() call.\n   */\n  getTurnSpans(): Span[] {\n    const result: Span[] = []\n    for (const [key, span] of this.spanMap) {\n      if (!this.turnStartKeys.has(key)) {\n        result.push(span)\n      }\n    }\n    return result\n  }\n\n  /**\n   * Return all spans accumulated since connect().\n   */\n  getAllSpans(): Span[] {\n    return Array.from(this.spanMap.values())\n  }\n\n  /**\n   * Close the SSE connection and clean up.\n   */\n  disconnect(): void {\n    this.connected = false\n    this.abortController?.abort()\n    this.abortController = null\n    // Short-circuit an in-flight settle phase so its hardCap / settleTimer\n    // don't fire after disconnect.\n    this._cancelSettle?.()\n    this._cancelSettle = null\n    // Waiters hold setTimeout references that will reject on their own;\n    // clearing the array just prevents further notifications.\n    this.waiters = []\n    this.snapshotResolvers = []\n  }\n\n  // \u2500\u2500 Private \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  /** Whether we have received the initial snapshot. */\n  private snapshotReceived = false\n  private snapshotResolvers: Array<() => void> = []\n\n  private static readonly SNAPSHOT_TIMEOUT_MS = 10_000\n\n  /**\n   * Returns a promise that resolves once the snapshot event has been processed,\n   * or rejects after a timeout to prevent connect() from hanging indefinitely.\n   */\n  private _waitForSnapshot(): Promise<void> {\n    if (this.snapshotReceived) {\n      return Promise.resolve()\n    }\n    return new Promise<void>((resolve, reject) => {\n      let settled = false\n      const resolver = () => {\n        if (settled) return\n        settled = true\n        clearTimeout(timer)\n        resolve()\n      }\n      const timer = setTimeout(() => {\n        if (settled) return\n        settled = true\n        const idx = this.snapshotResolvers.indexOf(resolver)\n        if (idx !== -1) this.snapshotResolvers.splice(idx, 1)\n        reject(\n          new Error(\n            `SSE snapshot not received within ${SSECollector.SNAPSHOT_TIMEOUT_MS}ms \u2014 is the dev server healthy?`\n          )\n        )\n      }, SSECollector.SNAPSHOT_TIMEOUT_MS)\n      this.snapshotResolvers.push(resolver)\n    })\n  }\n\n  /**\n   * Consume the ReadableStream body, parse SSE lines, and dispatch events.\n   */\n  private async _consumeStream(body: ReadableStream<Uint8Array>): Promise<void> {\n    const decoder = new TextDecoder()\n    let buffer = ''\n\n    const reader = body.getReader()\n\n    try {\n      while (this.connected) {\n        const { done, value } = await reader.read()\n        if (done) break\n\n        buffer += decoder.decode(value, { stream: true })\n\n        // Split on double-newline boundaries to get complete SSE events\n        const parts = buffer.split('\\n\\n')\n        // The last part may be incomplete \u2014 keep it in the buffer\n        buffer = parts.pop() ?? ''\n\n        for (const part of parts) {\n          this._parseSSEBlock(part)\n        }\n      }\n    } finally {\n      reader.releaseLock()\n    }\n  }\n\n  /**\n   * Parse a single SSE event block (text between \\n\\n boundaries).\n   * Extracts event name and data payload.\n   */\n  private _parseSSEBlock(block: string): void {\n    const lines = block.split('\\n')\n    let eventName = ''\n    let dataLine = ''\n\n    for (const line of lines) {\n      if (line.startsWith('event:')) {\n        eventName = line.slice('event:'.length).trim()\n      } else if (line.startsWith('data:')) {\n        const value = line.slice('data:'.length).trim()\n        dataLine = dataLine ? `${dataLine}\\n${value}` : value\n      }\n      // id: and retry: lines are ignored\n    }\n\n    if (!eventName || !dataLine) {\n      // Could be a retry: line or empty block \u2014 skip\n      if (!this.snapshotReceived && block.includes('retry:')) {\n        // The retry line comes before the snapshot; don't resolve yet\n      }\n      return\n    }\n\n    try {\n      const payload = JSON.parse(dataLine)\n      this._handleEvent(eventName, payload)\n    } catch {\n      // Malformed JSON \u2014 skip\n    }\n  }\n\n  /**\n   * Dispatch a parsed SSE event to the appropriate handler.\n   */\n  private _handleEvent(eventName: string, payload: unknown): void {\n    if (eventName === 'snapshot') {\n      const data = payload as { spans: Span[] }\n      const spans = Array.isArray(data.spans) ? data.spans : []\n      for (const span of spans) {\n        this._upsertSpan(span)\n      }\n      this.snapshotReceived = true\n      for (const resolve of this.snapshotResolvers) {\n        resolve()\n      }\n      this.snapshotResolvers = []\n    } else if (eventName === 'update') {\n      const data = payload as { span: Span }\n      if (data.span) {\n        this._upsertSpan(data.span)\n        this._notifyWaiters(data.span)\n      }\n    }\n    // keepalive events are intentionally ignored\n  }\n\n  /**\n   * Insert or replace a span in the map (dedup by trace+span ID).\n   */\n  private _upsertSpan(span: Span): void {\n    this.spanMap.set(spanKey(span), span)\n  }\n\n  /**\n   * Notify all waiters that a new span was received.\n   */\n  private _notifyWaiters(span: Span): void {\n    // Copy the array since waiters may remove themselves during iteration\n    const current = [...this.waiters]\n    for (const waiter of current) {\n      waiter(span)\n    }\n  }\n\n  /**\n   * Remove a waiter callback from the list.\n   */\n  private _removeWaiter(fn: (span: Span) => void): void {\n    const idx = this.waiters.indexOf(fn)\n    if (idx !== -1) {\n      this.waiters.splice(idx, 1)\n    }\n  }\n\n  /**\n   * Check if there is a handler.conversation span that:\n   * 1. Was NOT seen before startTurn()\n   * 2. Has terminal status (ok or error)\n   *\n   * Marks the matched key as consumed so the same handler span cannot\n   * satisfy both the current turn's settle phase AND the next turn's\n   * waitForTurnComplete check (which would otherwise resolve instantly\n   * against stale data instead of waiting for the new turn's handler).\n   */\n  private _hasNewCompletedHandler(): boolean {\n    for (const [key, span] of this.spanMap) {\n      if (\n        span.name === 'handler.conversation' &&\n        !this.seenHandlerKeys.has(key) &&\n        (span.status === 'ok' || span.status === 'error')\n      ) {\n        this.seenHandlerKeys.add(key)\n        return true\n      }\n    }\n    return false\n  }\n\n  /**\n   * Check if a handler.workflow span matches the given condition.\n   */\n  private _matchesWorkflowCondition(name: string, signal: 'entered' | 'completed'): boolean {\n    for (const span of this.spanMap.values()) {\n      if (span.name !== 'handler.workflow') continue\n\n      const d = spanData(span)\n      const wfName = d['workflow.name'] || d['workflowName']\n      if (wfName !== name) continue\n\n      if (signal === 'entered') {\n        // Any status means the workflow was entered\n        return true\n      }\n\n      if (signal === 'completed') {\n        return span.status === 'ok' || span.status === 'error'\n      }\n    }\n    return false\n  }\n}\n"],
+  "mappings": ";AAiBO,IAAM,WAAN,cAAqD,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMhE,OAAuB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQpC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA;AAAA;AAAA,EAEA;AAAA;AAAA,EAEA;AAAA,EAET,YAAY,MAOT;AACD,UAAM,KAAK,SAAS,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,MAAS;AAChF,SAAK,OAAO,KAAK,YAAY;AAC7B,SAAK,OAAO,KAAK;AACjB,SAAK,WAAW,KAAK,YAAY;AAGjC,QAAI,KAAK,YAAY,QAAW;AAC9B,WAAK,UAAU,KAAK;AAAA,IACtB;AACA,QAAI,KAAK,eAAe,QAAW;AACjC,WAAK,aAAa,KAAK;AAAA,IACzB;AAAA,EACF;AACF;;;ACpCO,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;ACI9D,SAAS,QAAQ,MAAoB;AACnC,SAAO,GAAG,KAAK,GAAG,KAAK,IAAI,KAAK,GAAG,IAAI;AACzC;AAKA,SAAS,SAAS,MAAqC;AACrD,SAAO,KAAK,QAAQ,OAAO,KAAK,SAAS,WAAY,KAAK,OAAmC,CAAC;AAChG;AAEO,IAAM,eAAN,MAAM,cAAa;AAAA,EAChB;AAAA,EACA;AAAA;AAAA,EAGA,UAAU,oBAAI,IAAkB;AAAA;AAAA,EAGhC,gBAAgB,oBAAI,IAAY;AAAA;AAAA,EAGhC,kBAAkB,oBAAI,IAAY;AAAA;AAAA,EAGlC,UAAuC,CAAC;AAAA;AAAA,EAGxC,kBAA0C;AAAA;AAAA,EAG1C,YAAY;AAAA;AAAA,EAGZ,cAA4B;AAAA;AAAA,EAG5B,gBAAqC;AAAA,EAE7C,YAAY,cAAsB,UAAkC,CAAC,GAAG;AACtE,SAAK,eAAe;AACpB,SAAK,UAAU;AAAA,EACjB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,QAAQ,QAAkC;AAC9C,UAAM,gBAAgB,oBAAoB,SAAS,mBAAmB;AACtE,UAAM,iBAAiB,oBAAoB,SAAS,OAAO,iBAAiB,OAAO;AAEnF,UAAM,MAAM,GAAG,KAAK,YAAY,oCAAoC,mBAAmB,aAAa,CAAC,mBAAmB,mBAAmB,cAAc,CAAC;AAE1J,SAAK,kBAAkB,IAAI,gBAAgB;AAE3C,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,SAAS,EAAE,GAAG,KAAK,SAAS,QAAQ,oBAAoB;AAAA,MACxD,QAAQ,KAAK,gBAAgB;AAAA,IAC/B,CAAC;AAED,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS,0BAA0B,SAAS,MAAM,IAAI,SAAS,UAAU;AAAA,QACzE,UAAU;AAAA,QACV,YAAY;AAAA,MACd,CAAC;AAAA,IACH;AAEA,QAAI,CAAC,SAAS,MAAM;AAClB,YAAM,IAAI,gBAAgB,EAAE,MAAM,eAAe,SAAS,2BAA2B,CAAC;AAAA,IACxF;AAEA,SAAK,YAAY;AAGjB,SAAK,eAAe,SAAS,IAAI,EAAE,MAAM,CAAC,QAAQ;AAChD,UAAI,KAAK,WAAW;AAClB,aAAK,cAAc,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC;AAAA,MACvE;AAAA,IACF,CAAC;AAKD,UAAM,KAAK,iBAAiB;AAAA,EAC9B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,YAAkB;AAEhB,SAAK,gBAAgB,IAAI,IAAI,KAAK,QAAQ,KAAK,CAAC;AAGhD,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UAAI,KAAK,SAAS,wBAAwB;AACxC,aAAK,gBAAgB,IAAI,GAAG;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAwB,YAAY;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAWpC,MAAM,oBAAoB,MAAkC;AAE1D,QAAI,KAAK,wBAAwB,GAAG;AAClC,aAAO,KAAK,oBAAoB;AAAA,IAClC;AACA,QAAI,KAAK,aAAa,SAAS;AAC7B,YAAM,IAAI,MAAM,6BAA6B;AAAA,IAC/C;AAEA,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,QAAQ,WAAW,MAAM;AAC7B,aAAK,cAAc,MAAM;AACzB,aAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,cAAM,SAAS,KAAK,cAChB,oBAAoB,KAAK,YAAY,OAAO,KAC5C;AACJ,eAAO,IAAI,MAAM,uCAAuC,KAAK,OAAO,aAAQ,MAAM,EAAE,CAAC;AAAA,MACvF,GAAG,KAAK,OAAO;AAEf,YAAM,UAAU,MAAM;AACpB,qBAAa,KAAK;AAClB,aAAK,cAAc,MAAM;AACzB,eAAO,IAAI,MAAM,6BAA6B,CAAC;AAAA,MACjD;AACA,WAAK,aAAa,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAEnE,YAAM,SAAS,CAAC,UAAgB;AAC9B,YAAI,KAAK,wBAAwB,GAAG;AAClC,uBAAa,KAAK;AAClB,eAAK,cAAc,MAAM;AACzB,eAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,eAAK,oBAAoB,EAAE,KAAK,SAAS,MAAM;AAAA,QACjD;AAAA,MACF;AAEA,WAAK,QAAQ,KAAK,MAAM;AAAA,IAC1B,CAAC;AAAA,EACH;AAAA,EAEA,OAAwB,gBAAgB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAShC,sBAAqC;AAC3C,WAAO,IAAI,QAAc,CAAC,YAAY;AACpC,UAAI,WAAW;AACf,YAAM,OAAO,MAAM;AACjB,YAAI,SAAU;AACd,mBAAW;AACX,qBAAa,OAAO;AACpB,qBAAa,WAAW;AACxB,aAAK,cAAc,YAAY;AAC/B,aAAK,gBAAgB;AACrB,gBAAQ;AAAA,MACV;AAEA,YAAM,UAAU,WAAW,MAAM,cAAa,aAAa;AAE3D,UAAI,cAAc,WAAW,MAAM,cAAa,SAAS;AAEzD,YAAM,eAAe,MAAM;AACzB,YAAI,SAAU;AACd,qBAAa,WAAW;AACxB,sBAAc,WAAW,MAAM,cAAa,SAAS;AAAA,MACvD;AAEA,WAAK,QAAQ,KAAK,YAAY;AAC9B,WAAK,gBAAgB;AAAA,IACvB,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAM,gBAAgB,MAAc,MAA0C;AAE5E,QAAI,KAAK,0BAA0B,MAAM,KAAK,MAAM,GAAG;AACrD;AAAA,IACF;AACA,QAAI,KAAK,aAAa,SAAS;AAC7B,YAAM,IAAI,MAAM,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,cAAc;AAAA,IACpF;AAEA,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,YAAM,QAAQ,WAAW,MAAM;AAC7B,aAAK,cAAc,MAAM;AACzB,aAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,cAAM,SAAS,KAAK,cAAc,oBAAoB,KAAK,YAAY,OAAO,KAAK;AACnF;AAAA,UACE,IAAI;AAAA,YACF,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,wBAAwB,KAAK,OAAO,aAAQ,MAAM;AAAA,UACxG;AAAA,QACF;AAAA,MACF,GAAG,KAAK,OAAO;AAEf,YAAM,UAAU,MAAM;AACpB,qBAAa,KAAK;AAClB,aAAK,cAAc,MAAM;AACzB,eAAO,IAAI,MAAM,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,cAAc,CAAC;AAAA,MACtF;AACA,WAAK,aAAa,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAEnE,YAAM,SAAS,CAAC,UAAgB;AAC9B,YAAI,KAAK,0BAA0B,MAAM,KAAK,MAAM,GAAG;AACrD,uBAAa,KAAK;AAClB,eAAK,cAAc,MAAM;AACzB,eAAK,aAAa,oBAAoB,SAAS,OAAO;AACtD,kBAAQ;AAAA,QACV;AAAA,MACF;AAEA,WAAK,QAAQ,KAAK,MAAM;AAAA,IAC1B,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,eAAuB;AACrB,UAAM,SAAiB,CAAC;AACxB,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UAAI,CAAC,KAAK,cAAc,IAAI,GAAG,GAAG;AAChC,eAAO,KAAK,IAAI;AAAA,MAClB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,cAAsB;AACpB,WAAO,MAAM,KAAK,KAAK,QAAQ,OAAO,CAAC;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA,EAKA,aAAmB;AACjB,SAAK,YAAY;AACjB,SAAK,iBAAiB,MAAM;AAC5B,SAAK,kBAAkB;AAGvB,SAAK,gBAAgB;AACrB,SAAK,gBAAgB;AAGrB,SAAK,UAAU,CAAC;AAChB,SAAK,oBAAoB,CAAC;AAAA,EAC5B;AAAA;AAAA;AAAA,EAKQ,mBAAmB;AAAA,EACnB,oBAAuC,CAAC;AAAA,EAEhD,OAAwB,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMtC,mBAAkC;AACxC,QAAI,KAAK,kBAAkB;AACzB,aAAO,QAAQ,QAAQ;AAAA,IACzB;AACA,WAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,UAAI,UAAU;AACd,YAAM,WAAW,MAAM;AACrB,YAAI,QAAS;AACb,kBAAU;AACV,qBAAa,KAAK;AAClB,gBAAQ;AAAA,MACV;AACA,YAAM,QAAQ,WAAW,MAAM;AAC7B,YAAI,QAAS;AACb,kBAAU;AACV,cAAM,MAAM,KAAK,kBAAkB,QAAQ,QAAQ;AACnD,YAAI,QAAQ,GAAI,MAAK,kBAAkB,OAAO,KAAK,CAAC;AACpD;AAAA,UACE,IAAI;AAAA,YACF,oCAAoC,cAAa,mBAAmB;AAAA,UACtE;AAAA,QACF;AAAA,MACF,GAAG,cAAa,mBAAmB;AACnC,WAAK,kBAAkB,KAAK,QAAQ;AAAA,IACtC,CAAC;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,eAAe,MAAiD;AAC5E,UAAM,UAAU,IAAI,YAAY;AAChC,QAAI,SAAS;AAEb,UAAM,SAAS,KAAK,UAAU;AAE9B,QAAI;AACF,aAAO,KAAK,WAAW;AACrB,cAAM,EAAE,MAAM,MAAM,IAAI,MAAM,OAAO,KAAK;AAC1C,YAAI,KAAM;AAEV,kBAAU,QAAQ,OAAO,OAAO,EAAE,QAAQ,KAAK,CAAC;AAGhD,cAAM,QAAQ,OAAO,MAAM,MAAM;AAEjC,iBAAS,MAAM,IAAI,KAAK;AAExB,mBAAW,QAAQ,OAAO;AACxB,eAAK,eAAe,IAAI;AAAA,QAC1B;AAAA,MACF;AAAA,IACF,UAAE;AACA,aAAO,YAAY;AAAA,IACrB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,eAAe,OAAqB;AAC1C,UAAM,QAAQ,MAAM,MAAM,IAAI;AAC9B,QAAI,YAAY;AAChB,QAAI,WAAW;AAEf,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,WAAW,QAAQ,GAAG;AAC7B,oBAAY,KAAK,MAAM,SAAS,MAAM,EAAE,KAAK;AAAA,MAC/C,WAAW,KAAK,WAAW,OAAO,GAAG;AACnC,cAAM,QAAQ,KAAK,MAAM,QAAQ,MAAM,EAAE,KAAK;AAC9C,mBAAW,WAAW,GAAG,QAAQ;AAAA,EAAK,KAAK,KAAK;AAAA,MAClD;AAAA,IAEF;AAEA,QAAI,CAAC,aAAa,CAAC,UAAU;AAE3B,UAAI,CAAC,KAAK,oBAAoB,MAAM,SAAS,QAAQ,GAAG;AAAA,MAExD;AACA;AAAA,IACF;AAEA,QAAI;AACF,YAAM,UAAU,KAAK,MAAM,QAAQ;AACnC,WAAK,aAAa,WAAW,OAAO;AAAA,IACtC,QAAQ;AAAA,IAER;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,aAAa,WAAmB,SAAwB;AAC9D,QAAI,cAAc,YAAY;AAC5B,YAAM,OAAO;AACb,YAAM,QAAQ,MAAM,QAAQ,KAAK,KAAK,IAAI,KAAK,QAAQ,CAAC;AACxD,iBAAW,QAAQ,OAAO;AACxB,aAAK,YAAY,IAAI;AAAA,MACvB;AACA,WAAK,mBAAmB;AACxB,iBAAW,WAAW,KAAK,mBAAmB;AAC5C,gBAAQ;AAAA,MACV;AACA,WAAK,oBAAoB,CAAC;AAAA,IAC5B,WAAW,cAAc,UAAU;AACjC,YAAM,OAAO;AACb,UAAI,KAAK,MAAM;AACb,aAAK,YAAY,KAAK,IAAI;AAC1B,aAAK,eAAe,KAAK,IAAI;AAAA,MAC/B;AAAA,IACF;AAAA,EAEF;AAAA;AAAA;AAAA;AAAA,EAKQ,YAAY,MAAkB;AACpC,SAAK,QAAQ,IAAI,QAAQ,IAAI,GAAG,IAAI;AAAA,EACtC;AAAA;AAAA;AAAA;AAAA,EAKQ,eAAe,MAAkB;AAEvC,UAAM,UAAU,CAAC,GAAG,KAAK,OAAO;AAChC,eAAW,UAAU,SAAS;AAC5B,aAAO,IAAI;AAAA,IACb;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,cAAc,IAAgC;AACpD,UAAM,MAAM,KAAK,QAAQ,QAAQ,EAAE;AACnC,QAAI,QAAQ,IAAI;AACd,WAAK,QAAQ,OAAO,KAAK,CAAC;AAAA,IAC5B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYQ,0BAAmC;AACzC,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UACE,KAAK,SAAS,0BACd,CAAC,KAAK,gBAAgB,IAAI,GAAG,MAC5B,KAAK,WAAW,QAAQ,KAAK,WAAW,UACzC;AACA,aAAK,gBAAgB,IAAI,GAAG;AAC5B,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,0BAA0B,MAAc,QAA0C;AACxF,eAAW,QAAQ,KAAK,QAAQ,OAAO,GAAG;AACxC,UAAI,KAAK,SAAS,mBAAoB;AAEtC,YAAM,IAAI,SAAS,IAAI;AACvB,YAAM,SAAS,EAAE,eAAe,KAAK,EAAE,cAAc;AACrD,UAAI,WAAW,KAAM;AAErB,UAAI,WAAW,WAAW;AAExB,eAAO;AAAA,MACT;AAEA,UAAI,WAAW,aAAa;AAC1B,eAAO,KAAK,WAAW,QAAQ,KAAK,WAAW;AAAA,MACjD;AAAA,IACF;AACA,WAAO;AAAA,EACT;AACF;",
   "names": []
 }

package/dist/transformer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transformer.d.ts","sourceRoot":"","sources":["../src/transformer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,KAAK,EAA2B,QAAQ,EAAgB,MAAM,SAAS,CAAA;AA4B9E;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,CAQtD"}
+{"version":3,"file":"transformer.d.ts","sourceRoot":"","sources":["../src/transformer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,KAAK,EAA2B,QAAQ,EAAgB,MAAM,SAAS,CAAA;AA8B9E;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,CAQtD"}

package/dist/transformer.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
   "sources": ["../src/transformer.ts"],
-  "sourcesContent": ["/**\n * Transformer: projects raw trace spans into grader-friendly TurnData.\n * Pure function \u2014 no side effects, no network calls.\n */\n\nimport type { Span } from './trace'\nimport type { StateMutation, ToolCall, TurnData, WorkflowSpan } from './types'\n\n/** Safe accessor for span.data as a string-keyed record. */\nfunction data(span: Span): Record<string, unknown> {\n  return (span.data && typeof span.data === 'object' ? span.data : {}) as Record<string, unknown>\n}\n\n/** Attempt to parse a JSON string. Returns parsed result, raw value on parse failure, or fallback if value is nullish. */\nfunction tryParseJson(value: unknown): unknown {\n  if (value == null) return undefined\n  if (typeof value !== 'string') return value\n  try {\n    return JSON.parse(value)\n  } catch {\n    return value\n  }\n}\n\n/** Parse JSON with a strict fallback \u2014 returns fallback on parse failure instead of the raw string. */\nfunction parseJsonOrFallback<T>(value: unknown, fallback: T): T {\n  if (typeof value !== 'string') return fallback\n  try {\n    return JSON.parse(value) as T\n  } catch {\n    return fallback\n  }\n}\n\n/**\n * Transform raw trace spans into structured TurnData for graders.\n *\n * Extraction logic:\n * - messages: from `chat.sendMessage` spans, ordered by startedAt\n * - toolCalls: from `autonomous.tool` spans with terminal status, deduped by span id, ordered by endedAt\n * - stateMutations: from `state.save` spans where changed_keys has length > 0\n * - workflowSpans: from `handler.workflow` spans\n * - handler: from `handler.conversation` or `handler.workflow` span with terminal status\n */\nexport function transformSpans(spans: Span[]): TurnData {\n  return {\n    messages: extractMessages(spans),\n    toolCalls: extractToolCalls(spans),\n    stateMutations: extractStateMutations(spans),\n    workflowSpans: extractWorkflowSpans(spans),\n    ...extractHandler(spans),\n  }\n}\n\nfunction extractMessages(spans: Span[]): string[] {\n  return spans\n    .filter((s) => s.name === 'chat.sendMessage' && data(s)['message.preview'])\n    .sort((a, b) => a.timing.startedAt - b.timing.startedAt)\n    .map((s) => String(data(s)['message.preview']))\n}\n\nfunction extractToolCalls(spans: Span[]): ToolCall[] {\n  const toolSpans = spans.filter(\n    (s) => s.name === 'autonomous.tool' && s.status !== 'running' && data(s)['autonomous.tool.name']\n  )\n\n  // Deduplicate by span id\n  const seen = new Set<string>()\n  const unique = toolSpans.filter((s) => {\n    if (seen.has(s.id.span)) return false\n    seen.add(s.id.span)\n    return true\n  })\n\n  return unique\n    .sort((a, b) => (a.timing.endedAt ?? Infinity) - (b.timing.endedAt ?? Infinity))\n    .map((s) => {\n      const d = data(s)\n      const input = parseJsonOrFallback<Record<string, unknown>>(d['autonomous.tool.input'], {})\n      return {\n        name: String(d['autonomous.tool.name']),\n        input,\n        output: String(d['autonomous.tool.output'] ?? ''),\n        status: String(d['autonomous.tool.status'] ?? 'unknown'),\n      }\n    })\n}\n\nfunction extractStateMutations(spans: Span[]): StateMutation[] {\n  return spans\n    .filter((s) => {\n      if (s.name !== 'state.save') return false\n      const d = data(s)\n      const changedKeys = d['state.changed_keys']\n      return Array.isArray(changedKeys) && changedKeys.length > 0\n    })\n    .map((s) => {\n      const d = data(s)\n      return {\n        type: String(d['type'] ?? 'unknown'),\n        changedKeys: (d['state.changed_keys'] as string[]) ?? [],\n        previous: tryParseJson(d['state.previous_value']),\n        current: tryParseJson(d['state.value']),\n        swappedToFile: d['swapped_to_file'] === true,\n      }\n    })\n}\n\nfunction extractWorkflowSpans(spans: Span[]): WorkflowSpan[] {\n  return spans\n    .filter((s) => s.name === 'handler.workflow')\n    .map((s) => {\n      const d = data(s)\n      const result: WorkflowSpan = {\n        name: String(d['workflow.name'] || d['workflowName'] || ''),\n        status: s.status as WorkflowSpan['status'],\n      }\n      const statusFinal = d['workflow.status.final']\n      if (statusFinal) {\n        result.statusFinal = String(statusFinal)\n      }\n      return result\n    })\n}\n\nfunction extractHandler(spans: Span[]): { handlerDuration: number; handlerStatus: 'ok' | 'error' } {\n  const handler = spans.find(\n    (s) => (s.name === 'handler.conversation' || s.name === 'handler.workflow') && s.status !== 'running'\n  )\n\n  if (!handler) {\n    return { handlerDuration: 0, handlerStatus: 'ok' }\n  }\n\n  return {\n    handlerDuration: handler.timing.duration ?? 0,\n    handlerStatus: handler.status === 'error' ? 'error' : 'ok',\n  }\n}\n"],
-  "mappings": ";AASA,SAAS,KAAK,MAAqC;AACjD,SAAQ,KAAK,QAAQ,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO,CAAC;AACpE;AAGA,SAAS,aAAa,OAAyB;AAC7C,MAAI,SAAS,KAAM,QAAO;AAC1B,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI;AACF,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAGA,SAAS,oBAAuB,OAAgB,UAAgB;AAC9D,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI;AACF,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAYO,SAAS,eAAe,OAAyB;AACtD,SAAO;AAAA,IACL,UAAU,gBAAgB,KAAK;AAAA,IAC/B,WAAW,iBAAiB,KAAK;AAAA,IACjC,gBAAgB,sBAAsB,KAAK;AAAA,IAC3C,eAAe,qBAAqB,KAAK;AAAA,IACzC,GAAG,eAAe,KAAK;AAAA,EACzB;AACF;AAEA,SAAS,gBAAgB,OAAyB;AAChD,SAAO,MACJ,OAAO,CAAC,MAAM,EAAE,SAAS,sBAAsB,KAAK,CAAC,EAAE,iBAAiB,CAAC,EACzE,KAAK,CAAC,GAAG,MAAM,EAAE,OAAO,YAAY,EAAE,OAAO,SAAS,EACtD,IAAI,CAAC,MAAM,OAAO,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;AAClD;AAEA,SAAS,iBAAiB,OAA2B;AACnD,QAAM,YAAY,MAAM;AAAA,IACtB,CAAC,MAAM,EAAE,SAAS,qBAAqB,EAAE,WAAW,aAAa,KAAK,CAAC,EAAE,sBAAsB;AAAA,EACjG;AAGA,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,SAAS,UAAU,OAAO,CAAC,MAAM;AACrC,QAAI,KAAK,IAAI,EAAE,GAAG,IAAI,EAAG,QAAO;AAChC,SAAK,IAAI,EAAE,GAAG,IAAI;AAClB,WAAO;AAAA,EACT,CAAC;AAED,SAAO,OACJ,KAAK,CAAC,GAAG,OAAO,EAAE,OAAO,WAAW,aAAa,EAAE,OAAO,WAAW,SAAS,EAC9E,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,KAAK,CAAC;AAChB,UAAM,QAAQ,oBAA6C,EAAE,uBAAuB,GAAG,CAAC,CAAC;AACzF,WAAO;AAAA,MACL,MAAM,OAAO,EAAE,sBAAsB,CAAC;AAAA,MACtC;AAAA,MACA,QAAQ,OAAO,EAAE,wBAAwB,KAAK,EAAE;AAAA,MAChD,QAAQ,OAAO,EAAE,wBAAwB,KAAK,SAAS;AAAA,IACzD;AAAA,EACF,CAAC;AACL;AAEA,SAAS,sBAAsB,OAAgC;AAC7D,SAAO,MACJ,OAAO,CAAC,MAAM;AACb,QAAI,EAAE,SAAS,aAAc,QAAO;AACpC,UAAM,IAAI,KAAK,CAAC;AAChB,UAAM,cAAc,EAAE,oBAAoB;AAC1C,WAAO,MAAM,QAAQ,WAAW,KAAK,YAAY,SAAS;AAAA,EAC5D,CAAC,EACA,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,KAAK,CAAC;AAChB,WAAO;AAAA,MACL,MAAM,OAAO,EAAE,MAAM,KAAK,SAAS;AAAA,MACnC,aAAc,EAAE,oBAAoB,KAAkB,CAAC;AAAA,MACvD,UAAU,aAAa,EAAE,sBAAsB,CAAC;AAAA,MAChD,SAAS,aAAa,EAAE,aAAa,CAAC;AAAA,MACtC,eAAe,EAAE,iBAAiB,MAAM;AAAA,IAC1C;AAAA,EACF,CAAC;AACL;AAEA,SAAS,qBAAqB,OAA+B;AAC3D,SAAO,MACJ,OAAO,CAAC,MAAM,EAAE,SAAS,kBAAkB,EAC3C,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,KAAK,CAAC;AAChB,UAAM,SAAuB;AAAA,MAC3B,MAAM,OAAO,EAAE,eAAe,KAAK,EAAE,cAAc,KAAK,EAAE;AAAA,MAC1D,QAAQ,EAAE;AAAA,IACZ;AACA,UAAM,cAAc,EAAE,uBAAuB;AAC7C,QAAI,aAAa;AACf,aAAO,cAAc,OAAO,WAAW;AAAA,IACzC;AACA,WAAO;AAAA,EACT,CAAC;AACL;AAEA,SAAS,eAAe,OAA2E;AACjG,QAAM,UAAU,MAAM;AAAA,IACpB,CAAC,OAAO,EAAE,SAAS,0BAA0B,EAAE,SAAS,uBAAuB,EAAE,WAAW;AAAA,EAC9F;AAEA,MAAI,CAAC,SAAS;AACZ,WAAO,EAAE,iBAAiB,GAAG,eAAe,KAAK;AAAA,EACnD;AAEA,SAAO;AAAA,IACL,iBAAiB,QAAQ,OAAO,YAAY;AAAA,IAC5C,eAAe,QAAQ,WAAW,UAAU,UAAU;AAAA,EACxD;AACF;",
+  "sourcesContent": ["/**\n * Transformer: projects raw trace spans into grader-friendly TurnData.\n * Pure function \u2014 no side effects, no network calls.\n */\n\nimport type { Span } from './trace'\nimport type { StateMutation, ToolCall, TurnData, WorkflowSpan } from './types'\n\n/** Safe accessor for span.data as a string-keyed record. */\nfunction data(span: Span): Record<string, unknown> {\n  return (span.data && typeof span.data === 'object' ? span.data : {}) as Record<string, unknown>\n}\n\n/** Attempt to parse a JSON string. Returns parsed result, raw value on parse failure, or fallback if value is nullish. */\nfunction tryParseJson(value: unknown): unknown {\n  if (value == null) return undefined\n  if (typeof value !== 'string') return value\n  try {\n    return JSON.parse(value)\n  } catch {\n    // Not JSON \u2014 fall back to the raw value, per this function's contract\n    return value\n  }\n}\n\n/** Parse JSON with a strict fallback \u2014 returns fallback on parse failure instead of the raw string. */\nfunction parseJsonOrFallback<T>(value: unknown, fallback: T): T {\n  if (typeof value !== 'string') return fallback\n  try {\n    return JSON.parse(value) as T\n  } catch {\n    // Not JSON \u2014 return the caller's fallback, per this function's contract\n    return fallback\n  }\n}\n\n/**\n * Transform raw trace spans into structured TurnData for graders.\n *\n * Extraction logic:\n * - messages: from `chat.sendMessage` spans, ordered by startedAt\n * - toolCalls: from `autonomous.tool` spans with terminal status, deduped by span id, ordered by endedAt\n * - stateMutations: from `state.save` spans where changed_keys has length > 0\n * - workflowSpans: from `handler.workflow` spans\n * - handler: from `handler.conversation` or `handler.workflow` span with terminal status\n */\nexport function transformSpans(spans: Span[]): TurnData {\n  return {\n    messages: extractMessages(spans),\n    toolCalls: extractToolCalls(spans),\n    stateMutations: extractStateMutations(spans),\n    workflowSpans: extractWorkflowSpans(spans),\n    ...extractHandler(spans),\n  }\n}\n\nfunction extractMessages(spans: Span[]): string[] {\n  return spans\n    .filter((s) => s.name === 'chat.sendMessage' && data(s)['message.preview'])\n    .sort((a, b) => a.timing.startedAt - b.timing.startedAt)\n    .map((s) => String(data(s)['message.preview']))\n}\n\nfunction extractToolCalls(spans: Span[]): ToolCall[] {\n  const toolSpans = spans.filter(\n    (s) => s.name === 'autonomous.tool' && s.status !== 'running' && data(s)['autonomous.tool.name']\n  )\n\n  // Deduplicate by span id\n  const seen = new Set<string>()\n  const unique = toolSpans.filter((s) => {\n    if (seen.has(s.id.span)) return false\n    seen.add(s.id.span)\n    return true\n  })\n\n  return unique\n    .sort((a, b) => (a.timing.endedAt ?? Infinity) - (b.timing.endedAt ?? Infinity))\n    .map((s) => {\n      const d = data(s)\n      const input = parseJsonOrFallback<Record<string, unknown>>(d['autonomous.tool.input'], {})\n      return {\n        name: String(d['autonomous.tool.name']),\n        input,\n        output: String(d['autonomous.tool.output'] ?? ''),\n        status: String(d['autonomous.tool.status'] ?? 'unknown'),\n      }\n    })\n}\n\nfunction extractStateMutations(spans: Span[]): StateMutation[] {\n  return spans\n    .filter((s) => {\n      if (s.name !== 'state.save') return false\n      const d = data(s)\n      const changedKeys = d['state.changed_keys']\n      return Array.isArray(changedKeys) && changedKeys.length > 0\n    })\n    .map((s) => {\n      const d = data(s)\n      return {\n        type: String(d['type'] ?? 'unknown'),\n        changedKeys: (d['state.changed_keys'] as string[]) ?? [],\n        previous: tryParseJson(d['state.previous_value']),\n        current: tryParseJson(d['state.value']),\n        swappedToFile: d['swapped_to_file'] === true,\n      }\n    })\n}\n\nfunction extractWorkflowSpans(spans: Span[]): WorkflowSpan[] {\n  return spans\n    .filter((s) => s.name === 'handler.workflow')\n    .map((s) => {\n      const d = data(s)\n      const result: WorkflowSpan = {\n        name: String(d['workflow.name'] || d['workflowName'] || ''),\n        status: s.status as WorkflowSpan['status'],\n      }\n      const statusFinal = d['workflow.status.final']\n      if (statusFinal) {\n        result.statusFinal = String(statusFinal)\n      }\n      return result\n    })\n}\n\nfunction extractHandler(spans: Span[]): { handlerDuration: number; handlerStatus: 'ok' | 'error' } {\n  const handler = spans.find(\n    (s) => (s.name === 'handler.conversation' || s.name === 'handler.workflow') && s.status !== 'running'\n  )\n\n  if (!handler) {\n    return { handlerDuration: 0, handlerStatus: 'ok' }\n  }\n\n  return {\n    handlerDuration: handler.timing.duration ?? 0,\n    handlerStatus: handler.status === 'error' ? 'error' : 'ok',\n  }\n}\n"],
+  "mappings": ";AASA,SAAS,KAAK,MAAqC;AACjD,SAAQ,KAAK,QAAQ,OAAO,KAAK,SAAS,WAAW,KAAK,OAAO,CAAC;AACpE;AAGA,SAAS,aAAa,OAAyB;AAC7C,MAAI,SAAS,KAAM,QAAO;AAC1B,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI;AACF,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB,QAAQ;AAEN,WAAO;AAAA,EACT;AACF;AAGA,SAAS,oBAAuB,OAAgB,UAAgB;AAC9D,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI;AACF,WAAO,KAAK,MAAM,KAAK;AAAA,EACzB,QAAQ;AAEN,WAAO;AAAA,EACT;AACF;AAYO,SAAS,eAAe,OAAyB;AACtD,SAAO;AAAA,IACL,UAAU,gBAAgB,KAAK;AAAA,IAC/B,WAAW,iBAAiB,KAAK;AAAA,IACjC,gBAAgB,sBAAsB,KAAK;AAAA,IAC3C,eAAe,qBAAqB,KAAK;AAAA,IACzC,GAAG,eAAe,KAAK;AAAA,EACzB;AACF;AAEA,SAAS,gBAAgB,OAAyB;AAChD,SAAO,MACJ,OAAO,CAAC,MAAM,EAAE,SAAS,sBAAsB,KAAK,CAAC,EAAE,iBAAiB,CAAC,EACzE,KAAK,CAAC,GAAG,MAAM,EAAE,OAAO,YAAY,EAAE,OAAO,SAAS,EACtD,IAAI,CAAC,MAAM,OAAO,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;AAClD;AAEA,SAAS,iBAAiB,OAA2B;AACnD,QAAM,YAAY,MAAM;AAAA,IACtB,CAAC,MAAM,EAAE,SAAS,qBAAqB,EAAE,WAAW,aAAa,KAAK,CAAC,EAAE,sBAAsB;AAAA,EACjG;AAGA,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,SAAS,UAAU,OAAO,CAAC,MAAM;AACrC,QAAI,KAAK,IAAI,EAAE,GAAG,IAAI,EAAG,QAAO;AAChC,SAAK,IAAI,EAAE,GAAG,IAAI;AAClB,WAAO;AAAA,EACT,CAAC;AAED,SAAO,OACJ,KAAK,CAAC,GAAG,OAAO,EAAE,OAAO,WAAW,aAAa,EAAE,OAAO,WAAW,SAAS,EAC9E,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,KAAK,CAAC;AAChB,UAAM,QAAQ,oBAA6C,EAAE,uBAAuB,GAAG,CAAC,CAAC;AACzF,WAAO;AAAA,MACL,MAAM,OAAO,EAAE,sBAAsB,CAAC;AAAA,MACtC;AAAA,MACA,QAAQ,OAAO,EAAE,wBAAwB,KAAK,EAAE;AAAA,MAChD,QAAQ,OAAO,EAAE,wBAAwB,KAAK,SAAS;AAAA,IACzD;AAAA,EACF,CAAC;AACL;AAEA,SAAS,sBAAsB,OAAgC;AAC7D,SAAO,MACJ,OAAO,CAAC,MAAM;AACb,QAAI,EAAE,SAAS,aAAc,QAAO;AACpC,UAAM,IAAI,KAAK,CAAC;AAChB,UAAM,cAAc,EAAE,oBAAoB;AAC1C,WAAO,MAAM,QAAQ,WAAW,KAAK,YAAY,SAAS;AAAA,EAC5D,CAAC,EACA,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,KAAK,CAAC;AAChB,WAAO;AAAA,MACL,MAAM,OAAO,EAAE,MAAM,KAAK,SAAS;AAAA,MACnC,aAAc,EAAE,oBAAoB,KAAkB,CAAC;AAAA,MACvD,UAAU,aAAa,EAAE,sBAAsB,CAAC;AAAA,MAChD,SAAS,aAAa,EAAE,aAAa,CAAC;AAAA,MACtC,eAAe,EAAE,iBAAiB,MAAM;AAAA,IAC1C;AAAA,EACF,CAAC;AACL;AAEA,SAAS,qBAAqB,OAA+B;AAC3D,SAAO,MACJ,OAAO,CAAC,MAAM,EAAE,SAAS,kBAAkB,EAC3C,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,KAAK,CAAC;AAChB,UAAM,SAAuB;AAAA,MAC3B,MAAM,OAAO,EAAE,eAAe,KAAK,EAAE,cAAc,KAAK,EAAE;AAAA,MAC1D,QAAQ,EAAE;AAAA,IACZ;AACA,UAAM,cAAc,EAAE,uBAAuB;AAC7C,QAAI,aAAa;AACf,aAAO,cAAc,OAAO,WAAW;AAAA,IACzC;AACA,WAAO;AAAA,EACT,CAAC;AACL;AAEA,SAAS,eAAe,OAA2E;AACjG,QAAM,UAAU,MAAM;AAAA,IACpB,CAAC,OAAO,EAAE,SAAS,0BAA0B,EAAE,SAAS,uBAAuB,EAAE,WAAW;AAAA,EAC9F;AAEA,MAAI,CAAC,SAAS;AACZ,WAAO,EAAE,iBAAiB,GAAG,eAAe,KAAK;AAAA,EACnD;AAEA,SAAO;AAAA,IACL,iBAAiB,QAAQ,OAAO,YAAY;AAAA,IAC5C,eAAe,QAAQ,WAAW,UAAU,UAAU;AAAA,EACxD;AACF;",
   "names": []
 }

package/dist/types.d.ts

@@ -56,6 +56,8 @@ export interface EvalReport {
     pass: boolean;
     duration: number;
     error?: string;
+    /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError — lets consumers distinguish setup/config failures from bot failures. */
+    errorCode?: string;
 }
 export type EvalProgressEvent = {
     type: 'suite_start';
@@ -120,6 +122,14 @@ export interface EvalLogger {
     warn: (message: string) => void;
     error: (message: string) => void;
 }
+/**
+ * Fallback when no logger is injected (programmatic use outside the CLI).
+ * console rather than a no-op so warnings stay visible — a silent default
+ * would hide judge failures and skipped pre-flights, the exact swallow
+ * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own
+ * structured logger via EvalRunnerConfig.logger.
+ */
+export declare const defaultLogger: EvalLogger;
 export interface EvalRunnerConfig {
     credentials: {
         token: string;
@@ -141,6 +151,13 @@ export interface EvalRunnerConfig {
     onProgress?: (event: EvalProgressEvent) => void;
     /** Logger for warnings and errors. Defaults to console. */
     logger?: EvalLogger;
+    /**
+     * Called for unexpected (internal-bug) eval failures that runEval folds
+     * into report.error — they never propagate to the CLI's command boundary,
+     * so without this hook they'd be invisible to error tracking. The CLI binds
+     * this to telemetry.captureException; evals itself stays telemetry-free.
+     */
+    onException?: (error: unknown, properties?: Record<string, unknown>) => void;
     /**
      * Optional abort signal — when fired, the suite stops starting new evals.
      * The currently running eval continues to completion (LLM calls in flight

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,IAAI,EACJ,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,cAAc,CAAA;AAErB,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE/C,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAA;IACpB,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC;QAAE,YAAY,EAAE;YAAE,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAA;IAGjG,aAAa,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/C,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAC9C;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAA;CACzE;AAID,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,YAAY,EAAE,CAAA;IAC1B,IAAI,EAAE,OAAO,CAAA;IACb,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAA;IACnB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAA;CACrB;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAClC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,iBAAiB,EAAE,YAAY,EAAE,CAAA;IACjC,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAID,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3E;IACE,IAAI,EAAE,YAAY,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;CACpB,GACD;IACE,IAAI,EAAE,eAAe,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,UAAU,CAAA;CACvB,GACD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAIrE,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;QAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;QACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;KACnC,CAAA;IACD;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAID,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACjC;AAID,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;IACD,SAAS,EAAE,MAAM,CAAA;IACjB,oGAAoG;IACpG,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,kGAAkG;IAClG,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC;;;OAGG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAA;IAC/C,2DAA2D;IAC3D,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,4FAA4F;IAC5F,WAAW,CAAC,EAAE;QACZ,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,mIAAmI;QACnI,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,wFAAwF;QACxF,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;CACF;AAID,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;CACnC;AAID,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,OAAO,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;IAChB,aAAa,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;IAClC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,cAAc,EAAE,aAAa,EAAE,CAAA;IAC/B,aAAa,EAAE,YAAY,EAAE,CAAA;IAC7B,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,IAAI,GAAG,OAAO,CAAA;CAC9B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,IAAI,EACJ,KAAK,cAAc,EACnB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,cAAc,CAAA;AAErB,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE/C,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAA;IACpB,kBAAkB,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC;QAAE,YAAY,EAAE;YAAE,EAAE,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAA;IAGjG,aAAa,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;IAC/C,WAAW,EAAE,CAAC,KAAK,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CAC9C;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAA;CACzE;AAID,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,YAAY,EAAE,CAAA;IAC1B,IAAI,EAAE,OAAO,CAAA;IACb,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAA;IACnB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAA;CACrB;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;IAClC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,iBAAiB,EAAE,YAAY,EAAE,CAAA;IACjC,IAAI,EAAE,OAAO,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,gJAAgJ;IAChJ,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAID,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3C;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC3E;IACE,IAAI,EAAE,YAAY,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;CACpB,GACD;IACE,IAAI,EAAE,eAAe,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,UAAU,CAAA;CACvB,GACD;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,CAAA;CAAE,GAC9E;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAIrE,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE;QACP,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;QAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;QACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;KACnC,CAAA;IACD;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAID,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B,KAAK,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CACjC;AAED;;;;;;GAMG;AACH,eAAO,MAAM,aAAa,EAAE,UAAoB,CAAA;AAIhD,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE;QACX,KAAK,EAAE,MAAM,CAAA;QACb,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;IACD,SAAS,EAAE,MAAM,CAAA;IACjB,oGAAoG;IACpG,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,kGAAkG;IAClG,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACzC;;;OAGG;IACH,UAAU,CAAC,EAAE,UAAU,CAAA;IACvB,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAA;IAC/C,2DAA2D;IAC3D,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB;;;;;OAKG;IACH,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAA;IAC5E;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,4FAA4F;IAC5F,WAAW,CAAC,EAAE;QACZ,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,mIAAmI;QACnI,kBAAkB,CAAC,EAAE,MAAM,CAAA;QAC3B,wFAAwF;QACxF,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;CACF;AAID,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;IACf,IAAI,CAAC,EAAE,YAAY,GAAG,YAAY,CAAA;CACnC;AAID,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,OAAO,CAAA;IACjB,OAAO,EAAE,OAAO,CAAA;IAChB,aAAa,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;IAClC,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,cAAc,EAAE,aAAa,EAAE,CAAA;IAC/B,aAAa,EAAE,YAAY,EAAE,CAAA;IAC7B,eAAe,EAAE,MAAM,CAAA;IACvB,aAAa,EAAE,IAAI,GAAG,OAAO,CAAA;CAC9B"}

package/dist/types.js

@@ -19,7 +19,11 @@ var Eval = class {
     if (def.options !== void 0) this.options = def.options;
   }
 };
+
+// src/types.ts
+var defaultLogger = console;
 export {
-  Eval
+  Eval,
+  defaultLogger
 };
 //# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -1,7 +1,7 @@
 {
   "version": 3,
-  "sources": ["../src/definition.ts"],
-  "sourcesContent": ["export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n"],
-  "mappings": ";AAuFO,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;",
+  "sources": ["../src/definition.ts", "../src/types.ts"],
+  "sourcesContent": ["export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { type: string; payload?: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n", "/**\n * Eval type definitions.\n *\n * User-facing types live in @botpress/evals so eval authors can update eval\n * tooling without tying the authoring API to the agent runtime package.\n */\n\nexport {\n  Eval,\n  type EvalDefinition,\n  type EvalSetup,\n  type ConversationTurn,\n  type TurnAssertions,\n  type OutcomeAssertions,\n  type ResponseAssertion,\n  type ToolAssertion,\n  type StateAssertion,\n  type WorkflowAssertion,\n  type TimingAssertion,\n  type MatchOperator,\n  type NumericOperator,\n} from './definition'\n\nexport type { Span, SpanStatus } from './trace'\n\nexport interface ChatClientConnection {\n  user: { id: string }\n  createConversation: (props: Record<string, unknown>) => Promise<{ conversation: { id: string } }>\n  // Broad parameter types keep this compatible with @botpress/chat's narrower\n  // generated payload unions while avoiding a dependency on those definitions.\n  createMessage: (props: any) => Promise<unknown>\n  createEvent: (props: any) => Promise<unknown>\n}\n\nexport interface ChatClient {\n  connect: (props: { webhookId: string }) => Promise<ChatClientConnection>\n}\n\n// --- Grader result (universal currency for all graders) ---\n\nexport interface GraderResult {\n  assertion: string\n  pass: boolean\n  expected: string\n  actual: string\n}\n\n// --- Tool call extracted from traces ---\n\nexport interface ToolCall {\n  name: string\n  input: Record<string, unknown>\n  output: string\n  status: string\n}\n\n// --- Turn report ---\n\nexport interface TurnReport {\n  turnNumber: number\n  userMessage: string\n  botResponse: string\n  assertions: GraderResult[]\n  pass: boolean\n  /** Time spent waiting for the bot to respond (ms) */\n  botDuration: number\n  /** Time spent fetching traces + running graders (ms) */\n  evalDuration: number\n}\n\n// --- Single eval report ---\n\nexport interface EvalReport {\n  name: string\n  description?: string\n  type?: 'capability' | 'regression'\n  tags?: string[]\n  turns: TurnReport[]\n  outcomeAssertions: GraderResult[]\n  pass: boolean\n  duration: number\n  error?: string\n  /** Stable EvalErrorCode when `error` came from a typed EvalRunnerError \u2014 lets consumers distinguish setup/config failures from bot failures. */\n  errorCode?: string\n}\n\n// --- Progress event for real-time UI updates ---\n\nexport type EvalProgressEvent =\n  | { type: 'suite_start'; totalEvals: number }\n  | { type: 'eval_start'; evalName: string; index: number; totalTurns: number }\n  | {\n      type: 'turn_start'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      userMessage: string\n    }\n  | {\n      type: 'turn_complete'\n      evalName: string\n      evalIndex: number\n      turnNumber: number\n      totalTurns: number\n      turnReport: TurnReport\n    }\n  | { type: 'eval_complete'; evalName: string; index: number; report: EvalReport }\n  | { type: 'suite_complete'; report: EvalRunReport; error?: string }\n\n// --- Eval run report (full suite) ---\n\nexport interface EvalRunReport {\n  id: string\n  timestamp: string\n  evals: EvalReport[]\n  passed: number\n  failed: number\n  total: number\n  duration: number\n  filter?: {\n    names?: string[]\n    tags?: string[]\n    type?: 'capability' | 'regression'\n  }\n  /**\n   * True when the suite was aborted mid-run (e.g. user clicked Stop). The\n   * persisted report is partial \u2014 `evals` reflects only the completed evals,\n   * and `total` matches that count, not the original suite size.\n   */\n  aborted?: boolean\n}\n\n// --- Bot connection info ---\n\nexport interface BotConnection {\n  webhookId: string\n  botId: string\n  token: string\n  apiUrl: string\n}\n\n// --- Logger interface ---\n\nexport interface EvalLogger {\n  info: (message: string) => void\n  warn: (message: string) => void\n  error: (message: string) => void\n}\n\n/**\n * Fallback when no logger is injected (programmatic use outside the CLI).\n * console rather than a no-op so warnings stay visible \u2014 a silent default\n * would hide judge failures and skipped pre-flights, the exact swallow\n * anti-pattern docs/ERROR-HANDLING.md bans. The CLI always injects its own\n * structured logger via EvalRunnerConfig.logger.\n */\nexport const defaultLogger: EvalLogger = console\n\n// --- Eval runner configuration ---\n\nexport interface EvalRunnerConfig {\n  credentials: {\n    token: string\n    apiUrl: string\n    botId: string\n    workspaceId?: string\n    webhookId?: string\n  }\n  agentPath: string\n  /** URL of the dev server providing traces and config. Defaults to the discovered DevConsole URL. */\n  devServerUrl?: string\n  /** Headers for dev-server requests, e.g. X-Agent-Path when routed through the singleton proxy. */\n  devServerHeaders?: Record<string, string>\n  /**\n   * Optional host-provided chat client. The CLI passes its bundled chat client\n   * here so the eval engine does not depend on @botpress/adk.\n   */\n  chatClient?: ChatClient\n  onProgress?: (event: EvalProgressEvent) => void\n  /** Logger for warnings and errors. Defaults to console. */\n  logger?: EvalLogger\n  /**\n   * Called for unexpected (internal-bug) eval failures that runEval folds\n   * into report.error \u2014 they never propagate to the CLI's command boundary,\n   * so without this hook they'd be invisible to error tracking. The CLI binds\n   * this to telemetry.captureException; evals itself stays telemetry-free.\n   */\n  onException?: (error: unknown, properties?: Record<string, unknown>) => void\n  /**\n   * Optional abort signal \u2014 when fired, the suite stops starting new evals.\n   * The currently running eval continues to completion (LLM calls in flight\n   * cannot always be cancelled), but no further evals are started.\n   */\n  signal?: AbortSignal\n  /** Bot-level eval options from agent.config.ts. Cascades: eval > agent config > default. */\n  evalOptions?: {\n    idleTimeout?: number\n    /** LLM judge pass threshold (integer, 1-5). Values outside this range are clamped. Cascades: eval > agent config > default (3). */\n    judgePassThreshold?: number\n    /** Model to use for llm_judge assertions (e.g. 'openai:gpt-4o'). Defaults to 'fast'. */\n    judgeModel?: string\n  }\n}\n\n// --- Eval suite filter ---\n\nexport interface EvalFilter {\n  names?: string[]\n  tags?: string[]\n  type?: 'capability' | 'regression'\n}\n\n// --- Transformer types (trace-driven eval engine) ---\n\nexport interface StateMutation {\n  type: string // 'bot', 'user', 'conversation'\n  changedKeys: string[]\n  previous: unknown\n  current: unknown\n  swappedToFile: boolean // data['swapped_to_file'] === true\n}\n\nexport interface WorkflowSpan {\n  name: string\n  status: 'running' | 'ok' | 'error'\n  statusFinal?: string // 'completed', 'continue', 'failed'\n}\n\nexport interface TurnData {\n  messages: string[]\n  toolCalls: ToolCall[]\n  stateMutations: StateMutation[]\n  workflowSpans: WorkflowSpan[]\n  handlerDuration: number\n  handlerStatus: 'ok' | 'error'\n}\n"],
+  "mappings": ";AAuFO,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAKT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;AC+CO,IAAM,gBAA4B;",
   "names": []
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botpress/evals",
-  "version": "1.19.0-beta.6",
+  "version": "1.19.0-beta.9",
   "description": "Evaluation definitions and runner for ADK-based Botpress agents",
   "keywords": [
     "adk",