@botpress/evals 1.19.0-beta.9 → 2.0.0-alpha.0

package/dist/spans/vortex-span-source.d.ts

@@ -0,0 +1,42 @@
+import type { SpanSource, TurnWaitOptions, WorkflowWaitOptions } from './span-source';
+import type { Span } from './trace';
+interface VortexSpanSourceConfig {
+    url: string;
+    botId: string;
+    workspaceId?: string;
+    token?: string;
+    pollIntervalMs?: number;
+}
+export declare class VortexSpanSource implements SpanSource {
+    private config;
+    private conversationId;
+    private traceIds;
+    private spanMap;
+    private turnStartKeys;
+    private seenHandlerKeys;
+    private consecutiveFailures;
+    constructor(config: VortexSpanSourceConfig);
+    connect(filter: {
+        conversationId: string;
+    }): Promise<void>;
+    /**
+     * Switch polling to a new conversation, keeping `spanMap` and
+     * `seenHandlerKeys`. `traceIds` resets — they belonged to the prior conversation.
+     */
+    repoint(filter: {
+        conversationId: string;
+    }): Promise<void>;
+    private static readonly TURN_HANDLER_SPAN_NAMES;
+    startTurn(): void;
+    private static readonly SETTLE_POLLS;
+    waitForTurnComplete(opts: TurnWaitOptions): Promise<void>;
+    waitForWorkflow(name: string, opts: WorkflowWaitOptions): Promise<void>;
+    getTurnSpans(): Span[];
+    getAllSpans(): Span[];
+    disconnect(): void;
+    private _pollOnce;
+    private _hasNewCompletedHandler;
+    private _matchesWorkflowCondition;
+}
+export {};
+//# sourceMappingURL=vortex-span-source.d.ts.map

package/dist/spans/vortex-span-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vortex-span-source.d.ts","sourceRoot":"","sources":["../../src/spans/vortex-span-source.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAe,mBAAmB,EAAE,MAAM,eAAe,CAAA;AAClG,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEnC,UAAU,sBAAsB;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AA0DD,qBAAa,gBAAiB,YAAW,UAAU;IACjD,OAAO,CAAC,MAAM,CACyC;IACvD,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,aAAa,CAAoB;IACzC,OAAO,CAAC,eAAe,CAAoB;IAC3C,OAAO,CAAC,mBAAmB,CAAI;gBAEnB,MAAM,EAAE,sBAAsB;IAUpC,OAAO,CAAC,MAAM,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKhE;;;OAGG;IACG,OAAO,CAAC,MAAM,EAAE;QAAE,cAAc,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAMhE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAqD;IAEpG,SAAS,IAAI,IAAI;IASjB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAI;IAElC,mBAAmB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAiCzD,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB7E,YAAY,IAAI,IAAI,EAAE;IAUtB,WAAW,IAAI,IAAI,EAAE;IAIrB,UAAU,IAAI,IAAI;YAIJ,SAAS;IAoCvB,OAAO,CAAC,uBAAuB;IAe/B,OAAO,CAAC,yBAAyB;CAWlC"}

package/dist/spans/vortex-span-source.js

@@ -0,0 +1,184 @@
+// src/spans/vortex-span-source.ts
+function spanKey(span) {
+  return `${span.id.trace}:${span.id.span}`;
+}
+function spanData(span) {
+  return span.data && typeof span.data === "object" ? span.data : {};
+}
+function normalizeSpan(span) {
+  if (span.status !== "running") return span;
+  const data = spanData(span);
+  const autonomousStatus = data["autonomous.status"];
+  if (autonomousStatus === "generation_error" || autonomousStatus === "execution_error" || autonomousStatus === "invalid_code_error" || autonomousStatus === "exit_error" || autonomousStatus === "aborted") {
+    return { ...span, status: "error" };
+  }
+  if (autonomousStatus === "thinking_requested" || autonomousStatus === "callback_requested" || autonomousStatus === "exit_success") {
+    return { ...span, status: "ok" };
+  }
+  const { startedAt, endedAt } = span.timing;
+  if (typeof endedAt === "number" && Number.isFinite(endedAt) && endedAt > startedAt) {
+    return { ...span, status: "ok" };
+  }
+  return span;
+}
+function sleep(ms, signal) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) return reject(new Error("aborted"));
+    const onAbort = () => {
+      clearTimeout(timer);
+      reject(new Error("aborted"));
+    };
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+var VortexSpanSource = class _VortexSpanSource {
+  config;
+  conversationId = "";
+  traceIds = /* @__PURE__ */ new Set();
+  spanMap = /* @__PURE__ */ new Map();
+  turnStartKeys = /* @__PURE__ */ new Set();
+  seenHandlerKeys = /* @__PURE__ */ new Set();
+  consecutiveFailures = 0;
+  constructor(config) {
+    this.config = {
+      url: config.url.replace(/\/$/, ""),
+      botId: config.botId,
+      pollIntervalMs: config.pollIntervalMs ?? 500,
+      ...config.workspaceId ? { workspaceId: config.workspaceId } : {},
+      ...config.token ? { token: config.token } : {}
+    };
+  }
+  async connect(filter) {
+    this.conversationId = filter.conversationId;
+    await this._pollOnce();
+  }
+  /**
+   * Switch polling to a new conversation, keeping `spanMap` and
+   * `seenHandlerKeys`. `traceIds` resets — they belonged to the prior conversation.
+   */
+  async repoint(filter) {
+    this.conversationId = filter.conversationId;
+    this.traceIds = /* @__PURE__ */ new Set();
+    await this._pollOnce();
+  }
+  static TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
+  startTurn() {
+    this.turnStartKeys = new Set(this.spanMap.keys());
+    for (const [key, span] of this.spanMap) {
+      if (_VortexSpanSource.TURN_HANDLER_SPAN_NAMES.has(span.name)) {
+        this.seenHandlerKeys.add(key);
+      }
+    }
+  }
+  static SETTLE_POLLS = 3;
+  async waitForTurnComplete(opts) {
+    const deadline = Date.now() + opts.timeout;
+    const maxPolls = opts.settleMaxMs ? Math.floor(opts.settleMaxMs / this.config.pollIntervalMs) : Infinity;
+    const settlePolls = Math.min(
+      maxPolls,
+      Math.max(
+        _VortexSpanSource.SETTLE_POLLS,
+        opts.settleQuietMs ? Math.ceil(opts.settleQuietMs / this.config.pollIntervalMs) : 0
+      )
+    );
+    while (Date.now() < deadline) {
+      if (opts.abortSignal?.aborted) throw new Error("waitForTurnComplete aborted");
+      await this._pollOnce();
+      if (this._hasNewCompletedHandler(opts.acceptSpanNames)) {
+        for (let i = 0; i < settlePolls; i++) {
+          await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {
+          });
+          await this._pollOnce();
+        }
+        return;
+      }
+      await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {
+      });
+    }
+    throw new Error(`waitForTurnComplete timed out after ${opts.timeout}ms`);
+  }
+  async waitForWorkflow(name, opts) {
+    const deadline = Date.now() + opts.timeout;
+    while (Date.now() < deadline) {
+      if (opts.abortSignal?.aborted) throw new Error(`waitForWorkflow("${name}") aborted`);
+      await this._pollOnce();
+      if (this._matchesWorkflowCondition(name, opts.signal)) return;
+      await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {
+      });
+    }
+    throw new Error(`waitForWorkflow("${name}", { signal: "${opts.signal}" }) timed out after ${opts.timeout}ms`);
+  }
+  getTurnSpans() {
+    const result = [];
+    for (const [key, span] of this.spanMap) {
+      if (!this.turnStartKeys.has(key)) {
+        result.push(span);
+      }
+    }
+    return result;
+  }
+  getAllSpans() {
+    return Array.from(this.spanMap.values());
+  }
+  disconnect() {
+  }
+  async _pollOnce() {
+    const url = new URL(`${this.config.url}/v1/traces/bot/${this.config.botId}/spans/adk`);
+    url.searchParams.set("conversationId", this.conversationId);
+    for (const traceId of this.traceIds) {
+      url.searchParams.append("traceId", traceId);
+    }
+    const res = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        ...this.config.token ? { Authorization: `Bearer ${this.config.token}` } : {}
+      }
+    });
+    if (!res.ok) {
+      this.consecutiveFailures++;
+      const body = await res.text().catch(() => "");
+      const msg = `Vortex poll failed: ${res.status} ${body.slice(0, 200)}`;
+      if (this.consecutiveFailures >= 5) {
+        throw new Error(`${msg} (${this.consecutiveFailures} consecutive failures)`);
+      }
+      console.warn(`[VortexSpanSource] ${msg}`);
+      return;
+    }
+    this.consecutiveFailures = 0;
+    const payload = await res.json();
+    const spans = Array.isArray(payload) ? payload : Array.isArray(payload.spans) ? payload.spans : [];
+    for (const rawSpan of spans) {
+      const span = normalizeSpan(rawSpan);
+      this.spanMap.set(spanKey(span), span);
+      this.traceIds.add(span.id.trace);
+    }
+  }
+  _hasNewCompletedHandler(acceptSpanNames) {
+    for (const [key, span] of this.spanMap) {
+      if (_VortexSpanSource.TURN_HANDLER_SPAN_NAMES.has(span.name) && (!acceptSpanNames || acceptSpanNames.has(span.name)) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
+        this.seenHandlerKeys.add(key);
+        return true;
+      }
+    }
+    return false;
+  }
+  _matchesWorkflowCondition(name, signal) {
+    for (const span of this.spanMap.values()) {
+      if (span.name !== "handler.workflow") continue;
+      const d = spanData(span);
+      const wfName = d["workflow.name"] || d["workflowName"];
+      if (wfName !== name) continue;
+      if (signal === "entered") return true;
+      if (signal === "completed") return span.status === "ok" || span.status === "error";
+    }
+    return false;
+  }
+};
+export {
+  VortexSpanSource
+};
+//# sourceMappingURL=vortex-span-source.js.map

package/dist/spans/vortex-span-source.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/spans/vortex-span-source.ts"],
+  "sourcesContent": ["import type { SpanSource, TurnWaitOptions, WaitOptions, WorkflowWaitOptions } from './span-source'\nimport type { Span } from './trace'\n\ninterface VortexSpanSourceConfig {\n  url: string\n  botId: string\n  workspaceId?: string\n  token?: string\n  pollIntervalMs?: number\n}\n\ntype VortexTraceResponse = Span[] | { spans?: Span[] }\n\nfunction spanKey(span: Span): string {\n  return `${span.id.trace}:${span.id.span}`\n}\n\nfunction spanData(span: Span): Record<string, unknown> {\n  return span.data && typeof span.data === 'object' ? (span.data as Record<string, unknown>) : {}\n}\n\nfunction normalizeSpan(span: Span): Span {\n  if (span.status !== 'running') return span\n  const data = spanData(span)\n  const autonomousStatus = data['autonomous.status']\n\n  if (\n    autonomousStatus === 'generation_error' ||\n    autonomousStatus === 'execution_error' ||\n    autonomousStatus === 'invalid_code_error' ||\n    autonomousStatus === 'exit_error' ||\n    autonomousStatus === 'aborted'\n  ) {\n    return { ...span, status: 'error' }\n  }\n\n  if (\n    autonomousStatus === 'thinking_requested' ||\n    autonomousStatus === 'callback_requested' ||\n    autonomousStatus === 'exit_success'\n  ) {\n    return { ...span, status: 'ok' }\n  }\n\n  const { startedAt, endedAt } = span.timing\n  if (typeof endedAt === 'number' && Number.isFinite(endedAt) && endedAt > startedAt) {\n    return { ...span, status: 'ok' }\n  }\n\n  return span\n}\n\nfunction sleep(ms: number, signal?: AbortSignal): Promise<void> {\n  return new Promise<void>((resolve, reject) => {\n    if (signal?.aborted) return reject(new Error('aborted'))\n    const onAbort = () => {\n      clearTimeout(timer)\n      reject(new Error('aborted'))\n    }\n    const timer = setTimeout(() => {\n      signal?.removeEventListener('abort', onAbort)\n      resolve()\n    }, ms)\n    signal?.addEventListener('abort', onAbort, { once: true })\n  })\n}\n\nexport class VortexSpanSource implements SpanSource {\n  private config: Required<Omit<VortexSpanSourceConfig, 'workspaceId' | 'token'>> &\n    Pick<VortexSpanSourceConfig, 'workspaceId' | 'token'>\n  private conversationId = ''\n  private traceIds = new Set<string>()\n  private spanMap = new Map<string, Span>()\n  private turnStartKeys = new Set<string>()\n  private seenHandlerKeys = new Set<string>()\n  private consecutiveFailures = 0\n\n  constructor(config: VortexSpanSourceConfig) {\n    this.config = {\n      url: config.url.replace(/\\/$/, ''),\n      botId: config.botId,\n      pollIntervalMs: config.pollIntervalMs ?? 500,\n      ...(config.workspaceId ? { workspaceId: config.workspaceId } : {}),\n      ...(config.token ? { token: config.token } : {}),\n    }\n  }\n\n  async connect(filter: { conversationId: string }): Promise<void> {\n    this.conversationId = filter.conversationId\n    await this._pollOnce()\n  }\n\n  /**\n   * Switch polling to a new conversation, keeping `spanMap` and\n   * `seenHandlerKeys`. `traceIds` resets \u2014 they belonged to the prior conversation.\n   */\n  async repoint(filter: { conversationId: string }): Promise<void> {\n    this.conversationId = filter.conversationId\n    this.traceIds = new Set()\n    await this._pollOnce()\n  }\n\n  private static readonly TURN_HANDLER_SPAN_NAMES = new Set(['handler.conversation', 'handler.event'])\n\n  startTurn(): void {\n    this.turnStartKeys = new Set(this.spanMap.keys())\n    for (const [key, span] of this.spanMap) {\n      if (VortexSpanSource.TURN_HANDLER_SPAN_NAMES.has(span.name)) {\n        this.seenHandlerKeys.add(key)\n      }\n    }\n  }\n\n  private static readonly SETTLE_POLLS = 3\n\n  async waitForTurnComplete(opts: TurnWaitOptions): Promise<void> {\n    const deadline = Date.now() + opts.timeout\n    // Honor the requested quiet window by polling at least that long after the\n    // handler closes (async follow-through such as a workflow run + callback\n    // only executes after the handler closes).\n    const maxPolls = opts.settleMaxMs ? Math.floor(opts.settleMaxMs / this.config.pollIntervalMs) : Infinity\n    const settlePolls = Math.min(\n      maxPolls,\n      Math.max(\n        VortexSpanSource.SETTLE_POLLS,\n        opts.settleQuietMs ? Math.ceil(opts.settleQuietMs / this.config.pollIntervalMs) : 0\n      )\n    )\n\n    while (Date.now() < deadline) {\n      if (opts.abortSignal?.aborted) throw new Error('waitForTurnComplete aborted')\n\n      await this._pollOnce()\n\n      if (this._hasNewCompletedHandler(opts.acceptSpanNames)) {\n        for (let i = 0; i < settlePolls; i++) {\n          await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {})\n          await this._pollOnce()\n        }\n        return\n      }\n\n      await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {})\n    }\n\n    throw new Error(`waitForTurnComplete timed out after ${opts.timeout}ms`)\n  }\n\n  async waitForWorkflow(name: string, opts: WorkflowWaitOptions): Promise<void> {\n    const deadline = Date.now() + opts.timeout\n\n    while (Date.now() < deadline) {\n      if (opts.abortSignal?.aborted) throw new Error(`waitForWorkflow(\"${name}\") aborted`)\n\n      await this._pollOnce()\n\n      if (this._matchesWorkflowCondition(name, opts.signal)) return\n\n      await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {})\n    }\n\n    throw new Error(`waitForWorkflow(\"${name}\", { signal: \"${opts.signal}\" }) timed out after ${opts.timeout}ms`)\n  }\n\n  getTurnSpans(): Span[] {\n    const result: Span[] = []\n    for (const [key, span] of this.spanMap) {\n      if (!this.turnStartKeys.has(key)) {\n        result.push(span)\n      }\n    }\n    return result\n  }\n\n  getAllSpans(): Span[] {\n    return Array.from(this.spanMap.values())\n  }\n\n  disconnect(): void {\n    // Stateless polling client.\n  }\n\n  private async _pollOnce(): Promise<void> {\n    const url = new URL(`${this.config.url}/v1/traces/bot/${this.config.botId}/spans/adk`)\n    url.searchParams.set('conversationId', this.conversationId)\n    for (const traceId of this.traceIds) {\n      url.searchParams.append('traceId', traceId)\n    }\n\n    const res = await fetch(url, {\n      headers: {\n        Accept: 'application/json',\n        ...(this.config.token ? { Authorization: `Bearer ${this.config.token}` } : {}),\n      },\n    })\n\n    if (!res.ok) {\n      this.consecutiveFailures++\n      const body = await res.text().catch(() => '')\n      const msg = `Vortex poll failed: ${res.status} ${body.slice(0, 200)}`\n      if (this.consecutiveFailures >= 5) {\n        throw new Error(`${msg} (${this.consecutiveFailures} consecutive failures)`)\n      }\n      console.warn(`[VortexSpanSource] ${msg}`)\n      return\n    }\n\n    this.consecutiveFailures = 0\n    const payload = (await res.json()) as VortexTraceResponse\n    const spans = Array.isArray(payload) ? payload : Array.isArray(payload.spans) ? payload.spans : []\n\n    for (const rawSpan of spans) {\n      const span = normalizeSpan(rawSpan)\n      this.spanMap.set(spanKey(span), span)\n      this.traceIds.add(span.id.trace)\n    }\n  }\n\n  private _hasNewCompletedHandler(acceptSpanNames?: ReadonlySet<string>): boolean {\n    for (const [key, span] of this.spanMap) {\n      if (\n        VortexSpanSource.TURN_HANDLER_SPAN_NAMES.has(span.name) &&\n        (!acceptSpanNames || acceptSpanNames.has(span.name)) &&\n        !this.seenHandlerKeys.has(key) &&\n        (span.status === 'ok' || span.status === 'error')\n      ) {\n        this.seenHandlerKeys.add(key)\n        return true\n      }\n    }\n    return false\n  }\n\n  private _matchesWorkflowCondition(name: string, signal: 'entered' | 'completed'): boolean {\n    for (const span of this.spanMap.values()) {\n      if (span.name !== 'handler.workflow') continue\n      const d = spanData(span)\n      const wfName = d['workflow.name'] || d['workflowName']\n      if (wfName !== name) continue\n      if (signal === 'entered') return true\n      if (signal === 'completed') return span.status === 'ok' || span.status === 'error'\n    }\n    return false\n  }\n}\n"],
+  "mappings": ";AAaA,SAAS,QAAQ,MAAoB;AACnC,SAAO,GAAG,KAAK,GAAG,KAAK,IAAI,KAAK,GAAG,IAAI;AACzC;AAEA,SAAS,SAAS,MAAqC;AACrD,SAAO,KAAK,QAAQ,OAAO,KAAK,SAAS,WAAY,KAAK,OAAmC,CAAC;AAChG;AAEA,SAAS,cAAc,MAAkB;AACvC,MAAI,KAAK,WAAW,UAAW,QAAO;AACtC,QAAM,OAAO,SAAS,IAAI;AAC1B,QAAM,mBAAmB,KAAK,mBAAmB;AAEjD,MACE,qBAAqB,sBACrB,qBAAqB,qBACrB,qBAAqB,wBACrB,qBAAqB,gBACrB,qBAAqB,WACrB;AACA,WAAO,EAAE,GAAG,MAAM,QAAQ,QAAQ;AAAA,EACpC;AAEA,MACE,qBAAqB,wBACrB,qBAAqB,wBACrB,qBAAqB,gBACrB;AACA,WAAO,EAAE,GAAG,MAAM,QAAQ,KAAK;AAAA,EACjC;AAEA,QAAM,EAAE,WAAW,QAAQ,IAAI,KAAK;AACpC,MAAI,OAAO,YAAY,YAAY,OAAO,SAAS,OAAO,KAAK,UAAU,WAAW;AAClF,WAAO,EAAE,GAAG,MAAM,QAAQ,KAAK;AAAA,EACjC;AAEA,SAAO;AACT;AAEA,SAAS,MAAM,IAAY,QAAqC;AAC9D,SAAO,IAAI,QAAc,CAAC,SAAS,WAAW;AAC5C,QAAI,QAAQ,QAAS,QAAO,OAAO,IAAI,MAAM,SAAS,CAAC;AACvD,UAAM,UAAU,MAAM;AACpB,mBAAa,KAAK;AAClB,aAAO,IAAI,MAAM,SAAS,CAAC;AAAA,IAC7B;AACA,UAAM,QAAQ,WAAW,MAAM;AAC7B,cAAQ,oBAAoB,SAAS,OAAO;AAC5C,cAAQ;AAAA,IACV,GAAG,EAAE;AACL,YAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,EAC3D,CAAC;AACH;AAEO,IAAM,mBAAN,MAAM,kBAAuC;AAAA,EAC1C;AAAA,EAEA,iBAAiB;AAAA,EACjB,WAAW,oBAAI,IAAY;AAAA,EAC3B,UAAU,oBAAI,IAAkB;AAAA,EAChC,gBAAgB,oBAAI,IAAY;AAAA,EAChC,kBAAkB,oBAAI,IAAY;AAAA,EAClC,sBAAsB;AAAA,EAE9B,YAAY,QAAgC;AAC1C,SAAK,SAAS;AAAA,MACZ,KAAK,OAAO,IAAI,QAAQ,OAAO,EAAE;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,gBAAgB,OAAO,kBAAkB;AAAA,MACzC,GAAI,OAAO,cAAc,EAAE,aAAa,OAAO,YAAY,IAAI,CAAC;AAAA,MAChE,GAAI,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;AAAA,IAChD;AAAA,EACF;AAAA,EAEA,MAAM,QAAQ,QAAmD;AAC/D,SAAK,iBAAiB,OAAO;AAC7B,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,QAAQ,QAAmD;AAC/D,SAAK,iBAAiB,OAAO;AAC7B,SAAK,WAAW,oBAAI,IAAI;AACxB,UAAM,KAAK,UAAU;AAAA,EACvB;AAAA,EAEA,OAAwB,0BAA0B,oBAAI,IAAI,CAAC,wBAAwB,eAAe,CAAC;AAAA,EAEnG,YAAkB;AAChB,SAAK,gBAAgB,IAAI,IAAI,KAAK,QAAQ,KAAK,CAAC;AAChD,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UAAI,kBAAiB,wBAAwB,IAAI,KAAK,IAAI,GAAG;AAC3D,aAAK,gBAAgB,IAAI,GAAG;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,OAAwB,eAAe;AAAA,EAEvC,MAAM,oBAAoB,MAAsC;AAC9D,UAAM,WAAW,KAAK,IAAI,IAAI,KAAK;AAInC,UAAM,WAAW,KAAK,cAAc,KAAK,MAAM,KAAK,cAAc,KAAK,OAAO,cAAc,IAAI;AAChG,UAAM,cAAc,KAAK;AAAA,MACvB;AAAA,MACA,KAAK;AAAA,QACH,kBAAiB;AAAA,QACjB,KAAK,gBAAgB,KAAK,KAAK,KAAK,gBAAgB,KAAK,OAAO,cAAc,IAAI;AAAA,MACpF;AAAA,IACF;AAEA,WAAO,KAAK,IAAI,IAAI,UAAU;AAC5B,UAAI,KAAK,aAAa,QAAS,OAAM,IAAI,MAAM,6BAA6B;AAE5E,YAAM,KAAK,UAAU;AAErB,UAAI,KAAK,wBAAwB,KAAK,eAAe,GAAG;AACtD,iBAAS,IAAI,GAAG,IAAI,aAAa,KAAK;AACpC,gBAAM,MAAM,KAAK,OAAO,gBAAgB,KAAK,WAAW,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AACxE,gBAAM,KAAK,UAAU;AAAA,QACvB;AACA;AAAA,MACF;AAEA,YAAM,MAAM,KAAK,OAAO,gBAAgB,KAAK,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IAC1E;AAEA,UAAM,IAAI,MAAM,uCAAuC,KAAK,OAAO,IAAI;AAAA,EACzE;AAAA,EAEA,MAAM,gBAAgB,MAAc,MAA0C;AAC5E,UAAM,WAAW,KAAK,IAAI,IAAI,KAAK;AAEnC,WAAO,KAAK,IAAI,IAAI,UAAU;AAC5B,UAAI,KAAK,aAAa,QAAS,OAAM,IAAI,MAAM,oBAAoB,IAAI,YAAY;AAEnF,YAAM,KAAK,UAAU;AAErB,UAAI,KAAK,0BAA0B,MAAM,KAAK,MAAM,EAAG;AAEvD,YAAM,MAAM,KAAK,OAAO,gBAAgB,KAAK,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IAC1E;AAEA,UAAM,IAAI,MAAM,oBAAoB,IAAI,iBAAiB,KAAK,MAAM,wBAAwB,KAAK,OAAO,IAAI;AAAA,EAC9G;AAAA,EAEA,eAAuB;AACrB,UAAM,SAAiB,CAAC;AACxB,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UAAI,CAAC,KAAK,cAAc,IAAI,GAAG,GAAG;AAChC,eAAO,KAAK,IAAI;AAAA,MAClB;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,cAAsB;AACpB,WAAO,MAAM,KAAK,KAAK,QAAQ,OAAO,CAAC;AAAA,EACzC;AAAA,EAEA,aAAmB;AAAA,EAEnB;AAAA,EAEA,MAAc,YAA2B;AACvC,UAAM,MAAM,IAAI,IAAI,GAAG,KAAK,OAAO,GAAG,kBAAkB,KAAK,OAAO,KAAK,YAAY;AACrF,QAAI,aAAa,IAAI,kBAAkB,KAAK,cAAc;AAC1D,eAAW,WAAW,KAAK,UAAU;AACnC,UAAI,aAAa,OAAO,WAAW,OAAO;AAAA,IAC5C;AAEA,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,SAAS;AAAA,QACP,QAAQ;AAAA,QACR,GAAI,KAAK,OAAO,QAAQ,EAAE,eAAe,UAAU,KAAK,OAAO,KAAK,GAAG,IAAI,CAAC;AAAA,MAC9E;AAAA,IACF,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,WAAK;AACL,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,MAAM,uBAAuB,IAAI,MAAM,IAAI,KAAK,MAAM,GAAG,GAAG,CAAC;AACnE,UAAI,KAAK,uBAAuB,GAAG;AACjC,cAAM,IAAI,MAAM,GAAG,GAAG,KAAK,KAAK,mBAAmB,wBAAwB;AAAA,MAC7E;AACA,cAAQ,KAAK,sBAAsB,GAAG,EAAE;AACxC;AAAA,IACF;AAEA,SAAK,sBAAsB;AAC3B,UAAM,UAAW,MAAM,IAAI,KAAK;AAChC,UAAM,QAAQ,MAAM,QAAQ,OAAO,IAAI,UAAU,MAAM,QAAQ,QAAQ,KAAK,IAAI,QAAQ,QAAQ,CAAC;AAEjG,eAAW,WAAW,OAAO;AAC3B,YAAM,OAAO,cAAc,OAAO;AAClC,WAAK,QAAQ,IAAI,QAAQ,IAAI,GAAG,IAAI;AACpC,WAAK,SAAS,IAAI,KAAK,GAAG,KAAK;AAAA,IACjC;AAAA,EACF;AAAA,EAEQ,wBAAwB,iBAAgD;AAC9E,eAAW,CAAC,KAAK,IAAI,KAAK,KAAK,SAAS;AACtC,UACE,kBAAiB,wBAAwB,IAAI,KAAK,IAAI,MACrD,CAAC,mBAAmB,gBAAgB,IAAI,KAAK,IAAI,MAClD,CAAC,KAAK,gBAAgB,IAAI,GAAG,MAC5B,KAAK,WAAW,QAAQ,KAAK,WAAW,UACzC;AACA,aAAK,gBAAgB,IAAI,GAAG;AAC5B,eAAO;AAAA,MACT;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,0BAA0B,MAAc,QAA0C;AACxF,eAAW,QAAQ,KAAK,QAAQ,OAAO,GAAG;AACxC,UAAI,KAAK,SAAS,mBAAoB;AACtC,YAAM,IAAI,SAAS,IAAI;AACvB,YAAM,SAAS,EAAE,eAAe,KAAK,EAAE,cAAc;AACrD,UAAI,WAAW,KAAM;AACrB,UAAI,WAAW,UAAW,QAAO;AACjC,UAAI,WAAW,YAAa,QAAO,KAAK,WAAW,QAAQ,KAAK,WAAW;AAAA,IAC7E;AACA,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/stores/eval-definition-loader.d.ts

@@ -0,0 +1,4 @@
+import type { EvalDefinition } from '../definition';
+export type EvalDefinitionLoader = () => Promise<EvalDefinition[]>;
+export declare function createDiskEvalLoader(evalsDir: string): EvalDefinitionLoader;
+//# sourceMappingURL=eval-definition-loader.d.ts.map

package/dist/stores/eval-definition-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-definition-loader.d.ts","sourceRoot":"","sources":["../../src/stores/eval-definition-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAGnD,MAAM,MAAM,oBAAoB,GAAG,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;AAElE,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,oBAAoB,CAE3E"}

package/dist/stores/eval-definition-loader.js

@@ -0,0 +1,104 @@
+// src/loader.ts
+import { readdirSync, existsSync } from "fs";
+import { resolve } from "path";
+
+// src/definition.ts
+var Eval = class {
+  name;
+  description;
+  tags;
+  type;
+  setup;
+  conversation;
+  outcome;
+  options;
+  constructor(def) {
+    this.name = def.name;
+    this.conversation = def.conversation;
+    if (def.description !== void 0) this.description = def.description;
+    if (def.tags !== void 0) this.tags = def.tags;
+    if (def.type !== void 0) this.type = def.type;
+    if (def.setup !== void 0) this.setup = def.setup;
+    if (def.outcome !== void 0) this.outcome = def.outcome;
+    if (def.options !== void 0) this.options = def.options;
+  }
+};
+
+// src/errors.ts
+import { AdkError } from "@botpress/analytics";
+var EvalRunnerError = class extends AdkError {
+};
+
+// src/loader.ts
+function isEvalDefinition(value) {
+  return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation);
+}
+async function loadEvalFile(filePath) {
+  const absPath = resolve(filePath);
+  let mod;
+  try {
+    mod = await import(absPath);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    throw new EvalRunnerError({
+      code: "EVAL_LOAD_FAILED",
+      message: `Failed to load eval file ${filePath}: ${msg}
+
+Make sure your eval file:
+  - Has no syntax or type errors
+  - Exports one or more \`new Eval({...})\` instances
+  - Has all dependencies installed (\`bun install\`)`,
+      expected: true,
+      cause: err
+    });
+  }
+  const results = [];
+  for (const [key, value] of Object.entries(mod)) {
+    if (key === "__esModule") continue;
+    if (value instanceof Eval || isEvalDefinition(value)) {
+      results.push(value);
+    }
+  }
+  if (results.length === 0) {
+    throw new EvalRunnerError({
+      code: "EVAL_FILE_EMPTY",
+      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`,
+      expected: true
+    });
+  }
+  return results;
+}
+async function loadEvalsFromDir(dirPath) {
+  const absDir = resolve(dirPath);
+  if (!existsSync(absDir)) {
+    return [];
+  }
+  const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts"));
+  const evals = [];
+  for (const f of files) {
+    const defs = await loadEvalFile(`${absDir}/${f}`);
+    evals.push(...defs);
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const e of evals) {
+    if (seen.has(e.name)) {
+      throw new EvalRunnerError({
+        code: "EVAL_DUPLICATE_NAME",
+        message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`,
+        expected: true,
+        details: { name: e.name }
+      });
+    }
+    seen.add(e.name);
+  }
+  return evals;
+}
+
+// src/stores/eval-definition-loader.ts
+function createDiskEvalLoader(evalsDir) {
+  return () => loadEvalsFromDir(evalsDir);
+}
+export {
+  createDiskEvalLoader
+};
+//# sourceMappingURL=eval-definition-loader.js.map

package/dist/stores/eval-definition-loader.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/loader.ts", "../../src/definition.ts", "../../src/errors.ts", "../../src/stores/eval-definition-loader.ts"],
+  "sourcesContent": ["/**\n * Eval file loader.\n * Loads *.eval.ts files from a directory using dynamic imports.\n * Supports both default and named exports, and multiple evals per file.\n */\n\nimport { readdirSync, existsSync } from 'fs'\nimport { resolve } from 'path'\nimport type { EvalDefinition, EvalFilter } from './types'\nimport { Eval } from './types'\nimport { EvalRunnerError } from './errors'\n\nfunction isEvalDefinition(value: unknown): value is EvalDefinition {\n  return (\n    value !== null &&\n    typeof value === 'object' &&\n    typeof (value as Record<string, unknown>).name === 'string' &&\n    (value as Record<string, unknown>).name !== '' &&\n    Array.isArray((value as Record<string, unknown>).conversation)\n  )\n}\n\nexport async function loadEvalFile(filePath: string): Promise<EvalDefinition[]> {\n  const absPath = resolve(filePath)\n\n  let mod: Record<string, unknown>\n  try {\n    mod = await import(absPath)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    // Guidance stays in the message until the CLI renders `suggestion`\n    // (cli-package remediation) \u2014 report.error/logger.fatal only show message.\n    throw new EvalRunnerError({\n      code: 'EVAL_LOAD_FAILED',\n      message:\n        `Failed to load eval file ${filePath}: ${msg}\\n\\n` +\n        'Make sure your eval file:\\n' +\n        '  - Has no syntax or type errors\\n' +\n        '  - Exports one or more `new Eval({...})` instances\\n' +\n        '  - Has all dependencies installed (`bun install`)',\n      expected: true,\n      cause: err,\n    })\n  }\n\n  const results: EvalDefinition[] = []\n\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === '__esModule') continue\n    if (value instanceof Eval || isEvalDefinition(value)) {\n      results.push(value)\n    }\n  }\n\n  if (results.length === 0) {\n    throw new EvalRunnerError({\n      code: 'EVAL_FILE_EMPTY',\n      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \\`new Eval({...})\\` instances (as default or named exports).`,\n      expected: true,\n    })\n  }\n\n  return results\n}\n\nexport async function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) {\n    return []\n  }\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n  const evals: EvalDefinition[] = []\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    evals.push(...defs)\n  }\n\n  const seen = new Set<string>()\n  for (const e of evals) {\n    if (seen.has(e.name)) {\n      throw new EvalRunnerError({\n        code: 'EVAL_DUPLICATE_NAME',\n        message: `Duplicate eval name \"${e.name}\" found in ${dirPath} \u2014 names must be unique across the evals directory.`,\n        expected: true,\n        details: { name: e.name },\n      })\n    }\n    seen.add(e.name)\n  }\n\n  return evals\n}\n\nexport async function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) return null\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    const found = defs.find((d) => d.name === name)\n    if (found) return found\n  }\n\n  return null\n}\n\nexport function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[] {\n  if (!filter) return evals\n\n  return evals.filter((e) => {\n    if (filter.names && filter.names.length > 0) {\n      if (!filter.names.includes(e.name)) return false\n    }\n    if (filter.tags && filter.tags.length > 0) {\n      if (!e.tags || !filter.tags.some((t) => e.tags!.includes(t))) return false\n    }\n    if (filter.type) {\n      if (e.type !== filter.type) return false\n    }\n    return true\n  })\n}\n", "export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { payload: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n  /**\n   * Run this turn in a NEW conversation under the same user. The prior\n   * transcript is absent, so recall here can only come from user-scoped\n   * persistence, not context. Ignored on the first turn.\n   */\n  newConversation?: boolean\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n", "import type { EvalDefinition } from '../definition'\nimport { loadEvalsFromDir } from '../loader'\n\nexport type EvalDefinitionLoader = () => Promise<EvalDefinition[]>\n\nexport function createDiskEvalLoader(evalsDir: string): EvalDefinitionLoader {\n  return () => loadEvalsFromDir(evalsDir)\n}\n"],
+  "mappings": ";AAMA,SAAS,aAAa,kBAAkB;AACxC,SAAS,eAAe;;;ACuFjB,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAMT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;ACtHA,SAAS,gBAAgB;AA4BlB,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;AFhB9D,SAAS,iBAAiB,OAAyC;AACjE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAQ,MAAkC,SAAS,YAClD,MAAkC,SAAS,MAC5C,MAAM,QAAS,MAAkC,YAAY;AAEjE;AAEA,eAAsB,aAAa,UAA6C;AAC9E,QAAM,UAAU,QAAQ,QAAQ;AAEhC,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,OAAO;AAAA,EACrB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAG3D,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SACE,4BAA4B,QAAQ,KAAK,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAK9C,UAAU;AAAA,MACV,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,QAAM,UAA4B,CAAC;AAEnC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,QAAQ,aAAc;AAC1B,QAAI,iBAAiB,QAAQ,iBAAiB,KAAK,GAAG;AACpD,cAAQ,KAAK,KAAK;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS,qBAAqB,QAAQ;AAAA,MACtC,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAiB,SAA4C;AACjF,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AACtE,QAAM,QAA0B,CAAC;AAEjC,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,KAAK,GAAG,IAAI;AAAA,EACpB;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,OAAO;AACrB,QAAI,KAAK,IAAI,EAAE,IAAI,GAAG;AACpB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS,wBAAwB,EAAE,IAAI,cAAc,OAAO;AAAA,QAC5D,UAAU;AAAA,QACV,SAAS,EAAE,MAAM,EAAE,KAAK;AAAA,MAC1B,CAAC;AAAA,IACH;AACA,SAAK,IAAI,EAAE,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;;;AGxFO,SAAS,qBAAqB,UAAwC;AAC3E,SAAO,MAAM,iBAAiB,QAAQ;AACxC;",
+  "names": []
+}

package/dist/stores/eval-store.d.ts

@@ -0,0 +1,61 @@
+import type { EvalDefinition } from '../definition';
+import type { EvalFilter, EvalProgressEvent, EvalReport, EvalRunReport } from '../types';
+export interface EvalSummary {
+    name: string;
+    description?: string;
+    tags: string[];
+    type: 'capability' | 'regression';
+    turnCount: number;
+    hasOutcome: boolean;
+}
+export interface EvalRunSummary {
+    id: string;
+    timestamp: string;
+    passed: number;
+    failed: number;
+    total: number;
+    duration: number;
+    botDuration: number;
+    evalDuration: number;
+    filter?: EvalFilter;
+    evalNames: string[];
+    aborted: boolean;
+}
+export interface EvalReportHistoryEntry {
+    runId: string;
+    timestamp: string;
+    report: EvalReport;
+    totalEvalsInRun: number;
+    filter?: EvalFilter;
+}
+export interface EvalWatchOptions {
+    runId?: string;
+    workflowId?: string;
+}
+export interface EvalStore {
+    listEvals(filter?: EvalFilter): Promise<EvalSummary[]>;
+    getEval(name: string): Promise<EvalDefinition | null>;
+    createRun(runType?: string, metadata?: Record<string, unknown>): Promise<string>;
+    addRunResults(runId: string, evalReport: EvalReport): Promise<void>;
+    completeRun(runId: string, report: EvalRunReport): Promise<void>;
+    loadRunResult(runId: string): Promise<EvalRunReport | null>;
+    getLatestRun(): Promise<EvalRunReport | null>;
+    listRunSummaries(opts?: {
+        limit?: number;
+        since?: number;
+    }): Promise<EvalRunSummary[]>;
+    listEvalReportsByName(evalName: string, opts?: {
+        limit?: number;
+        since?: number;
+    }): Promise<EvalReportHistoryEntry[]>;
+    listEvalReportsBulk(opts?: {
+        perEval?: number;
+        since?: number;
+    }): Promise<Record<string, EvalReportHistoryEntry[]>>;
+    watchRun(signal?: AbortSignal, options?: EvalWatchOptions): AsyncIterable<EvalProgressEvent>;
+    getRunnerState(): Promise<{
+        running: boolean;
+        runId: string | null;
+    }>;
+}
+//# sourceMappingURL=eval-store.d.ts.map

package/dist/stores/eval-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-store.d.ts","sourceRoot":"","sources":["../../src/stores/eval-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAIxF,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,IAAI,EAAE,YAAY,GAAG,YAAY,CAAA;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,OAAO,CAAA;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,EAAE,MAAM,EAAE,CAAA;IACnB,OAAO,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,UAAU,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,CAAC,EAAE,UAAU,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAID,MAAM,WAAW,SAAS;IAExB,SAAS,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;IACtD,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAA;IAGrD,SAAS,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAChF,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACnE,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAGhE,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAA;IAC3D,YAAY,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAA;IAC7C,gBAAgB,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAA;IACtF,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAAA;IACrH,mBAAmB,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC,CAAA;IAGnH,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,gBAAgB,GAAG,aAAa,CAAC,iBAAiB,CAAC,CAAA;IAG5F,cAAc,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAA;CACtE"}

package/dist/stores/eval-store.js

@@ -0,0 +1 @@
+//# sourceMappingURL=eval-store.js.map

package/dist/stores/eval-store.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/stores/index.d.ts

@@ -0,0 +1,8 @@
+export type { EvalStore, EvalSummary, EvalRunSummary, EvalReportHistoryEntry, EvalWatchOptions } from './eval-store';
+export { LocalEvalStore, getLocalEvalStore, closeLocalEvalStores } from './local-eval-store';
+export type { LocalEvalStoreConfig } from './local-eval-store';
+export { VortexEvalStore } from './vortex-eval-store';
+export type { VortexEvalStoreConfig } from './vortex-eval-store';
+export { createDiskEvalLoader } from './eval-definition-loader';
+export type { EvalDefinitionLoader } from './eval-definition-loader';
+//# sourceMappingURL=index.d.ts.map

package/dist/stores/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stores/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AACpH,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAC5F,YAAY,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,YAAY,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAA;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAC/D,YAAY,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA"}