@botpress/evals 1.19.0-beta.9 → 2.0.0-alpha.0

package/dist/stores/vortex-eval-store-entry.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/stores/vortex-eval-store.ts", "../../src/loader.ts", "../../src/definition.ts", "../../src/errors.ts", "../../src/stores/eval-definition-loader.ts"],
+  "sourcesContent": ["import type { EvalDefinition } from '../definition'\nimport type { EvalFilter, EvalProgressEvent, EvalReport, EvalRunReport, TurnReport, GraderResult } from '../types'\nimport type { EvalStore, EvalSummary, EvalRunSummary, EvalReportHistoryEntry, EvalWatchOptions } from './eval-store'\nimport type { EvalDefinitionLoader } from './eval-definition-loader'\n\n// \u2500\u2500 Vortex response types \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\ninterface VortexEvalRun {\n  id: string\n  botId: string\n  workspaceId: string\n  evalManifestId: string\n  status: 'pending' | 'running' | 'completed' | 'failed'\n  triggerType: string\n  startedAt: string | null\n  completedAt: string | null\n  createdAt: string\n  updatedAt: string\n  metadata?: Record<string, unknown> | null\n}\n\ninterface VortexEvalResult {\n  id: string\n  evalEntryId: string\n  turnIndex: number\n  graderName: string\n  passed: boolean\n  score: number | null\n  evidence: Record<string, unknown>\n  userMessage: string\n  botResponse: string\n  botDurationMs: number | null\n  graderDurationMs: number | null\n  createdAt: string\n}\n\ninterface VortexEvalEntry {\n  id: string\n  evalRunId: string\n  evalName: string\n  evalType: 'capability' | 'regression'\n  description: string\n  tags: string[]\n  /** null while the entry is in-progress (per-turn lifecycle); set on finalize. */\n  passed: boolean | null\n  durationMs: number | null\n  error: string | null\n  createdAt: string\n  results: VortexEvalResult[]\n}\n\n/** A single grader result row, as posted to Vortex. */\ninterface VortexResultInput {\n  turnIndex: number\n  graderName: string\n  passed: boolean\n  evidence: { expected: string; actual: string }\n  userMessage: string\n  botResponse: string\n  botDurationMs: number\n  graderDurationMs: number\n}\n\ntype VortexRunWithEntries = VortexEvalRun & { entries: VortexEvalEntry[] }\n\nconst OUTCOME_TURN_INDEX = -1\n\n// \u2500\u2500 Config \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport interface VortexEvalStoreConfig {\n  url: string\n  botId: string\n  workspaceId?: string\n  token?: string\n  evalManifestId?: string | (() => string | undefined)\n  loadEvalDefinitions?: EvalDefinitionLoader\n}\n\n// \u2500\u2500 Transformers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nfunction vortexEntryToEvalReport(entry: VortexEvalEntry): EvalReport {\n  const outcomeAssertions: GraderResult[] = entry.results\n    .filter((r) => r.turnIndex === OUTCOME_TURN_INDEX)\n    .map((r) => ({\n      assertion: r.graderName,\n      pass: r.passed,\n      expected: (r.evidence?.expected as string) ?? '',\n      actual: (r.evidence?.actual as string) ?? '',\n    }))\n\n  const resultsByTurn = new Map<number, VortexEvalResult[]>()\n  for (const r of entry.results) {\n    if (r.turnIndex === OUTCOME_TURN_INDEX) continue\n    const existing = resultsByTurn.get(r.turnIndex) ?? []\n    existing.push(r)\n    resultsByTurn.set(r.turnIndex, existing)\n  }\n\n  const turns: TurnReport[] = [...resultsByTurn.entries()]\n    .sort(([a], [b]) => a - b)\n    .map(([turnIndex, results]) => {\n      const assertions: GraderResult[] = results.map((r) => ({\n        assertion: r.graderName,\n        pass: r.passed,\n        expected: (r.evidence?.expected as string) ?? '',\n        actual: (r.evidence?.actual as string) ?? '',\n      }))\n      const first = results[0]!\n      return {\n        turnIndex,\n        userMessage: first.userMessage,\n        botResponse: first.botResponse,\n        assertions,\n        pass: assertions.every((a) => a.pass),\n        botDuration: first.botDurationMs ?? 0,\n        evalDuration: first.graderDurationMs ?? 0,\n      }\n    })\n\n  return {\n    name: entry.evalName,\n    description: entry.description || undefined,\n    type: entry.evalType,\n    tags: entry.tags,\n    turns,\n    outcomeAssertions,\n    pass: entry.passed ?? false,\n    duration: entry.durationMs ?? 0,\n    error: entry.error ?? undefined,\n  }\n}\n\n/** Run an async fn over items with bounded concurrency, preserving input order. */\nasync function mapWithConcurrency<T, R>(items: T[], concurrency: number, fn: (item: T) => Promise<R>): Promise<R[]> {\n  const results = new Array<R>(items.length)\n  let next = 0\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {\n    while (next < items.length) {\n      const i = next++\n      results[i] = await fn(items[i]!)\n    }\n  })\n  await Promise.all(workers)\n  return results\n}\n\n/** Map one turn's assertions to Vortex result rows (shared by batch + per-turn writes). */\nfunction turnToResultRows(turn: TurnReport): VortexResultInput[] {\n  return graderResultsToRows(turn.assertions, {\n    turnIndex: turn.turnIndex,\n    userMessage: turn.userMessage,\n    botResponse: turn.botResponse,\n    botDurationMs: turn.botDuration,\n    graderDurationMs: turn.evalDuration,\n  })\n}\n\nfunction outcomeToResultRows(assertions: GraderResult[]): VortexResultInput[] {\n  return graderResultsToRows(assertions, {\n    turnIndex: OUTCOME_TURN_INDEX,\n    userMessage: '',\n    botResponse: '',\n    botDurationMs: 0,\n    graderDurationMs: 0,\n  })\n}\n\nfunction graderResultsToRows(\n  assertions: GraderResult[],\n  context: Omit<VortexResultInput, 'graderName' | 'passed' | 'evidence'>\n): VortexResultInput[] {\n  return assertions.map((assertion) => ({\n    ...context,\n    graderName: assertion.assertion,\n    passed: assertion.pass,\n    evidence: { expected: assertion.expected, actual: assertion.actual },\n  }))\n}\n\nfunction runMatchesWatchOptions(run: VortexEvalRun, options: EvalWatchOptions): boolean {\n  if (options.runId && run.id !== options.runId) return false\n  if (options.workflowId && run.metadata?.workflowId !== options.workflowId) return false\n  return true\n}\n\nfunction vortexRunToReport(run: VortexRunWithEntries): EvalRunReport {\n  const evals = run.entries.map(vortexEntryToEvalReport)\n  return {\n    id: run.id,\n    timestamp: run.createdAt,\n    evals,\n    passed: evals.filter((e) => e.pass).length,\n    failed: evals.filter((e) => !e.pass).length,\n    total: evals.length,\n    duration: evals.reduce((s, e) => s + e.duration, 0),\n  }\n}\n\nfunction vortexRunToSummary(run: VortexEvalRun & { entries?: VortexEvalEntry[] }): EvalRunSummary {\n  const startMs = run.startedAt ? new Date(run.startedAt).getTime() : 0\n  const endMs = run.completedAt ? new Date(run.completedAt).getTime() : 0\n\n  if (run.entries) {\n    const evals = run.entries.map(vortexEntryToEvalReport)\n    return {\n      id: run.id,\n      timestamp: run.createdAt,\n      passed: evals.filter((e) => e.pass).length,\n      failed: evals.filter((e) => !e.pass).length,\n      total: evals.length,\n      duration: evals.reduce((s, e) => s + e.duration, 0),\n      botDuration: evals.reduce((s, e) => e.turns.reduce((ts, t) => ts + t.botDuration, s), 0),\n      evalDuration: evals.reduce((s, e) => e.turns.reduce((ts, t) => ts + t.evalDuration, s), 0),\n      evalNames: evals.map((e) => e.name),\n      aborted: false,\n    }\n  }\n\n  return {\n    id: run.id,\n    timestamp: run.createdAt,\n    passed: 0,\n    failed: 0,\n    total: 0,\n    duration: startMs && endMs ? endMs - startMs : 0,\n    botDuration: 0,\n    evalDuration: 0,\n    evalNames: [],\n    aborted: false,\n  }\n}\n\n// \u2500\u2500 VortexEvalStore \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport class VortexEvalStore implements EvalStore {\n  private url: string\n  private botId: string\n  private workspaceId?: string\n  private token?: string\n  private _evalManifestId?: string | (() => string | undefined)\n  private _loadEvalDefinitions?: EvalDefinitionLoader\n\n  constructor(config: VortexEvalStoreConfig) {\n    this.url = config.url.replace(/\\/$/, '')\n    this.botId = config.botId\n    if (config.workspaceId) this.workspaceId = config.workspaceId\n    if (config.token) this.token = config.token\n    this._evalManifestId = config.evalManifestId\n    this._loadEvalDefinitions = config.loadEvalDefinitions\n  }\n\n  // \u2500\u2500 Eval definitions (from Files API manifest) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  async listEvals(filter?: EvalFilter): Promise<EvalSummary[]> {\n    const defs = await this.getEvalDefinitions()\n    const filtered = defs.filter((e) => {\n      if (filter?.names && !filter.names.includes(e.name)) return false\n      if (filter?.type && e.type !== filter.type) return false\n      if (filter?.tags && !filter.tags.every((tag) => e.tags?.includes(tag))) return false\n      return true\n    })\n    return filtered.map((e) => ({\n      name: e.name,\n      description: e.description,\n      tags: e.tags || [],\n      type: e.type || 'capability',\n      turnCount: e.conversation.length,\n      hasOutcome: !!e.outcome,\n    }))\n  }\n\n  async getEval(name: string): Promise<EvalDefinition | null> {\n    const defs = await this.getEvalDefinitions()\n    return defs.find((e) => e.name === name) ?? null\n  }\n\n  // \u2500\u2500 Run lifecycle \u2014 write \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  async createRun(runType = 'scheduled', metadata?: Record<string, unknown>): Promise<string> {\n    if (!this.workspaceId) {\n      throw new Error('workspaceId is required to create Vortex eval runs')\n    }\n\n    const triggerType = runType === 'manual' ? 'manual' : 'scheduled'\n    const res = await this.fetch(`/v1/evals/bot/${this.botId}/runs`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({\n        evalManifestId:\n          (typeof this._evalManifestId === 'function' ? this._evalManifestId() : this._evalManifestId) ?? '',\n        workspaceId: this.workspaceId,\n        triggerType,\n        ...(metadata ? { metadata } : {}),\n      }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex create run failed: ${res.status} ${text}`)\n    }\n\n    const { id } = (await res.json()) as { id: string }\n    return id\n  }\n\n  async addRunResults(runId: string, evalReport: EvalReport): Promise<void> {\n    const entry = {\n      evalName: evalReport.name,\n      evalType: evalReport.type ?? 'capability',\n      description: evalReport.description ?? '',\n      tags: evalReport.tags ?? [],\n      passed: evalReport.pass,\n      durationMs: evalReport.duration,\n      error: evalReport.error,\n      results: [...evalReport.turns.flatMap(turnToResultRows), ...outcomeToResultRows(evalReport.outcomeAssertions)],\n    }\n\n    const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ entries: [entry] }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex ingest entries failed: ${res.status} ${text}`)\n    }\n  }\n\n  async completeRun(runId: string, report: EvalRunReport): Promise<void> {\n    const hasRunError = report.aborted === true || report.evals.some((e) => e.error !== undefined)\n    const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({\n        entries: [],\n        completed: !hasRunError,\n        failed: hasRunError,\n      }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex complete run failed: ${res.status} ${text}`)\n    }\n  }\n\n  // \u2500\u2500 Per-turn incremental lifecycle (Option B) \u2500\u2500\n  //\n  // The eval-runner workflow builds each entry up as the eval runs (rather than\n  // writing it whole at the end), so the dev console can stream turn-by-turn\n  // progress. This is the prod ingestion path and requires Vortex to provide:\n  //   startEntry        POST  /runs/{runId}/entries                    \u2192 { entries:[{ id }] }  (passed:null, in-progress)\n  //   appendTurnResults POST  /runs/{runId}/entries/{entryId}/results  (idempotent upsert by turnIndex+graderName)\n  //   finalizeEntry     PATCH /runs/{runId}/entries/{entryId}          (sets the verdict)\n  //   markRunComplete   POST  /runs/{runId}/complete                   (status \u2192 completed|failed)\n\n  /** Create an in-progress entry (metadata only, no results) and return its server id. */\n  async startEntry(\n    runId: string,\n    meta: { evalName: string; evalType?: 'capability' | 'regression'; description?: string; tags?: string[] }\n  ): Promise<string> {\n    const res = await this.fetch(`/v1/evals/runs/${runId}/entries`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({\n        entries: [\n          {\n            evalName: meta.evalName,\n            evalType: meta.evalType ?? 'capability',\n            description: meta.description ?? '',\n            tags: meta.tags ?? [],\n          },\n        ],\n      }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex start entry failed: ${res.status} ${text}`)\n    }\n\n    const data = (await res.json()) as { entries?: { id: string }[]; id?: string }\n    const entryId = data.entries?.[0]?.id ?? data.id\n    if (!entryId) {\n      throw new Error('Vortex start entry returned no entry id')\n    }\n    return entryId\n  }\n\n  /** Append one turn's grader results to an existing entry. */\n  async appendTurnResults(runId: string, entryId: string, turn: TurnReport): Promise<void> {\n    const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}/results`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ results: turnToResultRows(turn) }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex append results failed: ${res.status} ${text}`)\n    }\n  }\n\n  /** Append outcome-level grader results to an existing entry. */\n  async appendOutcomeResults(runId: string, entryId: string, assertions: GraderResult[]): Promise<void> {\n    if (assertions.length === 0) return\n\n    const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}/results`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ results: outcomeToResultRows(assertions) }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex append results failed: ${res.status} ${text}`)\n    }\n  }\n\n  /** Set the verdict on an entry once its eval finishes. */\n  async finalizeEntry(\n    runId: string,\n    entryId: string,\n    verdict: { passed: boolean; durationMs?: number; error?: string }\n  ): Promise<void> {\n    const res = await this.fetch(`/v1/evals/runs/${runId}/entries/${entryId}`, {\n      method: 'PATCH',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({\n        passed: verdict.passed,\n        ...(verdict.durationMs !== undefined ? { durationMs: verdict.durationMs } : {}),\n        ...(verdict.error !== undefined ? { error: verdict.error } : {}),\n      }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex finalize entry failed: ${res.status} ${text}`)\n    }\n  }\n\n  /** Flip the run to its terminal status (replaces completeRun's empty-entries hack). */\n  async markRunComplete(runId: string, opts: { failed?: boolean } = {}): Promise<void> {\n    const res = await this.fetch(`/v1/evals/runs/${runId}/complete`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/json' },\n      body: JSON.stringify({ ...(opts.failed !== undefined ? { failed: opts.failed } : {}) }),\n    })\n\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex complete run failed: ${res.status} ${text}`)\n    }\n  }\n\n  // \u2500\u2500 Run results \u2014 read \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  async loadRunResult(runId: string): Promise<EvalRunReport | null> {\n    const data = await this.fetchJson<VortexRunWithEntries>(`/v1/evals/runs/${runId}`)\n    return vortexRunToReport(data)\n  }\n\n  async getLatestRun(): Promise<EvalRunReport | null> {\n    const data = await this.fetchJson<{ runs: VortexEvalRun[] }>(`/v1/evals/bot/${this.botId}/runs`, { limit: '1' })\n    if (!data.runs.length) return null\n    const latest = data.runs[0]!\n    return this.loadRunResult(latest.id)\n  }\n\n  async listRunSummaries(opts?: { limit?: number; since?: number }): Promise<EvalRunSummary[]> {\n    // Vortex's list endpoint returns runs WITHOUT entries, so summary counts\n    // would all be zero \u2014 which the heatmap renders as \"No evals\". Pull each\n    // run's entries to compute real pass/fail/total. N+1, bounded by the list\n    // size; a counts field on the list endpoint would let us drop the per-run\n    // fetch.\n    const runs = await this.fetchRunsWithEntries(opts?.limit ?? 50, opts?.since)\n    return runs.map(vortexRunToSummary)\n  }\n\n  async listEvalReportsByName(\n    evalName: string,\n    opts?: { limit?: number; since?: number }\n  ): Promise<EvalReportHistoryEntry[]> {\n    const limit = opts?.limit ?? 20\n    const runs = await this.fetchRunsWithEntries(50, opts?.since)\n\n    const entries: EvalReportHistoryEntry[] = []\n    for (const run of runs) {\n      if (entries.length >= limit) break\n      const entry = run.entries.find((e) => e.evalName === evalName)\n      if (entry) {\n        entries.push({\n          runId: run.id,\n          timestamp: run.createdAt,\n          report: vortexEntryToEvalReport(entry),\n          totalEvalsInRun: run.entries.length,\n        })\n      }\n    }\n    return entries\n  }\n\n  async listEvalReportsBulk(opts?: {\n    perEval?: number\n    since?: number\n  }): Promise<Record<string, EvalReportHistoryEntry[]>> {\n    const per = opts?.perEval ?? 20\n    const runs = await this.fetchRunsWithEntries(50, opts?.since)\n\n    const grouped: Record<string, EvalReportHistoryEntry[]> = {}\n    for (const run of runs) {\n      for (const entry of run.entries) {\n        if (!grouped[entry.evalName]) grouped[entry.evalName] = []\n        if (grouped[entry.evalName]!.length < per) {\n          grouped[entry.evalName]!.push({\n            runId: run.id,\n            timestamp: run.createdAt,\n            report: vortexEntryToEvalReport(entry),\n            totalEvalsInRun: run.entries.length,\n          })\n        }\n      }\n    }\n    return grouped\n  }\n\n  // \u2500\u2500 Live run tracking \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  async *watchRun(signal?: AbortSignal, options: EvalWatchOptions = {}): AsyncIterable<EvalProgressEvent> {\n    // Poll fast enough that batched turns don't arrive in a single visible\n    // burst \u2014 the prod path can't push per-event like the dev SSE stream, so a\n    // tighter cadence is what keeps it feeling live.\n    const POLL_INTERVAL = 1_500\n    const POLL_TIMEOUT = 60 * 60 * 1_000 // 60min\n    const start = Date.now()\n\n    // Capture the run that's already the latest before this invocation begins.\n    // The eval-runner workflow now creates its Vortex run UP FRONT (before\n    // running the suite), so a genuinely new run appears almost immediately \u2014\n    // but until it does, the \"latest\" run is the PREVIOUS one. Reporting that\n    // would surface a stale, mismatched run (the runner shows eval \"A\" while\n    // watchRun reports the prior run of eval \"B\", and the per-eval replay\n    // filter renders \"no results\"). Only act on a run whose id differs from\n    // this baseline.\n    const hasExpectedRun = !!options.runId || !!options.workflowId\n    let baselineRunId: string | null = null\n    if (!hasExpectedRun) {\n      try {\n        const initial = await this.fetchJson<{ runs: VortexEvalRun[] }>(`/v1/evals/bot/${this.botId}/runs`, {\n          limit: '1',\n        })\n        baselineRunId = initial.runs[0]?.id ?? null\n      } catch {\n        // Couldn't read the baseline; fall back to reporting the first completed run.\n      }\n    }\n\n    // Diff the run on every poll and stream progress: `eval_start` when an\n    // entry first appears, `turn_start` for the next in-flight turn (synthesized\n    // from the definition so the pending-turn UI shows before results land),\n    // `turn_complete` as each turn's results arrive, and `eval_complete` once\n    // the entry is finalized (verdict set \u2192 `passed` is non-null).\n    // `suite_complete` fires when the run reaches a terminal status. (Depends on\n    // the per-turn lifecycle endpoints; see the write methods above.)\n    const evalMeta = await this.loadEvalMeta()\n    const startedEvals = new Set<string>()\n    const seenTurns = new Set<string>()\n    const startedTurns = new Set<string>()\n    const finalizedEvals = new Set<string>()\n\n    while (!signal?.aborted && Date.now() - start < POLL_TIMEOUT) {\n      try {\n        const data = await this.fetchJson<{ runs: VortexEvalRun[] }>(`/v1/evals/bot/${this.botId}/runs`, {\n          limit: hasExpectedRun ? '20' : '1',\n        })\n        let latest: VortexEvalRun | null = null\n        let raw: VortexRunWithEntries | null = null\n        for (const candidate of data.runs) {\n          if (options.runId && candidate.id !== options.runId) continue\n          if (options.workflowId && candidate.metadata?.workflowId !== options.workflowId) {\n            const candidateRaw = await this.fetchJson<VortexRunWithEntries>(`/v1/evals/runs/${candidate.id}`)\n            if (!runMatchesWatchOptions(candidateRaw, options)) continue\n            raw = candidateRaw\n          }\n          latest = candidate\n          break\n        }\n\n        // Still seeing the pre-existing run (or none yet) \u2014 the new run hasn't\n        // been created. Wait and retry.\n        if (!latest || (!hasExpectedRun && latest.id === baselineRunId)) {\n          await new Promise((r) => setTimeout(r, POLL_INTERVAL))\n          continue\n        }\n\n        raw ??= await this.fetchJson<VortexRunWithEntries>(`/v1/evals/runs/${latest.id}`)\n        const terminal = raw.status === 'completed' || raw.status === 'failed'\n\n        for (let i = 0; i < raw.entries.length; i++) {\n          const entry = raw.entries[i]!\n          const evalReport = vortexEntryToEvalReport(entry)\n          const meta = evalMeta.get(entry.evalName)\n          const totalTurns = meta?.totalTurns ?? evalReport.turns.length\n\n          if (!startedEvals.has(entry.evalName)) {\n            startedEvals.add(entry.evalName)\n            yield { type: 'eval_start', evalName: entry.evalName, index: i, totalTurns }\n          }\n          for (const turn of evalReport.turns) {\n            const turnKey = `${entry.evalName}::${turn.turnIndex}`\n            if (seenTurns.has(turnKey)) continue\n            seenTurns.add(turnKey)\n            yield {\n              type: 'turn_complete',\n              evalName: entry.evalName,\n              evalIndex: i,\n              turnIndex: turn.turnIndex,\n              totalTurns,\n              turnReport: turn,\n            }\n          }\n          // Verdict set (passed non-null) \u2192 the eval is done.\n          if (entry.passed != null && !finalizedEvals.has(entry.evalName)) {\n            finalizedEvals.add(entry.evalName)\n            yield { type: 'eval_complete', evalName: entry.evalName, index: i, report: evalReport }\n          } else if (entry.passed == null && !terminal) {\n            // Still running: announce the next expected turn so the UI renders\n            // the in-flight turn (its user message + spinner) instead of jumping\n            // straight from completed turns to the next completed turn. Results\n            // land per-turn, so the next index is simply the completed count.\n            const nextTurnIndex = evalReport.turns.length\n            const startKey = `${entry.evalName}::${nextTurnIndex}`\n            if (nextTurnIndex < totalTurns && !startedTurns.has(startKey)) {\n              startedTurns.add(startKey)\n              yield {\n                type: 'turn_start',\n                evalName: entry.evalName,\n                evalIndex: i,\n                turnIndex: nextTurnIndex,\n                totalTurns,\n                userMessage: meta?.userMessages[nextTurnIndex] ?? '',\n              }\n            }\n          }\n        }\n\n        if (terminal) {\n          const finalReport = vortexRunToReport(raw)\n          // Flush any eval that never received an explicit verdict (e.g. it\n          // errored out before finalizeEntry was called).\n          for (let i = 0; i < finalReport.evals.length; i++) {\n            const evalReport = finalReport.evals[i]!\n            if (finalizedEvals.has(evalReport.name)) continue\n            finalizedEvals.add(evalReport.name)\n            yield { type: 'eval_complete', evalName: evalReport.name, index: i, report: evalReport }\n          }\n          yield {\n            type: 'suite_complete',\n            report: finalReport,\n            error: raw.status === 'failed' ? 'Eval run failed' : undefined,\n          }\n          return\n        }\n      } catch {\n        // Poll failed, retry next interval\n      }\n\n      // Settled-but-not-terminal poll (or a failed poll): wait before the next.\n      // The \"no new run yet\" branch above already slept and continued, so this\n      // is the only other delay point.\n      await new Promise((r) => setTimeout(r, POLL_INTERVAL))\n    }\n\n    if (!signal?.aborted) {\n      yield {\n        type: 'suite_complete',\n        report: {\n          id: 'timeout',\n          timestamp: new Date().toISOString(),\n          evals: [],\n          passed: 0,\n          failed: 0,\n          total: 0,\n          duration: 0,\n        },\n        error: 'Eval run timed out',\n      }\n    }\n  }\n\n  // \u2500\u2500 Runner state \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  async getRunnerState(): Promise<{ running: boolean; runId: string | null }> {\n    try {\n      const data = await this.fetchJson<{ runs: VortexEvalRun[] }>(`/v1/evals/bot/${this.botId}/runs`, {\n        status: 'running',\n        limit: '1',\n      })\n      const active = data.runs[0] ?? null\n      return { running: !!active, runId: active?.id ?? null }\n    } catch {\n      return { running: false, runId: null }\n    }\n  }\n\n  // \u2500\u2500 Internal helpers \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\n  private async getEvalDefinitions(): Promise<EvalDefinition[]> {\n    if (!this._loadEvalDefinitions) return []\n    return this._loadEvalDefinitions()\n  }\n\n  /**\n   * evalName \u2192 { totalTurns, userMessages } from the eval definitions, so\n   * watchRun can report `totalTurns` and synthesize `turn_start` events (with\n   * the turn's user message) for the in-flight turn \u2014 matching the dev SSE\n   * experience where the pending turn is visible before its results land.\n   */\n  private async loadEvalMeta(): Promise<Map<string, { totalTurns: number; userMessages: string[] }>> {\n    try {\n      const defs = await this.getEvalDefinitions()\n      return new Map(\n        defs.map((d) => [\n          d.name,\n          { totalTurns: d.conversation.length, userMessages: d.conversation.map((t) => t.user ?? '') },\n        ])\n      )\n    } catch {\n      return new Map()\n    }\n  }\n\n  private async fetchRunsWithEntries(limit: number, since?: number): Promise<VortexRunWithEntries[]> {\n    const data = await this.fetchJson<{ runs: VortexEvalRun[] }>(`/v1/evals/bot/${this.botId}/runs`, {\n      limit: String(limit),\n    })\n\n    let runs = data.runs\n    if (since) {\n      runs = runs.filter((r) => new Date(r.createdAt).getTime() >= since)\n    }\n\n    // The list endpoint omits entries, so fetch each run's detail. Bounded\n    // concurrency keeps a wide `since` window from opening hundreds of sockets.\n    return mapWithConcurrency(runs, 8, (r) => this.fetchJson<VortexRunWithEntries>(`/v1/evals/runs/${r.id}`))\n  }\n\n  private async fetch(path: string, init?: RequestInit): Promise<Response> {\n    const target = new URL(`${this.url}${path}`)\n    const headers = new Headers(init?.headers)\n    if (this.token) headers.set('Authorization', `Bearer ${this.token}`)\n    return globalThis.fetch(target, { ...init, headers })\n  }\n\n  private async fetchJson<T>(path: string, params?: Record<string, string>): Promise<T> {\n    const target = new URL(`${this.url}${path}`)\n    if (params) {\n      for (const [k, v] of Object.entries(params)) target.searchParams.set(k, v)\n    }\n    const headers = new Headers()\n    if (this.token) headers.set('Authorization', `Bearer ${this.token}`)\n\n    const res = await globalThis.fetch(target, { headers })\n    if (!res.ok) {\n      const text = await res.text().catch(() => '')\n      throw new Error(`Vortex ${res.status}: ${text}`)\n    }\n    return res.json() as Promise<T>\n  }\n}\n", "/**\n * Eval file loader.\n * Loads *.eval.ts files from a directory using dynamic imports.\n * Supports both default and named exports, and multiple evals per file.\n */\n\nimport { readdirSync, existsSync } from 'fs'\nimport { resolve } from 'path'\nimport type { EvalDefinition, EvalFilter } from './types'\nimport { Eval } from './types'\nimport { EvalRunnerError } from './errors'\n\nfunction isEvalDefinition(value: unknown): value is EvalDefinition {\n  return (\n    value !== null &&\n    typeof value === 'object' &&\n    typeof (value as Record<string, unknown>).name === 'string' &&\n    (value as Record<string, unknown>).name !== '' &&\n    Array.isArray((value as Record<string, unknown>).conversation)\n  )\n}\n\nexport async function loadEvalFile(filePath: string): Promise<EvalDefinition[]> {\n  const absPath = resolve(filePath)\n\n  let mod: Record<string, unknown>\n  try {\n    mod = await import(absPath)\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    // Guidance stays in the message until the CLI renders `suggestion`\n    // (cli-package remediation) \u2014 report.error/logger.fatal only show message.\n    throw new EvalRunnerError({\n      code: 'EVAL_LOAD_FAILED',\n      message:\n        `Failed to load eval file ${filePath}: ${msg}\\n\\n` +\n        'Make sure your eval file:\\n' +\n        '  - Has no syntax or type errors\\n' +\n        '  - Exports one or more `new Eval({...})` instances\\n' +\n        '  - Has all dependencies installed (`bun install`)',\n      expected: true,\n      cause: err,\n    })\n  }\n\n  const results: EvalDefinition[] = []\n\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === '__esModule') continue\n    if (value instanceof Eval || isEvalDefinition(value)) {\n      results.push(value)\n    }\n  }\n\n  if (results.length === 0) {\n    throw new EvalRunnerError({\n      code: 'EVAL_FILE_EMPTY',\n      message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \\`new Eval({...})\\` instances (as default or named exports).`,\n      expected: true,\n    })\n  }\n\n  return results\n}\n\nexport async function loadEvalsFromDir(dirPath: string): Promise<EvalDefinition[]> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) {\n    return []\n  }\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n  const evals: EvalDefinition[] = []\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    evals.push(...defs)\n  }\n\n  const seen = new Set<string>()\n  for (const e of evals) {\n    if (seen.has(e.name)) {\n      throw new EvalRunnerError({\n        code: 'EVAL_DUPLICATE_NAME',\n        message: `Duplicate eval name \"${e.name}\" found in ${dirPath} \u2014 names must be unique across the evals directory.`,\n        expected: true,\n        details: { name: e.name },\n      })\n    }\n    seen.add(e.name)\n  }\n\n  return evals\n}\n\nexport async function loadEvalByName(dirPath: string, name: string): Promise<EvalDefinition | null> {\n  const absDir = resolve(dirPath)\n  if (!existsSync(absDir)) return null\n\n  const files = readdirSync(absDir).filter((f) => f.endsWith('.eval.ts'))\n\n  for (const f of files) {\n    const defs = await loadEvalFile(`${absDir}/${f}`)\n    const found = defs.find((d) => d.name === name)\n    if (found) return found\n  }\n\n  return null\n}\n\nexport function filterEvals(evals: EvalDefinition[], filter?: EvalFilter): EvalDefinition[] {\n  if (!filter) return evals\n\n  return evals.filter((e) => {\n    if (filter.names && filter.names.length > 0) {\n      if (!filter.names.includes(e.name)) return false\n    }\n    if (filter.tags && filter.tags.length > 0) {\n      if (!e.tags || !filter.tags.some((t) => e.tags!.includes(t))) return false\n    }\n    if (filter.type) {\n      if (e.type !== filter.type) return false\n    }\n    return true\n  })\n}\n", "export type MatchOperator =\n  | string\n  | { equals: unknown }\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { in: unknown[] }\n  | { exists: boolean }\n  | { gte: number }\n  | { lte: number }\n\nexport type ResponseAssertion =\n  | { contains: string }\n  | { not_contains: string }\n  | { matches: string }\n  | { llm_judge: string }\n// | { similar_to: string } TODO: uncomment this when it gets implemented\n\nexport type ToolAssertion =\n  | { called: string; params?: Record<string, MatchOperator> }\n  | { not_called: string }\n  | { call_order: string[] }\n\nexport interface StateAssertion {\n  path: string\n  equals?: unknown\n  changed?: boolean\n}\n\nexport interface WorkflowAssertion {\n  name: string\n  entered?: boolean\n  completed?: boolean\n}\n\nexport type NumericOperator = { equals: number } | { gte: number } | { lte: number }\n\nexport interface TimingAssertion {\n  response_time: NumericOperator\n}\n\nexport interface TurnAssertions {\n  response?: ResponseAssertion[]\n  tools?: ToolAssertion[]\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n  timing?: TimingAssertion[]\n}\n\nexport interface ConversationTurn {\n  user?: string\n  event?: { payload: Record<string, unknown> }\n  expectSilence?: boolean\n  assert?: TurnAssertions\n  /**\n   * Run this turn in a NEW conversation under the same user. The prior\n   * transcript is absent, so recall here can only come from user-scoped\n   * persistence, not context. Ignored on the first turn.\n   */\n  newConversation?: boolean\n}\n\nexport interface OutcomeAssertions {\n  state?: StateAssertion[]\n  workflow?: WorkflowAssertion[]\n}\n\nexport interface EvalSetup {\n  state?: {\n    bot?: Record<string, unknown>\n    user?: Record<string, unknown>\n    conversation?: Record<string, unknown>\n  }\n  workflow?: {\n    trigger: string\n    input?: Record<string, unknown>\n  }\n}\n\nexport interface EvalDefinition {\n  name: string\n  description?: string\n  tags?: string[]\n  type?: 'capability' | 'regression'\n  setup?: EvalSetup\n  conversation: ConversationTurn[]\n  outcome?: OutcomeAssertions\n  options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n}\n\nexport class Eval implements EvalDefinition {\n  readonly name: string\n  readonly description?: string\n  readonly tags?: string[]\n  readonly type?: 'capability' | 'regression'\n  readonly setup?: EvalSetup\n  readonly conversation: ConversationTurn[]\n  readonly outcome?: OutcomeAssertions\n  readonly options?: {\n    idleTimeout?: number\n    /** @deprecated No-op: the LLM judge is boolean (zai.check); there is no score threshold. */\n    judgePassThreshold?: number\n  }\n\n  constructor(def: EvalDefinition) {\n    this.name = def.name\n    this.conversation = def.conversation\n    if (def.description !== undefined) this.description = def.description\n    if (def.tags !== undefined) this.tags = def.tags\n    if (def.type !== undefined) this.type = def.type\n    if (def.setup !== undefined) this.setup = def.setup\n    if (def.outcome !== undefined) this.outcome = def.outcome\n    if (def.options !== undefined) this.options = def.options\n  }\n}\n", "import { AdkError } from '@botpress/analytics'\n\n/**\n * Stable error codes for the eval engine. Typed as a literal union so a typo'd\n * code is a compile error and PostHog issue-slicing stays consistent.\n * Conventions per docs/ERROR-HANDLING.md.\n */\nexport const EVAL_ERROR_CODES = [\n  'EVAL_LOAD_FAILED',\n  'EVAL_FILE_EMPTY',\n  'EVAL_DUPLICATE_NAME',\n  'EVAL_SEED_NO_CONVERSATION',\n  'EVAL_NO_CONVERSATION_ID',\n  'EVAL_TURN_CONFIG_INVALID',\n  'SSE_CONNECT_FAILED',\n  'SSE_NO_BODY',\n  'CHAT_CLIENT_MISSING',\n  'CHAT_NOT_CONNECTED',\n  'CHAT_INTEGRATION_MISSING',\n  'CHAT_CHANNEL_UNBOUND',\n] as const\n\nexport type EvalErrorCode = (typeof EVAL_ERROR_CODES)[number]\n\n/**\n * Typed error for the eval engine. Named EvalRunnerError (not EvalError) to\n * avoid shadowing the JS built-in `EvalError`.\n */\nexport class EvalRunnerError extends AdkError<EvalErrorCode> {}\n", "import type { EvalDefinition } from '../definition'\nimport { loadEvalsFromDir } from '../loader'\n\nexport type EvalDefinitionLoader = () => Promise<EvalDefinition[]>\n\nexport function createDiskEvalLoader(evalsDir: string): EvalDefinitionLoader {\n  return () => loadEvalsFromDir(evalsDir)\n}\n"],
+  "mappings": ";AAiEA,IAAM,qBAAqB;AAe3B,SAAS,wBAAwB,OAAoC;AACnE,QAAM,oBAAoC,MAAM,QAC7C,OAAO,CAAC,MAAM,EAAE,cAAc,kBAAkB,EAChD,IAAI,CAAC,OAAO;AAAA,IACX,WAAW,EAAE;AAAA,IACb,MAAM,EAAE;AAAA,IACR,UAAW,EAAE,UAAU,YAAuB;AAAA,IAC9C,QAAS,EAAE,UAAU,UAAqB;AAAA,EAC5C,EAAE;AAEJ,QAAM,gBAAgB,oBAAI,IAAgC;AAC1D,aAAW,KAAK,MAAM,SAAS;AAC7B,QAAI,EAAE,cAAc,mBAAoB;AACxC,UAAM,WAAW,cAAc,IAAI,EAAE,SAAS,KAAK,CAAC;AACpD,aAAS,KAAK,CAAC;AACf,kBAAc,IAAI,EAAE,WAAW,QAAQ;AAAA,EACzC;AAEA,QAAM,QAAsB,CAAC,GAAG,cAAc,QAAQ,CAAC,EACpD,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,EACxB,IAAI,CAAC,CAAC,WAAW,OAAO,MAAM;AAC7B,UAAM,aAA6B,QAAQ,IAAI,CAAC,OAAO;AAAA,MACrD,WAAW,EAAE;AAAA,MACb,MAAM,EAAE;AAAA,MACR,UAAW,EAAE,UAAU,YAAuB;AAAA,MAC9C,QAAS,EAAE,UAAU,UAAqB;AAAA,IAC5C,EAAE;AACF,UAAM,QAAQ,QAAQ,CAAC;AACvB,WAAO;AAAA,MACL;AAAA,MACA,aAAa,MAAM;AAAA,MACnB,aAAa,MAAM;AAAA,MACnB;AAAA,MACA,MAAM,WAAW,MAAM,CAAC,MAAM,EAAE,IAAI;AAAA,MACpC,aAAa,MAAM,iBAAiB;AAAA,MACpC,cAAc,MAAM,oBAAoB;AAAA,IAC1C;AAAA,EACF,CAAC;AAEH,SAAO;AAAA,IACL,MAAM,MAAM;AAAA,IACZ,aAAa,MAAM,eAAe;AAAA,IAClC,MAAM,MAAM;AAAA,IACZ,MAAM,MAAM;AAAA,IACZ;AAAA,IACA;AAAA,IACA,MAAM,MAAM,UAAU;AAAA,IACtB,UAAU,MAAM,cAAc;AAAA,IAC9B,OAAO,MAAM,SAAS;AAAA,EACxB;AACF;AAGA,eAAe,mBAAyB,OAAY,aAAqB,IAA2C;AAClH,QAAM,UAAU,IAAI,MAAS,MAAM,MAAM;AACzC,MAAI,OAAO;AACX,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,YAAY;AACtF,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,IAAI;AACV,cAAQ,CAAC,IAAI,MAAM,GAAG,MAAM,CAAC,CAAE;AAAA,IACjC;AAAA,EACF,CAAC;AACD,QAAM,QAAQ,IAAI,OAAO;AACzB,SAAO;AACT;AAGA,SAAS,iBAAiB,MAAuC;AAC/D,SAAO,oBAAoB,KAAK,YAAY;AAAA,IAC1C,WAAW,KAAK;AAAA,IAChB,aAAa,KAAK;AAAA,IAClB,aAAa,KAAK;AAAA,IAClB,eAAe,KAAK;AAAA,IACpB,kBAAkB,KAAK;AAAA,EACzB,CAAC;AACH;AAEA,SAAS,oBAAoB,YAAiD;AAC5E,SAAO,oBAAoB,YAAY;AAAA,IACrC,WAAW;AAAA,IACX,aAAa;AAAA,IACb,aAAa;AAAA,IACb,eAAe;AAAA,IACf,kBAAkB;AAAA,EACpB,CAAC;AACH;AAEA,SAAS,oBACP,YACA,SACqB;AACrB,SAAO,WAAW,IAAI,CAAC,eAAe;AAAA,IACpC,GAAG;AAAA,IACH,YAAY,UAAU;AAAA,IACtB,QAAQ,UAAU;AAAA,IAClB,UAAU,EAAE,UAAU,UAAU,UAAU,QAAQ,UAAU,OAAO;AAAA,EACrE,EAAE;AACJ;AAEA,SAAS,uBAAuB,KAAoB,SAAoC;AACtF,MAAI,QAAQ,SAAS,IAAI,OAAO,QAAQ,MAAO,QAAO;AACtD,MAAI,QAAQ,cAAc,IAAI,UAAU,eAAe,QAAQ,WAAY,QAAO;AAClF,SAAO;AACT;AAEA,SAAS,kBAAkB,KAA0C;AACnE,QAAM,QAAQ,IAAI,QAAQ,IAAI,uBAAuB;AACrD,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,WAAW,IAAI;AAAA,IACf;AAAA,IACA,QAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE;AAAA,IACpC,QAAQ,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE;AAAA,IACrC,OAAO,MAAM;AAAA,IACb,UAAU,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,UAAU,CAAC;AAAA,EACpD;AACF;AAEA,SAAS,mBAAmB,KAAsE;AAChG,QAAM,UAAU,IAAI,YAAY,IAAI,KAAK,IAAI,SAAS,EAAE,QAAQ,IAAI;AACpE,QAAM,QAAQ,IAAI,cAAc,IAAI,KAAK,IAAI,WAAW,EAAE,QAAQ,IAAI;AAEtE,MAAI,IAAI,SAAS;AACf,UAAM,QAAQ,IAAI,QAAQ,IAAI,uBAAuB;AACrD,WAAO;AAAA,MACL,IAAI,IAAI;AAAA,MACR,WAAW,IAAI;AAAA,MACf,QAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE;AAAA,MACpC,QAAQ,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE;AAAA,MACrC,OAAO,MAAM;AAAA,MACb,UAAU,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,UAAU,CAAC;AAAA,MAClD,aAAa,MAAM,OAAO,CAAC,GAAG,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,MAAM,KAAK,EAAE,aAAa,CAAC,GAAG,CAAC;AAAA,MACvF,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,MAAM,KAAK,EAAE,cAAc,CAAC,GAAG,CAAC;AAAA,MACzF,WAAW,MAAM,IAAI,CAAC,MAAM,EAAE,IAAI;AAAA,MAClC,SAAS;AAAA,IACX;AAAA,EACF;AAEA,SAAO;AAAA,IACL,IAAI,IAAI;AAAA,IACR,WAAW,IAAI;AAAA,IACf,QAAQ;AAAA,IACR,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,WAAW,QAAQ,QAAQ,UAAU;AAAA,IAC/C,aAAa;AAAA,IACb,cAAc;AAAA,IACd,WAAW,CAAC;AAAA,IACZ,SAAS;AAAA,EACX;AACF;AAIO,IAAM,kBAAN,MAA2C;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAA+B;AACzC,SAAK,MAAM,OAAO,IAAI,QAAQ,OAAO,EAAE;AACvC,SAAK,QAAQ,OAAO;AACpB,QAAI,OAAO,YAAa,MAAK,cAAc,OAAO;AAClD,QAAI,OAAO,MAAO,MAAK,QAAQ,OAAO;AACtC,SAAK,kBAAkB,OAAO;AAC9B,SAAK,uBAAuB,OAAO;AAAA,EACrC;AAAA;AAAA,EAIA,MAAM,UAAU,QAA6C;AAC3D,UAAM,OAAO,MAAM,KAAK,mBAAmB;AAC3C,UAAM,WAAW,KAAK,OAAO,CAAC,MAAM;AAClC,UAAI,QAAQ,SAAS,CAAC,OAAO,MAAM,SAAS,EAAE,IAAI,EAAG,QAAO;AAC5D,UAAI,QAAQ,QAAQ,EAAE,SAAS,OAAO,KAAM,QAAO;AACnD,UAAI,QAAQ,QAAQ,CAAC,OAAO,KAAK,MAAM,CAAC,QAAQ,EAAE,MAAM,SAAS,GAAG,CAAC,EAAG,QAAO;AAC/E,aAAO;AAAA,IACT,CAAC;AACD,WAAO,SAAS,IAAI,CAAC,OAAO;AAAA,MAC1B,MAAM,EAAE;AAAA,MACR,aAAa,EAAE;AAAA,MACf,MAAM,EAAE,QAAQ,CAAC;AAAA,MACjB,MAAM,EAAE,QAAQ;AAAA,MAChB,WAAW,EAAE,aAAa;AAAA,MAC1B,YAAY,CAAC,CAAC,EAAE;AAAA,IAClB,EAAE;AAAA,EACJ;AAAA,EAEA,MAAM,QAAQ,MAA8C;AAC1D,UAAM,OAAO,MAAM,KAAK,mBAAmB;AAC3C,WAAO,KAAK,KAAK,CAAC,MAAM,EAAE,SAAS,IAAI,KAAK;AAAA,EAC9C;AAAA;AAAA,EAIA,MAAM,UAAU,UAAU,aAAa,UAAqD;AAC1F,QAAI,CAAC,KAAK,aAAa;AACrB,YAAM,IAAI,MAAM,oDAAoD;AAAA,IACtE;AAEA,UAAM,cAAc,YAAY,WAAW,WAAW;AACtD,UAAM,MAAM,MAAM,KAAK,MAAM,iBAAiB,KAAK,KAAK,SAAS;AAAA,MAC/D,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU;AAAA,QACnB,iBACG,OAAO,KAAK,oBAAoB,aAAa,KAAK,gBAAgB,IAAI,KAAK,oBAAoB;AAAA,QAClG,aAAa,KAAK;AAAA,QAClB;AAAA,QACA,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,MACjC,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,6BAA6B,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACnE;AAEA,UAAM,EAAE,GAAG,IAAK,MAAM,IAAI,KAAK;AAC/B,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,cAAc,OAAe,YAAuC;AACxE,UAAM,QAAQ;AAAA,MACZ,UAAU,WAAW;AAAA,MACrB,UAAU,WAAW,QAAQ;AAAA,MAC7B,aAAa,WAAW,eAAe;AAAA,MACvC,MAAM,WAAW,QAAQ,CAAC;AAAA,MAC1B,QAAQ,WAAW;AAAA,MACnB,YAAY,WAAW;AAAA,MACvB,OAAO,WAAW;AAAA,MAClB,SAAS,CAAC,GAAG,WAAW,MAAM,QAAQ,gBAAgB,GAAG,GAAG,oBAAoB,WAAW,iBAAiB,CAAC;AAAA,IAC/G;AAEA,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,YAAY;AAAA,MAC9D,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;AAAA,IAC3C,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,iCAAiC,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACvE;AAAA,EACF;AAAA,EAEA,MAAM,YAAY,OAAe,QAAsC;AACrE,UAAM,cAAc,OAAO,YAAY,QAAQ,OAAO,MAAM,KAAK,CAAC,MAAM,EAAE,UAAU,MAAS;AAC7F,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,YAAY;AAAA,MAC9D,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU;AAAA,QACnB,SAAS,CAAC;AAAA,QACV,WAAW,CAAC;AAAA,QACZ,QAAQ;AAAA,MACV,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACrE;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAaA,MAAM,WACJ,OACA,MACiB;AACjB,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,YAAY;AAAA,MAC9D,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU;AAAA,QACnB,SAAS;AAAA,UACP;AAAA,YACE,UAAU,KAAK;AAAA,YACf,UAAU,KAAK,YAAY;AAAA,YAC3B,aAAa,KAAK,eAAe;AAAA,YACjC,MAAM,KAAK,QAAQ,CAAC;AAAA,UACtB;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,8BAA8B,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACpE;AAEA,UAAM,OAAQ,MAAM,IAAI,KAAK;AAC7B,UAAM,UAAU,KAAK,UAAU,CAAC,GAAG,MAAM,KAAK;AAC9C,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,yCAAyC;AAAA,IAC3D;AACA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,MAAM,kBAAkB,OAAe,SAAiB,MAAiC;AACvF,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,YAAY,OAAO,YAAY;AAAA,MACjF,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,SAAS,iBAAiB,IAAI,EAAE,CAAC;AAAA,IAC1D,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,iCAAiC,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACvE;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,qBAAqB,OAAe,SAAiB,YAA2C;AACpG,QAAI,WAAW,WAAW,EAAG;AAE7B,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,YAAY,OAAO,YAAY;AAAA,MACjF,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,SAAS,oBAAoB,UAAU,EAAE,CAAC;AAAA,IACnE,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,iCAAiC,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACvE;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,cACJ,OACA,SACA,SACe;AACf,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,YAAY,OAAO,IAAI;AAAA,MACzE,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU;AAAA,QACnB,QAAQ,QAAQ;AAAA,QAChB,GAAI,QAAQ,eAAe,SAAY,EAAE,YAAY,QAAQ,WAAW,IAAI,CAAC;AAAA,QAC7E,GAAI,QAAQ,UAAU,SAAY,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,MAChE,CAAC;AAAA,IACH,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,iCAAiC,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACvE;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,gBAAgB,OAAe,OAA6B,CAAC,GAAkB;AACnF,UAAM,MAAM,MAAM,KAAK,MAAM,kBAAkB,KAAK,aAAa;AAAA,MAC/D,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,GAAI,KAAK,WAAW,SAAY,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC,EAAG,CAAC;AAAA,IACxF,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,+BAA+B,IAAI,MAAM,IAAI,IAAI,EAAE;AAAA,IACrE;AAAA,EACF;AAAA;AAAA,EAIA,MAAM,cAAc,OAA8C;AAChE,UAAM,OAAO,MAAM,KAAK,UAAgC,kBAAkB,KAAK,EAAE;AACjF,WAAO,kBAAkB,IAAI;AAAA,EAC/B;AAAA,EAEA,MAAM,eAA8C;AAClD,UAAM,OAAO,MAAM,KAAK,UAAqC,iBAAiB,KAAK,KAAK,SAAS,EAAE,OAAO,IAAI,CAAC;AAC/G,QAAI,CAAC,KAAK,KAAK,OAAQ,QAAO;AAC9B,UAAM,SAAS,KAAK,KAAK,CAAC;AAC1B,WAAO,KAAK,cAAc,OAAO,EAAE;AAAA,EACrC;AAAA,EAEA,MAAM,iBAAiB,MAAsE;AAM3F,UAAM,OAAO,MAAM,KAAK,qBAAqB,MAAM,SAAS,IAAI,MAAM,KAAK;AAC3E,WAAO,KAAK,IAAI,kBAAkB;AAAA,EACpC;AAAA,EAEA,MAAM,sBACJ,UACA,MACmC;AACnC,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,OAAO,MAAM,KAAK,qBAAqB,IAAI,MAAM,KAAK;AAE5D,UAAM,UAAoC,CAAC;AAC3C,eAAW,OAAO,MAAM;AACtB,UAAI,QAAQ,UAAU,MAAO;AAC7B,YAAM,QAAQ,IAAI,QAAQ,KAAK,CAAC,MAAM,EAAE,aAAa,QAAQ;AAC7D,UAAI,OAAO;AACT,gBAAQ,KAAK;AAAA,UACX,OAAO,IAAI;AAAA,UACX,WAAW,IAAI;AAAA,UACf,QAAQ,wBAAwB,KAAK;AAAA,UACrC,iBAAiB,IAAI,QAAQ;AAAA,QAC/B,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAAoB,MAG4B;AACpD,UAAM,MAAM,MAAM,WAAW;AAC7B,UAAM,OAAO,MAAM,KAAK,qBAAqB,IAAI,MAAM,KAAK;AAE5D,UAAM,UAAoD,CAAC;AAC3D,eAAW,OAAO,MAAM;AACtB,iBAAW,SAAS,IAAI,SAAS;AAC/B,YAAI,CAAC,QAAQ,MAAM,QAAQ,EAAG,SAAQ,MAAM,QAAQ,IAAI,CAAC;AACzD,YAAI,QAAQ,MAAM,QAAQ,EAAG,SAAS,KAAK;AACzC,kBAAQ,MAAM,QAAQ,EAAG,KAAK;AAAA,YAC5B,OAAO,IAAI;AAAA,YACX,WAAW,IAAI;AAAA,YACf,QAAQ,wBAAwB,KAAK;AAAA,YACrC,iBAAiB,IAAI,QAAQ;AAAA,UAC/B,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA;AAAA,EAIA,OAAO,SAAS,QAAsB,UAA4B,CAAC,GAAqC;AAItG,UAAM,gBAAgB;AACtB,UAAM,eAAe,KAAK,KAAK;AAC/B,UAAM,QAAQ,KAAK,IAAI;AAUvB,UAAM,iBAAiB,CAAC,CAAC,QAAQ,SAAS,CAAC,CAAC,QAAQ;AACpD,QAAI,gBAA+B;AACnC,QAAI,CAAC,gBAAgB;AACnB,UAAI;AACF,cAAM,UAAU,MAAM,KAAK,UAAqC,iBAAiB,KAAK,KAAK,SAAS;AAAA,UAClG,OAAO;AAAA,QACT,CAAC;AACD,wBAAgB,QAAQ,KAAK,CAAC,GAAG,MAAM;AAAA,MACzC,QAAQ;AAAA,MAER;AAAA,IACF;AASA,UAAM,WAAW,MAAM,KAAK,aAAa;AACzC,UAAM,eAAe,oBAAI,IAAY;AACrC,UAAM,YAAY,oBAAI,IAAY;AAClC,UAAM,eAAe,oBAAI,IAAY;AACrC,UAAM,iBAAiB,oBAAI,IAAY;AAEvC,WAAO,CAAC,QAAQ,WAAW,KAAK,IAAI,IAAI,QAAQ,cAAc;AAC5D,UAAI;AACF,cAAM,OAAO,MAAM,KAAK,UAAqC,iBAAiB,KAAK,KAAK,SAAS;AAAA,UAC/F,OAAO,iBAAiB,OAAO;AAAA,QACjC,CAAC;AACD,YAAI,SAA+B;AACnC,YAAI,MAAmC;AACvC,mBAAW,aAAa,KAAK,MAAM;AACjC,cAAI,QAAQ,SAAS,UAAU,OAAO,QAAQ,MAAO;AACrD,cAAI,QAAQ,cAAc,UAAU,UAAU,eAAe,QAAQ,YAAY;AAC/E,kBAAM,eAAe,MAAM,KAAK,UAAgC,kBAAkB,UAAU,EAAE,EAAE;AAChG,gBAAI,CAAC,uBAAuB,cAAc,OAAO,EAAG;AACpD,kBAAM;AAAA,UACR;AACA,mBAAS;AACT;AAAA,QACF;AAIA,YAAI,CAAC,UAAW,CAAC,kBAAkB,OAAO,OAAO,eAAgB;AAC/D,gBAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;AACrD;AAAA,QACF;AAEA,gBAAQ,MAAM,KAAK,UAAgC,kBAAkB,OAAO,EAAE,EAAE;AAChF,cAAM,WAAW,IAAI,WAAW,eAAe,IAAI,WAAW;AAE9D,iBAAS,IAAI,GAAG,IAAI,IAAI,QAAQ,QAAQ,KAAK;AAC3C,gBAAM,QAAQ,IAAI,QAAQ,CAAC;AAC3B,gBAAM,aAAa,wBAAwB,KAAK;AAChD,gBAAM,OAAO,SAAS,IAAI,MAAM,QAAQ;AACxC,gBAAM,aAAa,MAAM,cAAc,WAAW,MAAM;AAExD,cAAI,CAAC,aAAa,IAAI,MAAM,QAAQ,GAAG;AACrC,yBAAa,IAAI,MAAM,QAAQ;AAC/B,kBAAM,EAAE,MAAM,cAAc,UAAU,MAAM,UAAU,OAAO,GAAG,WAAW;AAAA,UAC7E;AACA,qBAAW,QAAQ,WAAW,OAAO;AACnC,kBAAM,UAAU,GAAG,MAAM,QAAQ,KAAK,KAAK,SAAS;AACpD,gBAAI,UAAU,IAAI,OAAO,EAAG;AAC5B,sBAAU,IAAI,OAAO;AACrB,kBAAM;AAAA,cACJ,MAAM;AAAA,cACN,UAAU,MAAM;AAAA,cAChB,WAAW;AAAA,cACX,WAAW,KAAK;AAAA,cAChB;AAAA,cACA,YAAY;AAAA,YACd;AAAA,UACF;AAEA,cAAI,MAAM,UAAU,QAAQ,CAAC,eAAe,IAAI,MAAM,QAAQ,GAAG;AAC/D,2BAAe,IAAI,MAAM,QAAQ;AACjC,kBAAM,EAAE,MAAM,iBAAiB,UAAU,MAAM,UAAU,OAAO,GAAG,QAAQ,WAAW;AAAA,UACxF,WAAW,MAAM,UAAU,QAAQ,CAAC,UAAU;AAK5C,kBAAM,gBAAgB,WAAW,MAAM;AACvC,kBAAM,WAAW,GAAG,MAAM,QAAQ,KAAK,aAAa;AACpD,gBAAI,gBAAgB,cAAc,CAAC,aAAa,IAAI,QAAQ,GAAG;AAC7D,2BAAa,IAAI,QAAQ;AACzB,oBAAM;AAAA,gBACJ,MAAM;AAAA,gBACN,UAAU,MAAM;AAAA,gBAChB,WAAW;AAAA,gBACX,WAAW;AAAA,gBACX;AAAA,gBACA,aAAa,MAAM,aAAa,aAAa,KAAK;AAAA,cACpD;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAEA,YAAI,UAAU;AACZ,gBAAM,cAAc,kBAAkB,GAAG;AAGzC,mBAAS,IAAI,GAAG,IAAI,YAAY,MAAM,QAAQ,KAAK;AACjD,kBAAM,aAAa,YAAY,MAAM,CAAC;AACtC,gBAAI,eAAe,IAAI,WAAW,IAAI,EAAG;AACzC,2BAAe,IAAI,WAAW,IAAI;AAClC,kBAAM,EAAE,MAAM,iBAAiB,UAAU,WAAW,MAAM,OAAO,GAAG,QAAQ,WAAW;AAAA,UACzF;AACA,gBAAM;AAAA,YACJ,MAAM;AAAA,YACN,QAAQ;AAAA,YACR,OAAO,IAAI,WAAW,WAAW,oBAAoB;AAAA,UACvD;AACA;AAAA,QACF;AAAA,MACF,QAAQ;AAAA,MAER;AAKA,YAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,aAAa,CAAC;AAAA,IACvD;AAEA,QAAI,CAAC,QAAQ,SAAS;AACpB,YAAM;AAAA,QACJ,MAAM;AAAA,QACN,QAAQ;AAAA,UACN,IAAI;AAAA,UACJ,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,UAClC,OAAO,CAAC;AAAA,UACR,QAAQ;AAAA,UACR,QAAQ;AAAA,UACR,OAAO;AAAA,UACP,UAAU;AAAA,QACZ;AAAA,QACA,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA;AAAA,EAIA,MAAM,iBAAsE;AAC1E,QAAI;AACF,YAAM,OAAO,MAAM,KAAK,UAAqC,iBAAiB,KAAK,KAAK,SAAS;AAAA,QAC/F,QAAQ;AAAA,QACR,OAAO;AAAA,MACT,CAAC;AACD,YAAM,SAAS,KAAK,KAAK,CAAC,KAAK;AAC/B,aAAO,EAAE,SAAS,CAAC,CAAC,QAAQ,OAAO,QAAQ,MAAM,KAAK;AAAA,IACxD,QAAQ;AACN,aAAO,EAAE,SAAS,OAAO,OAAO,KAAK;AAAA,IACvC;AAAA,EACF;AAAA;AAAA,EAIA,MAAc,qBAAgD;AAC5D,QAAI,CAAC,KAAK,qBAAsB,QAAO,CAAC;AACxC,WAAO,KAAK,qBAAqB;AAAA,EACnC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,MAAc,eAAqF;AACjG,QAAI;AACF,YAAM,OAAO,MAAM,KAAK,mBAAmB;AAC3C,aAAO,IAAI;AAAA,QACT,KAAK,IAAI,CAAC,MAAM;AAAA,UACd,EAAE;AAAA,UACF,EAAE,YAAY,EAAE,aAAa,QAAQ,cAAc,EAAE,aAAa,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE;AAAA,QAC7F,CAAC;AAAA,MACH;AAAA,IACF,QAAQ;AACN,aAAO,oBAAI,IAAI;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,MAAc,qBAAqB,OAAe,OAAiD;AACjG,UAAM,OAAO,MAAM,KAAK,UAAqC,iBAAiB,KAAK,KAAK,SAAS;AAAA,MAC/F,OAAO,OAAO,KAAK;AAAA,IACrB,CAAC;AAED,QAAI,OAAO,KAAK;AAChB,QAAI,OAAO;AACT,aAAO,KAAK,OAAO,CAAC,MAAM,IAAI,KAAK,EAAE,SAAS,EAAE,QAAQ,KAAK,KAAK;AAAA,IACpE;AAIA,WAAO,mBAAmB,MAAM,GAAG,CAAC,MAAM,KAAK,UAAgC,kBAAkB,EAAE,EAAE,EAAE,CAAC;AAAA,EAC1G;AAAA,EAEA,MAAc,MAAM,MAAc,MAAuC;AACvE,UAAM,SAAS,IAAI,IAAI,GAAG,KAAK,GAAG,GAAG,IAAI,EAAE;AAC3C,UAAM,UAAU,IAAI,QAAQ,MAAM,OAAO;AACzC,QAAI,KAAK,MAAO,SAAQ,IAAI,iBAAiB,UAAU,KAAK,KAAK,EAAE;AACnE,WAAO,WAAW,MAAM,QAAQ,EAAE,GAAG,MAAM,QAAQ,CAAC;AAAA,EACtD;AAAA,EAEA,MAAc,UAAa,MAAc,QAA6C;AACpF,UAAM,SAAS,IAAI,IAAI,GAAG,KAAK,GAAG,GAAG,IAAI,EAAE;AAC3C,QAAI,QAAQ;AACV,iBAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,MAAM,EAAG,QAAO,aAAa,IAAI,GAAG,CAAC;AAAA,IAC3E;AACA,UAAM,UAAU,IAAI,QAAQ;AAC5B,QAAI,KAAK,MAAO,SAAQ,IAAI,iBAAiB,UAAU,KAAK,KAAK,EAAE;AAEnE,UAAM,MAAM,MAAM,WAAW,MAAM,QAAQ,EAAE,QAAQ,CAAC;AACtD,QAAI,CAAC,IAAI,IAAI;AACX,YAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,EAAE;AAC5C,YAAM,IAAI,MAAM,UAAU,IAAI,MAAM,KAAK,IAAI,EAAE;AAAA,IACjD;AACA,WAAO,IAAI,KAAK;AAAA,EAClB;AACF;;;AC5vBA,SAAS,aAAa,kBAAkB;AACxC,SAAS,eAAe;;;ACuFjB,IAAM,OAAN,MAAqC;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAMT,YAAY,KAAqB;AAC/B,SAAK,OAAO,IAAI;AAChB,SAAK,eAAe,IAAI;AACxB,QAAI,IAAI,gBAAgB,OAAW,MAAK,cAAc,IAAI;AAC1D,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,SAAS,OAAW,MAAK,OAAO,IAAI;AAC5C,QAAI,IAAI,UAAU,OAAW,MAAK,QAAQ,IAAI;AAC9C,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAClD,QAAI,IAAI,YAAY,OAAW,MAAK,UAAU,IAAI;AAAA,EACpD;AACF;;;ACtHA,SAAS,gBAAgB;AA4BlB,IAAM,kBAAN,cAA8B,SAAwB;AAAC;;;AFhB9D,SAAS,iBAAiB,OAAyC;AACjE,SACE,UAAU,QACV,OAAO,UAAU,YACjB,OAAQ,MAAkC,SAAS,YAClD,MAAkC,SAAS,MAC5C,MAAM,QAAS,MAAkC,YAAY;AAEjE;AAEA,eAAsB,aAAa,UAA6C;AAC9E,QAAM,UAAU,QAAQ,QAAQ;AAEhC,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,OAAO;AAAA,EACrB,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAG3D,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SACE,4BAA4B,QAAQ,KAAK,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,MAK9C,UAAU;AAAA,MACV,OAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,QAAM,UAA4B,CAAC;AAEnC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,QAAQ,aAAc;AAC1B,QAAI,iBAAiB,QAAQ,iBAAiB,KAAK,GAAG;AACpD,cAAQ,KAAK,KAAK;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,gBAAgB;AAAA,MACxB,MAAM;AAAA,MACN,SAAS,qBAAqB,QAAQ;AAAA,MACtC,UAAU;AAAA,IACZ,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAEA,eAAsB,iBAAiB,SAA4C;AACjF,QAAM,SAAS,QAAQ,OAAO;AAC9B,MAAI,CAAC,WAAW,MAAM,GAAG;AACvB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,YAAY,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AACtE,QAAM,QAA0B,CAAC;AAEjC,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,EAAE;AAChD,UAAM,KAAK,GAAG,IAAI;AAAA,EACpB;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,KAAK,OAAO;AACrB,QAAI,KAAK,IAAI,EAAE,IAAI,GAAG;AACpB,YAAM,IAAI,gBAAgB;AAAA,QACxB,MAAM;AAAA,QACN,SAAS,wBAAwB,EAAE,IAAI,cAAc,OAAO;AAAA,QAC5D,UAAU;AAAA,QACV,SAAS,EAAE,MAAM,EAAE,KAAK;AAAA,MAC1B,CAAC;AAAA,IACH;AACA,SAAK,IAAI,EAAE,IAAI;AAAA,EACjB;AAEA,SAAO;AACT;;;AGxFO,SAAS,qBAAqB,UAAwC;AAC3E,SAAO,MAAM,iBAAiB,QAAQ;AACxC;",
+  "names": []
+}

package/dist/stores/vortex-eval-store.d.ts

@@ -0,0 +1,78 @@
+import type { EvalDefinition } from '../definition';
+import type { EvalFilter, EvalProgressEvent, EvalReport, EvalRunReport, TurnReport, GraderResult } from '../types';
+import type { EvalStore, EvalSummary, EvalRunSummary, EvalReportHistoryEntry, EvalWatchOptions } from './eval-store';
+import type { EvalDefinitionLoader } from './eval-definition-loader';
+export interface VortexEvalStoreConfig {
+    url: string;
+    botId: string;
+    workspaceId?: string;
+    token?: string;
+    evalManifestId?: string | (() => string | undefined);
+    loadEvalDefinitions?: EvalDefinitionLoader;
+}
+export declare class VortexEvalStore implements EvalStore {
+    private url;
+    private botId;
+    private workspaceId?;
+    private token?;
+    private _evalManifestId?;
+    private _loadEvalDefinitions?;
+    constructor(config: VortexEvalStoreConfig);
+    listEvals(filter?: EvalFilter): Promise<EvalSummary[]>;
+    getEval(name: string): Promise<EvalDefinition | null>;
+    createRun(runType?: string, metadata?: Record<string, unknown>): Promise<string>;
+    addRunResults(runId: string, evalReport: EvalReport): Promise<void>;
+    completeRun(runId: string, report: EvalRunReport): Promise<void>;
+    /** Create an in-progress entry (metadata only, no results) and return its server id. */
+    startEntry(runId: string, meta: {
+        evalName: string;
+        evalType?: 'capability' | 'regression';
+        description?: string;
+        tags?: string[];
+    }): Promise<string>;
+    /** Append one turn's grader results to an existing entry. */
+    appendTurnResults(runId: string, entryId: string, turn: TurnReport): Promise<void>;
+    /** Append outcome-level grader results to an existing entry. */
+    appendOutcomeResults(runId: string, entryId: string, assertions: GraderResult[]): Promise<void>;
+    /** Set the verdict on an entry once its eval finishes. */
+    finalizeEntry(runId: string, entryId: string, verdict: {
+        passed: boolean;
+        durationMs?: number;
+        error?: string;
+    }): Promise<void>;
+    /** Flip the run to its terminal status (replaces completeRun's empty-entries hack). */
+    markRunComplete(runId: string, opts?: {
+        failed?: boolean;
+    }): Promise<void>;
+    loadRunResult(runId: string): Promise<EvalRunReport | null>;
+    getLatestRun(): Promise<EvalRunReport | null>;
+    listRunSummaries(opts?: {
+        limit?: number;
+        since?: number;
+    }): Promise<EvalRunSummary[]>;
+    listEvalReportsByName(evalName: string, opts?: {
+        limit?: number;
+        since?: number;
+    }): Promise<EvalReportHistoryEntry[]>;
+    listEvalReportsBulk(opts?: {
+        perEval?: number;
+        since?: number;
+    }): Promise<Record<string, EvalReportHistoryEntry[]>>;
+    watchRun(signal?: AbortSignal, options?: EvalWatchOptions): AsyncIterable<EvalProgressEvent>;
+    getRunnerState(): Promise<{
+        running: boolean;
+        runId: string | null;
+    }>;
+    private getEvalDefinitions;
+    /**
+     * evalName → { totalTurns, userMessages } from the eval definitions, so
+     * watchRun can report `totalTurns` and synthesize `turn_start` events (with
+     * the turn's user message) for the in-flight turn — matching the dev SSE
+     * experience where the pending turn is visible before its results land.
+     */
+    private loadEvalMeta;
+    private fetchRunsWithEntries;
+    private fetch;
+    private fetchJson;
+}
+//# sourceMappingURL=vortex-eval-store.d.ts.map

package/dist/stores/vortex-eval-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vortex-eval-store.d.ts","sourceRoot":"","sources":["../../src/stores/vortex-eval-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAClH,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AACpH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAA;AAkEpE,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,cAAc,CAAC,EAAE,MAAM,GAAG,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC,CAAA;IACpD,mBAAmB,CAAC,EAAE,oBAAoB,CAAA;CAC3C;AA8JD,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,GAAG,CAAQ;IACnB,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,WAAW,CAAC,CAAQ;IAC5B,OAAO,CAAC,KAAK,CAAC,CAAQ;IACtB,OAAO,CAAC,eAAe,CAAC,CAAqC;IAC7D,OAAO,CAAC,oBAAoB,CAAC,CAAsB;gBAEvC,MAAM,EAAE,qBAAqB;IAWnC,SAAS,CAAC,MAAM,CAAC,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAkBtD,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAOrD,SAAS,CAAC,OAAO,SAAc,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IA2BrF,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBnE,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtE,wFAAwF;IAClF,UAAU,CACd,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,YAAY,GAAG,YAAY,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GACxG,OAAO,CAAC,MAAM,CAAC;IA6BlB,6DAA6D;IACvD,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAaxF,gEAAgE;IAC1D,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAerG,0DAA0D;IACpD,aAAa,CACjB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM,EACf,OAAO,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAChE,OAAO,CAAC,IAAI,CAAC;IAiBhB,uFAAuF;IACjF,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAe9E,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAK3D,YAAY,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAO7C,gBAAgB,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAUtF,qBAAqB,CACzB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GACxC,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAoB9B,mBAAmB,CAAC,IAAI,CAAC,EAAE;QAC/B,OAAO,CAAC,EAAE,MAAM,CAAA;QAChB,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,EAAE,CAAC,CAAC;IAuB9C,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,OAAO,GAAE,gBAAqB,GAAG,aAAa,CAAC,iBAAiB,CAAC;IAoKjG,cAAc,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;YAe7D,kBAAkB;IAKhC;;;;;OAKG;YACW,YAAY;YAcZ,oBAAoB;YAepB,KAAK;YAOL,SAAS;CAexB"}