@axlsdk/eval 0.14.0 → 0.15.0

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/utils.ts","../src/runner.ts","../src/bootstrap.ts","../src/compare.ts","../src/rescore.ts","../src/multi-run.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer, normalizeScorerResult } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerContext, ScorerFn, ScorerResult } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\nexport { pairedBootstrapCI } from './bootstrap.js';\nexport type { BootstrapCIResult } from './bootstrap.js';\nexport { rescore } from './rescore.js';\nexport type { RescoreOptions } from './rescore.js';\nexport { aggregateRuns } from './multi-run.js';\nexport type { MultiRunSummary } from './multi-run.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n  ScorerDetail,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        if (err && typeof err === 'object') {\n          (err as any).cost = responseCost;\n        }\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          (error as any).cost = responseCost;\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","export function computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.output = result.output;\n      evalItem.cost = result.cost;\n      if (result.metadata) evalItem.metadata = result.metadata;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems[itemIndex] = evalItem;\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    evalItem.scoreDetails = {};\n    let itemScorerCost = 0;\n\n    for (const scorer of config.scorers) {\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n          evalItem.scoreDetails[scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          evalItem.scores[scorer.name] = round(scorerResult.score);\n          evalItem.scoreDetails[scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        // Capture cost from error (LLM scorer attaches it)\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n        evalItem.scoreDetails[scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    evalItems[itemIndex] = evalItem;\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of config.scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  // Aggregate per-model LLM call counts across all items\n  const totalModelCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.modelCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n      const itemModels = item.metadata?.models;\n      if (Array.isArray(itemModels)) {\n        for (const m of itemModels) {\n          if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // models: unique list sorted by total calls (most-called first)\n  // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n  const modelsMeta: Record<string, unknown> = {};\n  if (totalModelCalls.size > 0) {\n    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n    modelsMeta.models = sorted.map(([m]) => m);\n    modelsMeta.modelCounts = Object.fromEntries(sorted);\n  }\n\n  // Aggregate per-workflow call counts across all items (parallel to models).\n  // Workflows come from trackExecution's trace-event collection — callers\n  // don't specify workflow names anywhere, they just appear because the\n  // runtime emits workflow_start events for every execute() call.\n  const totalWorkflowCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.workflowCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique workflows per item\n      const itemWorkflows = item.metadata?.workflows;\n      if (Array.isArray(itemWorkflows)) {\n        for (const w of itemWorkflows) {\n          if (typeof w === 'string')\n            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // Fall back to config.workflow when the callback bypassed the runtime's\n  // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n  // metadata.workflows array non-empty for the common test-harness case.\n  const workflowsMeta: Record<string, unknown> = {};\n  if (totalWorkflowCalls.size > 0) {\n    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n    workflowsMeta.workflows = sorted.map(([w]) => w);\n    workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n  } else if (config.workflow) {\n    workflowsMeta.workflows = [config.workflow];\n    workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n  }\n\n  return {\n    id,\n    dataset: config.dataset.name,\n    metadata: {\n      ...config.metadata,\n      scorerTypes,\n      ...modelsMeta,\n      ...workflowsMeta,\n    },\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats, timing },\n  };\n}\n","export type BootstrapCIResult = {\n  /** Lower bound of the confidence interval. */\n  lower: number;\n  /** Upper bound of the confidence interval. */\n  upper: number;\n  /** Mean of the original differences. */\n  mean: number;\n  /** Proportion of resamples where the mean difference is negative (probability of regression). */\n  pRegression: number;\n  /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n  pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n  differences: number[],\n  options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n  const n = differences.length;\n  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n  const nResamples = options?.nResamples ?? 1000;\n  const alpha = options?.alpha ?? 0.05;\n\n  const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n  if (n === 1)\n    return {\n      lower: differences[0],\n      upper: differences[0],\n      mean,\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  if (nResamples <= 0)\n    return {\n      lower: round(mean),\n      upper: round(mean),\n      mean: round(mean),\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  // Seeded or random number generator\n  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n  const resampleMeans: number[] = new Array(nResamples);\n  for (let r = 0; r < nResamples; r++) {\n    let sum = 0;\n    for (let i = 0; i < n; i++) {\n      sum += differences[Math.floor(rng() * n)];\n    }\n    resampleMeans[r] = sum / n;\n  }\n\n  resampleMeans.sort((a, b) => a - b);\n\n  const lowerIdx = Math.floor((alpha / 2) * nResamples);\n  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n  // Count resamples below/above zero for directional probabilities\n  // resampleMeans is already sorted, so binary search for zero\n  let belowZero = 0;\n  for (let i = 0; i < nResamples; i++) {\n    if (resampleMeans[i] < 0) belowZero++;\n    else break;\n  }\n  let aboveZero = 0;\n  for (let i = nResamples - 1; i >= 0; i--) {\n    if (resampleMeans[i] > 0) aboveZero++;\n    else break;\n  }\n\n  return {\n    lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n    mean: round(mean),\n    pRegression: round(belowZero / nResamples),\n    pImprovement: round(aboveZero / nResamples),\n  };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n  let state = seed | 0 || 1; // Ensure non-zero\n  return () => {\n    state ^= state << 13;\n    state ^= state >> 17;\n    state ^= state << 5;\n    return (state >>> 0) / 0x100000000;\n  };\n}\n","import type {\n  EvalResult,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n  scorerName: string,\n  options: EvalCompareOptions | undefined,\n  metadata: Record<string, unknown>,\n): number {\n  // Explicit global threshold\n  if (typeof options?.thresholds === 'number') return options.thresholds;\n\n  // Per-scorer threshold map\n  if (options?.thresholds && typeof options.thresholds === 'object') {\n    const perScorer = options.thresholds[scorerName];\n    if (perScorer != null) return perScorer;\n    // Fall through to auto-calibration if scorer not in map\n  }\n\n  // Auto-calibrate from scorerTypes metadata\n  const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n  if (scorerTypes && scorerName in scorerTypes) {\n    return scorerTypes[scorerName] === 'llm'\n      ? DEFAULT_LLM_THRESHOLD\n      : DEFAULT_DETERMINISTIC_THRESHOLD;\n  }\n\n  // Legacy fallback for results without scorerTypes\n  return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n  baseline: EvalResult | EvalResult[],\n  candidate: EvalResult | EvalResult[],\n  options?: EvalCompareOptions,\n): EvalComparison {\n  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];\n  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];\n\n  if (baselineRuns.length === 0 || candidateRuns.length === 0) {\n    throw new Error('Cannot compare empty eval result arrays');\n  }\n\n  // Use the first run as representative for metadata, dataset, scorers\n  const baselineRef = baselineRuns[0];\n  const candidateRef = candidateRuns[0];\n\n  if (baselineRef.dataset !== candidateRef.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n  // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n  const pairedDiffs: Record<string, number[]> = {};\n  for (const name of baselineScorerNames) {\n    pairedDiffs[name] = [];\n  }\n  const runCount = Math.min(baselineRuns.length, candidateRuns.length);\n  for (let r = 0; r < runCount; r++) {\n    const bRun = baselineRuns[r];\n    const cRun = candidateRuns[r];\n    const minLength = Math.min(bRun.items.length, cRun.items.length);\n    for (let i = 0; i < minLength; i++) {\n      const bItem = bRun.items[i];\n      const cItem = cRun.items[i];\n      if (bItem.error || cItem.error) continue;\n      for (const name of baselineScorerNames) {\n        const bScore = bItem.scores[name];\n        const cScore = cItem.scores[name];\n        if (bScore != null && cScore != null) {\n          pairedDiffs[name].push(cScore - bScore);\n        }\n      }\n    }\n  }\n\n  // Compute aggregate means across runs\n  const baselineMeans: Record<string, number> = {};\n  const candidateMeans: Record<string, number> = {};\n  for (const name of baselineScorerNames) {\n    baselineMeans[name] =\n      baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      baselineRuns.length;\n    candidateMeans[name] =\n      candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      candidateRuns.length;\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMeanRaw = baselineMeans[name];\n    const cMeanRaw = candidateMeans[name];\n    const deltaRaw = cMeanRaw - bMeanRaw;\n    const bMean = round(bMeanRaw);\n    const cMean = round(cMeanRaw);\n    const delta = round(deltaRaw);\n    const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n    const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n    const entry: EvalComparison['scorers'][string] = {\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      deltaPercent,\n    };\n\n    // Compute bootstrap CI when we have enough paired data\n    const diffs = pairedDiffs[name];\n    if (diffs.length >= 2) {\n      const ci = pairedBootstrapCI(diffs);\n      entry.ci = { lower: ci.lower, upper: ci.upper };\n      entry.pRegression = ci.pRegression;\n      entry.pImprovement = ci.pImprovement;\n      entry.n = diffs.length;\n      // Significant when CI excludes zero AND delta exceeds practical threshold\n      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n    }\n\n    scorers[name] = entry;\n  }\n\n  // Per-item regressions/improvements.\n  // For multi-run: average each item's score across runs to get a stable per-item comparison.\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const itemCount = Math.min(\n    ...baselineRuns.map((r) => r.items.length),\n    ...candidateRuns.map((r) => r.items.length),\n  );\n\n  for (let i = 0; i < itemCount; i++) {\n    // Check if any run has an error for this item\n    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n    if (hasBaselineError || hasCandidateError) continue;\n\n    for (const name of baselineScorerNames) {\n      // Average this item's score across all runs on each side\n      const bScores = baselineRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      const cScores = candidateRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      if (bScores.length === 0 || cScores.length === 0) continue;\n\n      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n      const delta = round(cAvg - bAvg);\n      const threshold = resolveThreshold(name, options, baselineRef.metadata);\n      if (delta < -threshold)\n        regressions.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n      else if (delta > threshold)\n        improvements.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (average across all runs for multi-run)\n  let timing: EvalComparison['timing'];\n  const baselineTimings = baselineRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  const candidateTimings = candidateRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n    timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n  }\n\n  // Cost comparison (per-run average for multi-run)\n  let cost: EvalComparison['cost'];\n  const baselineAvgCost =\n    baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n  const candidateAvgCost =\n    candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n  if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n    const deltaRaw = candidateAvgCost - baselineAvgCost;\n    const delta = round(deltaRaw);\n    const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n    cost = {\n      baselineTotal: round(baselineAvgCost),\n      candidateTotal: round(candidateAvgCost),\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      let sigLabel = '';\n      if (s.significant === true) sigLabel = ' (significant)';\n      else if (s.significant === false) sigLabel = ' (not significant)';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },\n    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n  concurrency?: number;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n  result: EvalResult,\n  scorers: Scorer[],\n  runtime: AxlRuntime,\n  options?: RescoreOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const concurrency = options?.concurrency ?? 5;\n\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const rescored: EvalItem[] = new Array(result.items.length);\n  let totalCost = 0;\n\n  async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n    // Pass through error items without scoring\n    if (original.error) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: original.error,\n        scores: {},\n      };\n      return;\n    }\n\n    const item: EvalItem = {\n      input: original.input,\n      annotations: original.annotations,\n      output: original.output,\n      metadata: original.metadata,\n      scores: {},\n      scoreDetails: {},\n    };\n\n    let itemScorerCost = 0;\n\n    for (const scorer of scorers) {\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!item.scorerErrors) item.scorerErrors = [];\n          item.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          item.scores[scorer.name] = null;\n          item.scoreDetails![scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          item.scores[scorer.name] = round(scorerResult.score);\n          item.scoreDetails![scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!item.scorerErrors) item.scorerErrors = [];\n        item.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        item.scores[scorer.name] = null;\n        item.scoreDetails![scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    rescored[itemIndex] = item;\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < result.items.length) {\n      const currentIndex = index++;\n      await rescoreItem(result.items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, result.items.length) }, () =>\n    runNext(),\n  );\n  await Promise.all(workers);\n\n  const failures = rescored.filter((i) => i.error).length;\n  const scorerNames = scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = rescored\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  return {\n    id: randomUUID(),\n    dataset: result.dataset,\n    metadata: (() => {\n      // Strip run group membership — rescored results are independent evaluations.\n      // metadata.workflows is preserved via ...rest so the rescored result keeps\n      // the same workflow attribution as the original.\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;\n      const merged: Record<string, unknown> = {\n        ...rest,\n        rescored: true,\n        originalId: result.id,\n        scorerTypes,\n      };\n      // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n      // as a top-level string field with no `metadata.workflows`. Migrate it\n      // forward so rescored results from old artifacts retain their workflow\n      // attribution under the modern shape.\n      if (!Array.isArray(merged.workflows)) {\n        const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n        if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n          merged.workflows = [legacyWorkflow];\n          merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n        }\n      }\n      return merged;\n    })(),\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: rescored,\n    summary: { count: result.items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n  runGroupId: string;\n  runCount: number;\n  /**\n   * Workflow names observed across all runs in the group, deduped and ordered\n   * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n   * multi-run group produced by `axl-eval --runs N` typically has one entry;\n   * heterogeneous groups (custom callbacks) can have multiple.\n   */\n  workflows: string[];\n  dataset: string;\n  totalCost: number;\n  totalDuration: number;\n  scorers: Record<string, { mean: number; std: number; min: number; max: number }>;\n  timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n  if (values.length <= 1) return 0;\n  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n  return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n  if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n  const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n  const { dataset } = runs[0];\n\n  // Union workflow names across all runs in the group, first-seen first.\n  // Most groups have one, but custom callbacks can produce heterogeneous ones.\n  const seenWorkflows = new Set<string>();\n  const workflows: string[] = [];\n  for (const run of runs) {\n    const list = run.metadata.workflows;\n    if (Array.isArray(list)) {\n      for (const w of list) {\n        if (typeof w === 'string' && !seenWorkflows.has(w)) {\n          seenWorkflows.add(w);\n          workflows.push(w);\n        }\n      }\n    }\n  }\n\n  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n  // Collect per-scorer means from each run\n  const scorerNames = Object.keys(runs[0].summary.scorers);\n  const scorers: MultiRunSummary['scorers'] = {};\n\n  for (const name of scorerNames) {\n    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);\n    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n    const minMean = Math.min(...means);\n    const maxMean = Math.max(...means);\n\n    scorers[name] = {\n      mean: round(meanOfMeans),\n      std: round(std(means, meanOfMeans)),\n      min: round(minMean),\n      max: round(maxMean),\n    };\n  }\n\n  // Timing aggregation\n  let timing: MultiRunSummary['timing'];\n  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n  if (timingMeans.length === runs.length && timingMeans.length > 0) {\n    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n    timing = {\n      mean: round(meanTiming),\n      std: round(std(timingMeans, meanTiming)),\n    };\n  }\n\n  return {\n    runGroupId,\n    runCount: runs.length,\n    workflows,\n    dataset,\n    totalCost: round(totalCost),\n    totalDuration: round(totalDuration),\n    scorers,\n    timing,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACrDO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC/DA,iBAAkB;AAElB,iBAA6C;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,aAAE,OAAO,EAAE,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,aAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,cAAU,4BAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,UAAM,wBAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACJO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEO,SAAS,MAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;;;ACdA,yBAA2B;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,SAAS,OAAO;AACzB,eAAS,OAAO,OAAO;AACvB,UAAI,OAAO,SAAU,UAAS,WAAW,OAAO;AAChD,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,SAAS,IAAI;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWA,WAAU,OAAO,SAAS;AACnC,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACvD,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAGA,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;ACnPO,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAOC,OAAM,IAAI;AAAA,MACjB,OAAOA,OAAM,IAAI;AAAA,MACjB,MAAMA,OAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAOA,OAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAOA,OAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAMA,OAAM,IAAI;AAAA,IAChB,aAAaA,OAAM,YAAY,UAAU;AAAA,IACzC,cAAcA,OAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;AC5FA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACnE,QAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAEvE,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAGA,QAAM,cAAc,aAAa,CAAC;AAClC,QAAM,eAAe,cAAc,CAAC;AAEpC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,QAAM,WAAW,KAAK,IAAI,aAAa,QAAQ,cAAc,MAAM;AACnE,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAEtE,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,IACF;AAGA,UAAM,QAAQ,YAAY,IAAI;AAC9B,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AACxB,YAAM,IAAI,MAAM;AAEhB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAE/J,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,YAAY,IAAI,UAAU,YAAY,SAAS;AAAA,IAC/D,WAAW,EAAE,IAAI,aAAa,IAAI,UAAU,aAAa,SAAS;AAAA,IAClE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;ACjQA,IAAAC,sBAA2B;AAU3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAE5C,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAE/E,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAEA,QAAI,iBAAiB;AAErB,eAAWC,WAAU,SAAS;AAC5B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,eAAK,aAAa;AAAA,YAChB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,eAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,eAAK,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACnD,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,aAAK,aAAa;AAAA,UAChB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,aAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,UAChC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,aAAS,SAAS,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,OAAO,MAAM,QAAQ;AAClC,YAAM,eAAe;AACrB,YAAM,YAAY,OAAO,MAAM,YAAY,GAAG,YAAY;AAAA,IAC5D;AAAA,EACF;AAEA,QAAM,UAAU,MAAM;AAAA,IAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,MAAM,EAAE;AAAA,IAAG,MACjF,QAAQ;AAAA,EACV;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,QAAI,gCAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAKf,YAAM,EAAE,YAAY,GAAG,UAAU,IAAI,GAAG,KAAK,IAAI,OAAO;AACxD,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;ACxLA,IAAAC,sBAA2B;AAoB3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,kBAAyB,gCAAW;AACzE,QAAM,EAAE,SAAAC,SAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AAChE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAEjC,YAAQ,IAAI,IAAI;AAAA,MACd,MAAM,MAAM,WAAW;AAAA,MACvB,KAAK,MAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAK,MAAM,OAAO;AAAA,MAClB,KAAK,MAAM,OAAO;AAAA,IACpB;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAM,MAAM,UAAU;AAAA,MACtB,KAAK,MAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA,SAAAA;AAAA,IACA,WAAW,MAAM,SAAS;AAAA,IAC1B,eAAe,MAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;","names":["scorer","round","round","import_node_crypto","scorer","import_node_crypto","dataset"]}
+{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/utils.ts","../src/runner.ts","../src/bootstrap.ts","../src/compare.ts","../src/rescore.ts","../src/multi-run.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer, normalizeScorerResult } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerContext, ScorerFn, ScorerResult } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\nexport { pairedBootstrapCI } from './bootstrap.js';\nexport type { BootstrapCIResult } from './bootstrap.js';\nexport { rescore } from './rescore.js';\nexport type { RescoreOptions } from './rescore.js';\nexport { aggregateRuns } from './multi-run.js';\nexport type { MultiRunSummary } from './multi-run.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n  ScorerDetail,\n  EvalProgressEvent,\n  RunEvalOptions,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        if (err && typeof err === 'object') {\n          (err as any).cost = responseCost;\n        }\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          (error as any).cost = responseCost;\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","export function computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary, RunEvalOptions } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/**\n * Extract a user-returned cost only if it's a non-negative finite number.\n * Guards against workflows that return `{ cost: 'free' }`, `{ cost: NaN }`,\n * `{ cost: -1 }`, `{ cost: Infinity }`, etc. — the TS type says `cost?: number`\n * but at runtime we can't trust that.\n *\n * Negative costs are rejected because (a) cost is a USD amount and negative\n * values are nonsensical, and (b) a negative cost would silently shrink\n * `totalCost` below the budget limit check at the processItem level,\n * letting a buggy/malicious workflow run unbounded. `0` is preserved\n * because free operations on paid models are valid.\n *\n * When a user-supplied value is rejected (present but invalid), we log\n * once per item via `console.warn` so the type violation is visible\n * rather than silently swallowed.\n */\nfunction extractUserCost(result: unknown, label = 'executeWorkflow'): number | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const raw = (result as { cost?: unknown }).cost;\n  if (raw === undefined) return undefined;\n  if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) return raw;\n\n  console.warn(\n    `[axl-eval] Ignoring invalid \\`cost\\` from ${label} return: expected non-negative finite number, got ${typeof raw === 'number' ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`,\n  );\n  return undefined;\n}\n\n/**\n * Extract a user-returned metadata record only if it's a plain object.\n * Rejects arrays, null, scalars, and exotic object types (Date, Map, Set,\n * Error, class instances) that would satisfy a loose `typeof === 'object'`\n * check but break `Record<string, unknown>` assumptions in downstream\n * consumers (spread, Object.entries, property access).\n */\nfunction extractUserMetadata(\n  result: unknown,\n  label = 'executeWorkflow',\n): Record<string, unknown> | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const meta = (result as { metadata?: unknown }).metadata;\n  if (meta === undefined) return undefined;\n  if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${meta === null ? 'null' : Array.isArray(meta) ? 'array' : typeof meta}.`,\n    );\n    return undefined;\n  }\n  // Reject exotic objects (Date, Map, Set, Error, class instances) whose\n  // prototype chain differs from Object.prototype. Those pass `typeof ===\n  // 'object'` but don't behave like `Record<string, unknown>`.\n  const proto = Object.getPrototypeOf(meta);\n  if (proto !== Object.prototype && proto !== null) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${(proto?.constructor?.name as string | undefined) ?? 'exotic object'}.`,\n    );\n    return undefined;\n  }\n  return meta as Record<string, unknown>;\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n  runtime: AxlRuntime,\n  options?: RunEvalOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (options?.signal?.aborted) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Cancelled',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      // When captureTraces is on, wrap the user's executeWorkflow in\n      // trackExecution so we get a per-item TraceEvent[] scoped via\n      // AsyncLocalStorage. trackExecution's scope walks the parent chain, so\n      // any nested trackExecution (e.g. from the CLI) continues to work.\n      if (options?.captureTraces) {\n        const tracked = await runtime.trackExecution(\n          async () => executeWorkflow(item.input, runtime),\n          { captureTraces: true },\n        );\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = tracked.result.output;\n        // Prefer user-returned cost/metadata; fall back to tracked values.\n        // Type-guarded so a non-number cost (e.g. `{ cost: 'free' }`) or a\n        // non-object metadata (e.g. an array or scalar) doesn't silently\n        // corrupt downstream math or shape expectations.\n        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;\n        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;\n        if (tracked.traces && tracked.traces.length > 0) {\n          evalItem.traces = tracked.traces;\n        }\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      } else {\n        const result = await executeWorkflow(item.input, runtime);\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = result.output;\n        evalItem.cost = extractUserCost(result);\n        const meta = extractUserMetadata(result);\n        if (meta) evalItem.metadata = meta;\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      // Failed items are exactly when per-item traces are most valuable —\n      // recover them from the captured-traces side-channel `trackExecution`\n      // attaches to the error on the failure path.\n      if (options?.captureTraces) {\n        const captured = (err as { axlCapturedTraces?: unknown }).axlCapturedTraces;\n        if (Array.isArray(captured) && captured.length > 0) {\n          evalItem.traces = captured as EvalItem['traces'];\n        }\n      }\n      evalItems[itemIndex] = evalItem;\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    evalItem.scoreDetails = {};\n    let itemScorerCost = 0;\n\n    for (const scorer of config.scorers) {\n      if (options?.signal?.aborted) break;\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n          evalItem.scoreDetails[scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          evalItem.scores[scorer.name] = round(scorerResult.score);\n          evalItem.scoreDetails[scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        // Capture cost from error (LLM scorer attaches it)\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n        evalItem.scoreDetails[scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    evalItems[itemIndex] = evalItem;\n    options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of config.scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  // Aggregate per-model LLM call counts across all items\n  const totalModelCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.modelCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n      const itemModels = item.metadata?.models;\n      if (Array.isArray(itemModels)) {\n        for (const m of itemModels) {\n          if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // models: unique list sorted by total calls (most-called first)\n  // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n  const modelsMeta: Record<string, unknown> = {};\n  if (totalModelCalls.size > 0) {\n    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n    modelsMeta.models = sorted.map(([m]) => m);\n    modelsMeta.modelCounts = Object.fromEntries(sorted);\n  }\n\n  // Aggregate per-workflow call counts across all items (parallel to models).\n  // Workflows come from trackExecution's trace-event collection — callers\n  // don't specify workflow names anywhere, they just appear because the\n  // runtime emits workflow_start events for every execute() call.\n  const totalWorkflowCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.workflowCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique workflows per item\n      const itemWorkflows = item.metadata?.workflows;\n      if (Array.isArray(itemWorkflows)) {\n        for (const w of itemWorkflows) {\n          if (typeof w === 'string')\n            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // Fall back to config.workflow when the callback bypassed the runtime's\n  // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n  // metadata.workflows array non-empty for the common test-harness case.\n  const workflowsMeta: Record<string, unknown> = {};\n  if (totalWorkflowCalls.size > 0) {\n    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n    workflowsMeta.workflows = sorted.map(([w]) => w);\n    workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n  } else if (config.workflow) {\n    workflowsMeta.workflows = [config.workflow];\n    workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n  }\n\n  options?.onProgress?.({ type: 'run_done', totalItems: items.length, failures });\n\n  return {\n    id,\n    dataset: config.dataset.name,\n    metadata: {\n      ...config.metadata,\n      scorerTypes,\n      ...modelsMeta,\n      ...workflowsMeta,\n    },\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats, timing },\n  };\n}\n","export type BootstrapCIResult = {\n  /** Lower bound of the confidence interval. */\n  lower: number;\n  /** Upper bound of the confidence interval. */\n  upper: number;\n  /** Mean of the original differences. */\n  mean: number;\n  /** Proportion of resamples where the mean difference is negative (probability of regression). */\n  pRegression: number;\n  /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n  pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n  differences: number[],\n  options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n  const n = differences.length;\n  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n  const nResamples = options?.nResamples ?? 1000;\n  const alpha = options?.alpha ?? 0.05;\n\n  const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n  if (n === 1)\n    return {\n      lower: differences[0],\n      upper: differences[0],\n      mean,\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  if (nResamples <= 0)\n    return {\n      lower: round(mean),\n      upper: round(mean),\n      mean: round(mean),\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  // Seeded or random number generator\n  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n  const resampleMeans: number[] = new Array(nResamples);\n  for (let r = 0; r < nResamples; r++) {\n    let sum = 0;\n    for (let i = 0; i < n; i++) {\n      sum += differences[Math.floor(rng() * n)];\n    }\n    resampleMeans[r] = sum / n;\n  }\n\n  resampleMeans.sort((a, b) => a - b);\n\n  const lowerIdx = Math.floor((alpha / 2) * nResamples);\n  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n  // Count resamples below/above zero for directional probabilities\n  // resampleMeans is already sorted, so binary search for zero\n  let belowZero = 0;\n  for (let i = 0; i < nResamples; i++) {\n    if (resampleMeans[i] < 0) belowZero++;\n    else break;\n  }\n  let aboveZero = 0;\n  for (let i = nResamples - 1; i >= 0; i--) {\n    if (resampleMeans[i] > 0) aboveZero++;\n    else break;\n  }\n\n  return {\n    lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n    mean: round(mean),\n    pRegression: round(belowZero / nResamples),\n    pImprovement: round(aboveZero / nResamples),\n  };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n  let state = seed | 0 || 1; // Ensure non-zero\n  return () => {\n    state ^= state << 13;\n    state ^= state >> 17;\n    state ^= state << 5;\n    return (state >>> 0) / 0x100000000;\n  };\n}\n","import type {\n  EvalResult,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n  scorerName: string,\n  options: EvalCompareOptions | undefined,\n  metadata: Record<string, unknown>,\n): number {\n  // Explicit global threshold\n  if (typeof options?.thresholds === 'number') return options.thresholds;\n\n  // Per-scorer threshold map\n  if (options?.thresholds && typeof options.thresholds === 'object') {\n    const perScorer = options.thresholds[scorerName];\n    if (perScorer != null) return perScorer;\n    // Fall through to auto-calibration if scorer not in map\n  }\n\n  // Auto-calibrate from scorerTypes metadata\n  const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n  if (scorerTypes && scorerName in scorerTypes) {\n    return scorerTypes[scorerName] === 'llm'\n      ? DEFAULT_LLM_THRESHOLD\n      : DEFAULT_DETERMINISTIC_THRESHOLD;\n  }\n\n  // Legacy fallback for results without scorerTypes\n  return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n  baseline: EvalResult | EvalResult[],\n  candidate: EvalResult | EvalResult[],\n  options?: EvalCompareOptions,\n): EvalComparison {\n  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];\n  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];\n\n  if (baselineRuns.length === 0 || candidateRuns.length === 0) {\n    throw new Error('Cannot compare empty eval result arrays');\n  }\n\n  // Use the first run as representative for metadata, dataset, scorers\n  const baselineRef = baselineRuns[0];\n  const candidateRef = candidateRuns[0];\n\n  if (baselineRef.dataset !== candidateRef.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n  // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n  const pairedDiffs: Record<string, number[]> = {};\n  for (const name of baselineScorerNames) {\n    pairedDiffs[name] = [];\n  }\n  const runCount = Math.min(baselineRuns.length, candidateRuns.length);\n  for (let r = 0; r < runCount; r++) {\n    const bRun = baselineRuns[r];\n    const cRun = candidateRuns[r];\n    const minLength = Math.min(bRun.items.length, cRun.items.length);\n    for (let i = 0; i < minLength; i++) {\n      const bItem = bRun.items[i];\n      const cItem = cRun.items[i];\n      if (bItem.error || cItem.error) continue;\n      for (const name of baselineScorerNames) {\n        const bScore = bItem.scores[name];\n        const cScore = cItem.scores[name];\n        if (bScore != null && cScore != null) {\n          pairedDiffs[name].push(cScore - bScore);\n        }\n      }\n    }\n  }\n\n  // Compute aggregate means across runs\n  const baselineMeans: Record<string, number> = {};\n  const candidateMeans: Record<string, number> = {};\n  for (const name of baselineScorerNames) {\n    baselineMeans[name] =\n      baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      baselineRuns.length;\n    candidateMeans[name] =\n      candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      candidateRuns.length;\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMeanRaw = baselineMeans[name];\n    const cMeanRaw = candidateMeans[name];\n    const deltaRaw = cMeanRaw - bMeanRaw;\n    const bMean = round(bMeanRaw);\n    const cMean = round(cMeanRaw);\n    const delta = round(deltaRaw);\n    const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n    const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n    const entry: EvalComparison['scorers'][string] = {\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      deltaPercent,\n    };\n\n    // Compute bootstrap CI when we have enough paired data\n    const diffs = pairedDiffs[name];\n    if (diffs.length >= 2) {\n      const ci = pairedBootstrapCI(diffs);\n      entry.ci = { lower: ci.lower, upper: ci.upper };\n      entry.pRegression = ci.pRegression;\n      entry.pImprovement = ci.pImprovement;\n      entry.n = diffs.length;\n      // Significant when CI excludes zero AND delta exceeds practical threshold\n      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n    }\n\n    scorers[name] = entry;\n  }\n\n  // Per-item regressions/improvements.\n  // For multi-run: average each item's score across runs to get a stable per-item comparison.\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const itemCount = Math.min(\n    ...baselineRuns.map((r) => r.items.length),\n    ...candidateRuns.map((r) => r.items.length),\n  );\n\n  for (let i = 0; i < itemCount; i++) {\n    // Check if any run has an error for this item\n    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n    if (hasBaselineError || hasCandidateError) continue;\n\n    for (const name of baselineScorerNames) {\n      // Average this item's score across all runs on each side\n      const bScores = baselineRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      const cScores = candidateRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      if (bScores.length === 0 || cScores.length === 0) continue;\n\n      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n      const delta = round(cAvg - bAvg);\n      const threshold = resolveThreshold(name, options, baselineRef.metadata);\n      if (delta < -threshold)\n        regressions.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n      else if (delta > threshold)\n        improvements.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (average across all runs for multi-run)\n  let timing: EvalComparison['timing'];\n  const baselineTimings = baselineRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  const candidateTimings = candidateRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n    timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n  }\n\n  // Cost comparison (per-run average for multi-run)\n  let cost: EvalComparison['cost'];\n  const baselineAvgCost =\n    baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n  const candidateAvgCost =\n    candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n  if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n    const deltaRaw = candidateAvgCost - baselineAvgCost;\n    const delta = round(deltaRaw);\n    const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n    cost = {\n      baselineTotal: round(baselineAvgCost),\n      candidateTotal: round(candidateAvgCost),\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      let sigLabel = '';\n      if (s.significant === true) sigLabel = ' (significant)';\n      else if (s.significant === false) sigLabel = ' (not significant)';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },\n    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n  concurrency?: number;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n  result: EvalResult,\n  scorers: Scorer[],\n  runtime: AxlRuntime,\n  options?: RescoreOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const concurrency = options?.concurrency ?? 5;\n\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const rescored: EvalItem[] = new Array(result.items.length);\n  let totalCost = 0;\n\n  async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n    // Pass through error items without scoring\n    if (original.error) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: original.error,\n        scores: {},\n      };\n      return;\n    }\n\n    // Preserve per-item traces from the original run: rescore re-runs scorers\n    // but leaves the workflow untouched, so the original execution traces\n    // remain accurate and useful for diagnosing score changes.\n    const item: EvalItem = {\n      input: original.input,\n      annotations: original.annotations,\n      output: original.output,\n      metadata: original.metadata,\n      traces: original.traces,\n      scores: {},\n      scoreDetails: {},\n    };\n\n    let itemScorerCost = 0;\n\n    for (const scorer of scorers) {\n      const scorerStart = Date.now();\n      try {\n        const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n        const scorerResult = normalizeScorerResult(raw);\n\n        if (scorerResult.cost != null) {\n          itemScorerCost += scorerResult.cost;\n          totalCost += scorerResult.cost;\n        }\n\n        const scorerDuration = Date.now() - scorerStart;\n\n        if (\n          !Number.isFinite(scorerResult.score) ||\n          scorerResult.score < 0 ||\n          scorerResult.score > 1\n        ) {\n          if (!item.scorerErrors) item.scorerErrors = [];\n          item.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n          );\n          item.scores[scorer.name] = null;\n          item.scoreDetails![scorer.name] = {\n            score: null,\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        } else {\n          item.scores[scorer.name] = round(scorerResult.score);\n          item.scoreDetails![scorer.name] = {\n            score: round(scorerResult.score),\n            metadata: scorerResult.metadata,\n            duration: scorerDuration,\n            cost: scorerResult.cost,\n          };\n        }\n      } catch (err) {\n        const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n        if (errCost != null) {\n          itemScorerCost += errCost;\n          totalCost += errCost;\n        }\n        if (!item.scorerErrors) item.scorerErrors = [];\n        item.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        item.scores[scorer.name] = null;\n        item.scoreDetails![scorer.name] = {\n          score: null,\n          duration: Date.now() - scorerStart,\n          cost: errCost,\n        };\n      }\n    }\n    item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n    rescored[itemIndex] = item;\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < result.items.length) {\n      const currentIndex = index++;\n      await rescoreItem(result.items[currentIndex], currentIndex);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, result.items.length) }, () =>\n    runNext(),\n  );\n  await Promise.all(workers);\n\n  const failures = rescored.filter((i) => i.error).length;\n  const scorerNames = scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = rescored\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  return {\n    id: randomUUID(),\n    dataset: result.dataset,\n    metadata: (() => {\n      // Strip run group membership — rescored results are independent evaluations.\n      // metadata.workflows is preserved via ...rest so the rescored result keeps\n      // the same workflow attribution as the original.\n      // eslint-disable-next-line @typescript-eslint/no-unused-vars\n      const { runGroupId: _, runIndex: __, ...rest } = result.metadata;\n      const merged: Record<string, unknown> = {\n        ...rest,\n        rescored: true,\n        originalId: result.id,\n        scorerTypes,\n      };\n      // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n      // as a top-level string field with no `metadata.workflows`. Migrate it\n      // forward so rescored results from old artifacts retain their workflow\n      // attribution under the modern shape.\n      if (!Array.isArray(merged.workflows)) {\n        const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n        if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n          merged.workflows = [legacyWorkflow];\n          merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n        }\n      }\n      return merged;\n    })(),\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: rescored,\n    summary: { count: result.items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n  runGroupId: string;\n  runCount: number;\n  /**\n   * Workflow names observed across all runs in the group, deduped and ordered\n   * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n   * multi-run group produced by `axl-eval --runs N` typically has one entry;\n   * heterogeneous groups (custom callbacks) can have multiple.\n   */\n  workflows: string[];\n  dataset: string;\n  totalCost: number;\n  totalDuration: number;\n  scorers: Record<string, { mean: number; std: number; min: number; max: number }>;\n  timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n  if (values.length <= 1) return 0;\n  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n  return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n  if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n  const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n  const { dataset } = runs[0];\n\n  // Union workflow names across all runs in the group, first-seen first.\n  // Most groups have one, but custom callbacks can produce heterogeneous ones.\n  const seenWorkflows = new Set<string>();\n  const workflows: string[] = [];\n  for (const run of runs) {\n    const list = run.metadata.workflows;\n    if (Array.isArray(list)) {\n      for (const w of list) {\n        if (typeof w === 'string' && !seenWorkflows.has(w)) {\n          seenWorkflows.add(w);\n          workflows.push(w);\n        }\n      }\n    }\n  }\n\n  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n  // Collect per-scorer means from each run\n  const scorerNames = Object.keys(runs[0].summary.scorers);\n  const scorers: MultiRunSummary['scorers'] = {};\n\n  for (const name of scorerNames) {\n    const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);\n    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n    const minMean = Math.min(...means);\n    const maxMean = Math.max(...means);\n\n    scorers[name] = {\n      mean: round(meanOfMeans),\n      std: round(std(means, meanOfMeans)),\n      min: round(minMean),\n      max: round(maxMean),\n    };\n  }\n\n  // Timing aggregation\n  let timing: MultiRunSummary['timing'];\n  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n  if (timingMeans.length === runs.length && timingMeans.length > 0) {\n    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n    timing = {\n      mean: round(meanTiming),\n      std: round(std(timingMeans, meanTiming)),\n    };\n  }\n\n  return {\n    runGroupId,\n    runCount: runs.length,\n    workflows,\n    dataset,\n    totalCost: round(totalCost),\n    totalDuration: round(totalDuration),\n    scorers,\n    timing,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACrDO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC/DA,iBAAkB;AAElB,iBAA6C;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,aAAE,OAAO,EAAE,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,aAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,cAAU,4BAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,UAAM,wBAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACJO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEO,SAAS,MAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;;;ACdA,yBAA2B;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAkBA,SAAS,gBAAgB,QAAiB,QAAQ,mBAAuC;AACvF,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,MAAO,OAA8B;AAC3C,MAAI,QAAQ,OAAW,QAAO;AAC9B,MAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,KAAK,OAAO,EAAG,QAAO;AAExE,UAAQ;AAAA,IACN,6CAA6C,KAAK,qDAAqD,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI,OAAO,GAAG;AAAA,EAC3J;AACA,SAAO;AACT;AASA,SAAS,oBACP,QACA,QAAQ,mBAC6B;AACrC,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,OAAQ,OAAkC;AAChD,MAAI,SAAS,OAAW,QAAO;AAC/B,MAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;AACpE,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAuC,SAAS,OAAO,SAAS,MAAM,QAAQ,IAAI,IAAI,UAAU,OAAO,IAAI;AAAA,IACnK;AACA,WAAO;AAAA,EACT;AAIA,QAAM,QAAQ,OAAO,eAAe,IAAI;AACxC,MAAI,UAAU,OAAO,aAAa,UAAU,MAAM;AAChD,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAwC,OAAO,aAAa,QAA+B,eAAe;AAAA,IAClK;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,eAAsB,QACpB,QACA,iBAIA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,SAAS,QAAQ,SAAS;AAC5B,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AAKF,UAAI,SAAS,eAAe;AAC1B,cAAM,UAAU,MAAM,QAAQ;AAAA,UAC5B,YAAY,gBAAgB,KAAK,OAAO,OAAO;AAAA,UAC/C,EAAE,eAAe,KAAK;AAAA,QACxB;AACA,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,QAAQ,OAAO;AAKjC,iBAAS,OAAO,gBAAgB,QAAQ,MAAM,KAAK,QAAQ;AAC3D,iBAAS,WAAW,oBAAoB,QAAQ,MAAM,KAAK,QAAQ;AACnE,YAAI,QAAQ,UAAU,QAAQ,OAAO,SAAS,GAAG;AAC/C,mBAAS,SAAS,QAAQ;AAAA,QAC5B;AACA,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF,OAAO;AACL,cAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,OAAO;AACzB,iBAAS,OAAO,gBAAgB,MAAM;AACtC,cAAM,OAAO,oBAAoB,MAAM;AACvC,YAAI,KAAM,UAAS,WAAW;AAC9B,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAIhE,UAAI,SAAS,eAAe;AAC1B,cAAM,WAAY,IAAwC;AAC1D,YAAI,MAAM,QAAQ,QAAQ,KAAK,SAAS,SAAS,GAAG;AAClD,mBAAS,SAAS;AAAA,QACpB;AAAA,MACF;AACA,gBAAU,SAAS,IAAI;AACvB,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAI,SAAS,QAAQ,QAAS;AAC9B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACvD,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AACvB,aAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAAA,EAClF;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAGA,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,WAAS,aAAa,EAAE,MAAM,YAAY,YAAY,MAAM,QAAQ,SAAS,CAAC;AAE9E,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;ACtWO,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAOC,OAAM,IAAI;AAAA,MACjB,OAAOA,OAAM,IAAI;AAAA,MACjB,MAAMA,OAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAOA,OAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAOA,OAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAMA,OAAM,IAAI;AAAA,IAChB,aAAaA,OAAM,YAAY,UAAU;AAAA,IACzC,cAAcA,OAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;AC5FA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACnE,QAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAEvE,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAGA,QAAM,cAAc,aAAa,CAAC;AAClC,QAAM,eAAe,cAAc,CAAC;AAEpC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,QAAM,WAAW,KAAK,IAAI,aAAa,QAAQ,cAAc,MAAM;AACnE,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAEtE,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,IACF;AAGA,UAAM,QAAQ,YAAY,IAAI;AAC9B,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AACxB,YAAM,IAAI,MAAM;AAEhB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAE/J,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,YAAY,IAAI,UAAU,YAAY,SAAS;AAAA,IAC/D,WAAW,EAAE,IAAI,aAAa,IAAI,UAAU,aAAa,SAAS;AAAA,IAClE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;ACjQA,IAAAC,sBAA2B;AAU3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAE5C,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAE/E,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAKA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,SAAS;AAAA,MACjB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAEA,QAAI,iBAAiB;AAErB,eAAWC,WAAU,SAAS;AAC5B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,eAAK,aAAa;AAAA,YAChB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,eAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,eAAK,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACnD,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,aAAK,aAAa;AAAA,UAChB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,aAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,UAChC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,aAAS,SAAS,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,OAAO,MAAM,QAAQ;AAClC,YAAM,eAAe;AACrB,YAAM,YAAY,OAAO,MAAM,YAAY,GAAG,YAAY;AAAA,IAC5D;AAAA,EACF;AAEA,QAAM,UAAU,MAAM;AAAA,IAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,MAAM,EAAE;AAAA,IAAG,MACjF,QAAQ;AAAA,EACV;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,QAAI,gCAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAKf,YAAM,EAAE,YAAY,GAAG,UAAU,IAAI,GAAG,KAAK,IAAI,OAAO;AACxD,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;AC5LA,IAAAC,sBAA2B;AAoB3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,kBAAyB,gCAAW;AACzE,QAAM,EAAE,SAAAC,SAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AAChE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAEjC,YAAQ,IAAI,IAAI;AAAA,MACd,MAAM,MAAM,WAAW;AAAA,MACvB,KAAK,MAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAK,MAAM,OAAO;AAAA,MAClB,KAAK,MAAM,OAAO;AAAA,IACpB;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAM,MAAM,UAAU;AAAA,MACtB,KAAK,MAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA,SAAAA;AAAA,IACA,WAAW,MAAM,SAAS;AAAA,IAC1B,eAAe,MAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;","names":["scorer","round","round","import_node_crypto","scorer","import_node_crypto","dataset"]}

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { AxlRuntime } from '@axlsdk/axl';
+import { TraceEvent, AxlRuntime } from '@axlsdk/axl';
 
 type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
@@ -133,6 +133,11 @@ type EvalItem = {
     scoreDetails?: Record<string, ScorerDetail>;
     /** Execution metadata forwarded from the runtime (models, tokens, agentCalls, etc). */
     metadata?: Record<string, unknown>;
+    /** Trace events captured during this item's execution. Only populated when
+     *  `runEval` was called with `{ captureTraces: true }`. Verbose-mode
+     *  `agent_call.data.messages` snapshots are stripped to keep memory bounded;
+     *  subscribe to `runtime.on('trace', ...)` directly if you need those. */
+    traces?: TraceEvent[];
 };
 type EvalSummary = {
     count: number;
@@ -205,6 +210,47 @@ type EvalCompareOptions = {
     /** Global threshold or per-scorer map. Default: auto-calibrate from scorerTypes metadata. */
     thresholds?: Record<string, number> | number;
 };
+/**
+ * Emitted by `runEval` at two points:
+ *
+ * - `'item_done'` — after each dataset item is fully processed (executed +
+ *   scored). Emitted for every item regardless of outcome: success, workflow
+ *   error, scorer error, cancelled (via `signal`), or budget exhaustion.
+ * - `'run_done'` — once after all items have finished and the final summary
+ *   has been computed. Includes total item count and failure count so
+ *   consumers can show a completion toast without waiting for the full result.
+ *
+ * Consumers should narrow on `type` — the shape is a discriminated union.
+ */
+type EvalProgressEvent = {
+    type: 'item_done';
+    itemIndex: number;
+    totalItems: number;
+} | {
+    type: 'run_done';
+    totalItems: number;
+    failures: number;
+};
+/** Optional runtime behavior for `runEval()`. */
+type RunEvalOptions = {
+    /** Called after each dataset item completes (execution + scoring). */
+    onProgress?: (event: EvalProgressEvent) => void;
+    /** Abort signal — checked before starting each item. */
+    signal?: AbortSignal;
+    /**
+     * Capture per-item `TraceEvent[]` from the runtime and store them on
+     * `EvalItem.traces`. Off by default because traces multiply memory with
+     * dataset size × turns × agents. When on, the runner wraps the user-provided
+     * `executeWorkflow` with `runtime.trackExecution({ captureTraces: true })`,
+     * so any `runtime.execute()` / `ctx.ask()` activity inside the callback is
+     * captured and scoped to the current item.
+     *
+     * Note: verbose-mode `agent_call.data.messages` snapshots are stripped from
+     * captured traces to keep memory bounded — if you need the full verbose
+     * payload, subscribe to `runtime.on('trace', ...)` directly.
+     */
+    captureTraces?: boolean;
+};
 
 declare function defineEval(config: EvalConfig): EvalConfig;
 
@@ -212,7 +258,7 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
     output: unknown;
     cost?: number;
     metadata?: Record<string, unknown>;
-}>, runtime: AxlRuntime): Promise<EvalResult>;
+}>, runtime: AxlRuntime, options?: RunEvalOptions): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
 
@@ -278,4 +324,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import { AxlRuntime } from '@axlsdk/axl';
+import { TraceEvent, AxlRuntime } from '@axlsdk/axl';
 
 type DatasetItem<TInput, TAnnotations = undefined> = {
     input: TInput;
@@ -133,6 +133,11 @@ type EvalItem = {
     scoreDetails?: Record<string, ScorerDetail>;
     /** Execution metadata forwarded from the runtime (models, tokens, agentCalls, etc). */
     metadata?: Record<string, unknown>;
+    /** Trace events captured during this item's execution. Only populated when
+     *  `runEval` was called with `{ captureTraces: true }`. Verbose-mode
+     *  `agent_call.data.messages` snapshots are stripped to keep memory bounded;
+     *  subscribe to `runtime.on('trace', ...)` directly if you need those. */
+    traces?: TraceEvent[];
 };
 type EvalSummary = {
     count: number;
@@ -205,6 +210,47 @@ type EvalCompareOptions = {
     /** Global threshold or per-scorer map. Default: auto-calibrate from scorerTypes metadata. */
     thresholds?: Record<string, number> | number;
 };
+/**
+ * Emitted by `runEval` at two points:
+ *
+ * - `'item_done'` — after each dataset item is fully processed (executed +
+ *   scored). Emitted for every item regardless of outcome: success, workflow
+ *   error, scorer error, cancelled (via `signal`), or budget exhaustion.
+ * - `'run_done'` — once after all items have finished and the final summary
+ *   has been computed. Includes total item count and failure count so
+ *   consumers can show a completion toast without waiting for the full result.
+ *
+ * Consumers should narrow on `type` — the shape is a discriminated union.
+ */
+type EvalProgressEvent = {
+    type: 'item_done';
+    itemIndex: number;
+    totalItems: number;
+} | {
+    type: 'run_done';
+    totalItems: number;
+    failures: number;
+};
+/** Optional runtime behavior for `runEval()`. */
+type RunEvalOptions = {
+    /** Called after each dataset item completes (execution + scoring). */
+    onProgress?: (event: EvalProgressEvent) => void;
+    /** Abort signal — checked before starting each item. */
+    signal?: AbortSignal;
+    /**
+     * Capture per-item `TraceEvent[]` from the runtime and store them on
+     * `EvalItem.traces`. Off by default because traces multiply memory with
+     * dataset size × turns × agents. When on, the runner wraps the user-provided
+     * `executeWorkflow` with `runtime.trackExecution({ captureTraces: true })`,
+     * so any `runtime.execute()` / `ctx.ask()` activity inside the callback is
+     * captured and scoped to the current item.
+     *
+     * Note: verbose-mode `agent_call.data.messages` snapshots are stripped from
+     * captured traces to keep memory bounded — if you need the full verbose
+     * payload, subscribe to `runtime.on('trace', ...)` directly.
+     */
+    captureTraces?: boolean;
+};
 
 declare function defineEval(config: EvalConfig): EvalConfig;
 
@@ -212,7 +258,7 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
     output: unknown;
     cost?: number;
     metadata?: Record<string, unknown>;
-}>, runtime: AxlRuntime): Promise<EvalResult>;
+}>, runtime: AxlRuntime, options?: RunEvalOptions): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
 
@@ -278,4 +324,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.js

@@ -6,7 +6,7 @@ import {
   rescore,
   runEval,
   scorer
-} from "./chunk-T4OFZQCJ.js";
+} from "./chunk-CV6VE5VT.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.14.0"
+    "@axlsdk/axl": "0.15.0"
   },
   "peerDependencies": {
     "zod": "^4.0.0"