@axlsdk/eval 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (14) hide show
  1. package/README.md +46 -2
  2. package/dist/{chunk-T4OFZQCJ.js → chunk-CV6VE5VT.js} +79 -9
  3. package/dist/chunk-CV6VE5VT.js.map +1 -0
  4. package/dist/cli.cjs +92 -13
  5. package/dist/cli.cjs.map +1 -1
  6. package/dist/cli.js +15 -6
  7. package/dist/cli.js.map +1 -1
  8. package/dist/index.cjs +78 -8
  9. package/dist/index.cjs.map +1 -1
  10. package/dist/index.d.cts +49 -3
  11. package/dist/index.d.ts +49 -3
  12. package/dist/index.js +1 -1
  13. package/package.json +2 -2
  14. package/dist/chunk-T4OFZQCJ.js.map +0 -1
package/dist/cli.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/cli.ts","../src/bootstrap.ts","../src/compare.ts","../src/scorer.ts","../src/utils.ts","../src/runner.ts","../src/rescore.ts","../src/multi-run.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport { rescore } from './rescore.js';\nimport { aggregateRuns } from './multi-run.js';\nimport type { MultiRunSummary } from './multi-run.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set([\n '--output',\n '--config',\n '--conditions',\n '--fail-on-regression',\n '--threshold',\n '--runs',\n]);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --runs <n> Run eval N times (multi-run)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval rescore <results> <eval-file> Re-run scorers on saved outputs\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --threshold <v> Set regression threshold (global or per-scorer)\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n if (args[0] === 'rescore') {\n await runRescore(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nfunction parseThresholdArg(args: string[]): Record<string, number> | number | undefined {\n const idx = args.indexOf('--threshold');\n if (idx === -1 || idx + 1 >= args.length) return undefined;\n const value = args[idx + 1];\n\n // Global numeric threshold: --threshold 0.05\n if (/^[\\d.]+$/.test(value)) return parseFloat(value);\n\n // Per-scorer map: --threshold accuracy=0,tone=0.1\n const map: Record<string, number> = {};\n for (const pair of value.split(',')) {\n const [key, val] = pair.split('=');\n if (key && val != null) map[key.trim()] = parseFloat(val.trim());\n }\n return Object.keys(map).length > 0 ? map : undefined;\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const thresholds = parseThresholdArg(args);\n const files = args.filter(\n (a, i) => !a.startsWith('--') && !(i > 0 && args[i - 1] === '--threshold'),\n );\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]');\n process.exit(1);\n }\n\n const baselineRaw = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidateRaw = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const baseline: EvalResult | EvalResult[] = baselineRaw;\n const candidate: EvalResult | EvalResult[] = candidateRaw;\n const compareOptions = thresholds != null ? { thresholds } : undefined;\n const comparison = evalCompare(baseline, candidate, compareOptions);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n const hasCI = scorerNames.some((n) => comparison.scorers[n].ci != null);\n\n const ciHeader = hasCI ? ' CI 95% Sig' : '';\n const ciRule = hasCI ? ' ---------------- ---' : '';\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change${ciHeader}`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------${ciRule}`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n let line = ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`;\n if (hasCI) {\n if (s.ci) {\n const lo = (s.ci.lower >= 0 ? '+' : '') + s.ci.lower.toFixed(4);\n const hi = (s.ci.upper >= 0 ? '+' : '') + s.ci.upper.toFixed(4);\n const sig = s.significant ? ' *' : ' ';\n line += ` [${lo}, ${hi}]${sig}`;\n } else {\n line += ' —'.padEnd(22);\n }\n }\n console.log(line);\n }\n\n if (comparison.timing) {\n const t = comparison.timing;\n const sign = t.delta > 0 ? '+' : '';\n console.log(\n `\\n Timing: baseline ${(t.baselineMean / 1000).toFixed(2)}s -> candidate ${(t.candidateMean / 1000).toFixed(2)}s (${sign}${t.deltaPercent.toFixed(1)}%)`,\n );\n }\n if (comparison.cost) {\n const c = comparison.cost;\n const sign = c.delta > 0 ? '+' : '';\n console.log(\n ` Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`,\n );\n }\n\n const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;\n const stable = Math.max(\n 0,\n baselineRef.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n // When CI is available, only fail on significant regressions\n const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);\n if (hasSignificance) {\n const hasSignificantRegression = scorerNames.some(\n (n) => comparison.scorers[n].significant === true && comparison.scorers[n].delta < 0,\n );\n if (hasSignificantRegression) process.exit(1);\n } else {\n process.exit(1);\n }\n }\n}\n\nasync function runRescore(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length < 2) {\n console.error('Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]');\n process.exit(1);\n }\n\n const [resultsPath, evalFilePath] = paths;\n const raw = JSON.parse(await readFileAsync(resultsPath, 'utf-8'));\n const results: EvalResult[] = Array.isArray(raw) ? raw : [raw];\n\n const runtime = await getRuntime(configArg, conditions);\n try {\n const mod = await importModule(path.resolve(evalFilePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.scorers || evalConfig.scorers.length === 0) {\n console.error(`Error: ${evalFilePath} does not export scorers`);\n process.exit(1);\n }\n\n const rescored: EvalResult[] = [];\n for (const resultData of results) {\n rescored.push(await rescore(resultData, evalConfig.scorers, runtime));\n }\n\n for (const r of rescored) {\n console.log('\\n' + formatTable(r) + '\\n');\n }\n\n if (outputPath) {\n const output = Array.isArray(raw) ? rescored : rescored[0];\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Rescored results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\n/** Render observed workflows for a terse header line. Multiple → comma-separated. */\nfunction formatWorkflows(workflows: unknown): string {\n if (!Array.isArray(workflows) || workflows.length === 0) return '(unknown)';\n return (workflows as unknown[]).filter((w): w is string => typeof w === 'string').join(', ');\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(\n `Eval: ${formatWorkflows(result.metadata.workflows)} x ${result.dataset} (${result.summary.count} items)`,\n );\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n // Detect scorers with no valid scores (all items errored → computeStats([]) → all zeros)\n const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;\n if (validScoreCount === 0) {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)}`,\n );\n } else {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n }\n\n if (result.summary.timing) {\n const t = result.summary.timing;\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${(t.mean / 1000).toFixed(2).padStart(colWidth)}s ${(t.p50 / 1000).toFixed(2).padStart(colWidth)}s ${(t.p95 / 1000).toFixed(2).padStart(colWidth)}s`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n );\n\n const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);\n if (itemsWithErrors.length > 0) {\n const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors!))];\n lines.push('');\n lines.push(\n ` Scorer errors (${itemsWithErrors.length}/${result.summary.count} items affected):`,\n );\n for (const err of uniqueErrors.slice(0, 5)) {\n lines.push(` - ${err}`);\n }\n if (uniqueErrors.length > 5) {\n lines.push(` ... and ${uniqueErrors.length - 5} more`);\n }\n }\n\n return lines.join('\\n');\n}\n\nfunction formatMultiRunTable(summary: MultiRunSummary): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 16;\n\n lines.push(\n `Eval: ${formatWorkflows(summary.workflows)} x ${summary.dataset} \\u2014 ${summary.runCount} runs`,\n );\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean \\u00b1 Std'.padStart(colWidth)} ${'Min'.padStart(8)} ${'Max'.padStart(8)}`,\n );\n const ruleLen = maxNameLen + 2 + colWidth + 2 + 8 + 2 + 8;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = summary.scorers[name];\n const meanStd = `${s.mean.toFixed(3)} \\u00b1 ${s.std.toFixed(3)}`;\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${meanStd.padStart(colWidth)} ${s.min.toFixed(3).padStart(8)} ${s.max.toFixed(3).padStart(8)}`,\n );\n }\n\n if (summary.timing) {\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${((summary.timing.mean / 1000).toFixed(2) + ' \\u00b1 ' + (summary.timing.std / 1000).toFixed(2) + 's').padStart(colWidth)}`,\n );\n }\n\n const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : '$0.00';\n const durationStr = (summary.totalDuration / 1000).toFixed(1);\n lines.push('');\n lines.push(` Total Cost: ${costStr} | Total Duration: ${durationStr}s`);\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime || typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' || ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n runs: number;\n paths: string[];\n} {\n let outputPath: string | undefined;\n let configArg: string | undefined;\n let conditions: string[] = [];\n let runs = 1;\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' || arg === '--config' || arg === '--conditions' || arg === '--runs') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else if (arg === '--runs') runs = Math.max(1, parseInt(value, 10) || 1);\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, runs, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, runs, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackExecution for cost + metadata attribution\n executeWorkflow = async (input, rt) => {\n const {\n result,\n cost: trackedCost,\n metadata,\n } = await runtime.trackExecution(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return {\n output: result.output,\n cost: result.cost ?? trackedCost,\n metadata: result.metadata ?? metadata,\n };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost, metadata } = await runtime.trackExecution(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost, metadata };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n if (runs > 1) {\n // Multi-run mode\n const { randomUUID } = await import('node:crypto');\n const runGroupId = randomUUID();\n const runResults: EvalResult[] = [];\n\n for (let r = 0; r < runs; r++) {\n console.error(`[axl-eval] Run ${r + 1}/${runs}...`);\n const result = await runEval(evalConfig, executeWorkflow, runtime);\n result.metadata.runGroupId = runGroupId;\n result.metadata.runIndex = r;\n runResults.push(result);\n results.push(result);\n }\n\n const summary = aggregateRuns(runResults);\n console.log('\\n' + formatMultiRunTable(summary) + '\\n');\n } else {\n const result = await runEval(evalConfig, executeWorkflow, runtime);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n }\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","export type BootstrapCIResult = {\n /** Lower bound of the confidence interval. */\n lower: number;\n /** Upper bound of the confidence interval. */\n upper: number;\n /** Mean of the original differences. */\n mean: number;\n /** Proportion of resamples where the mean difference is negative (probability of regression). */\n pRegression: number;\n /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n differences: number[],\n options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n const n = differences.length;\n if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n const nResamples = options?.nResamples ?? 1000;\n const alpha = options?.alpha ?? 0.05;\n\n const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n if (n === 1)\n return {\n lower: differences[0],\n upper: differences[0],\n mean,\n pRegression: mean < 0 ? 1 : 0,\n pImprovement: mean > 0 ? 1 : 0,\n };\n\n if (nResamples <= 0)\n return {\n lower: round(mean),\n upper: round(mean),\n mean: round(mean),\n pRegression: mean < 0 ? 1 : 0,\n pImprovement: mean > 0 ? 1 : 0,\n };\n\n // Seeded or random number generator\n const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n const resampleMeans: number[] = new Array(nResamples);\n for (let r = 0; r < nResamples; r++) {\n let sum = 0;\n for (let i = 0; i < n; i++) {\n sum += differences[Math.floor(rng() * n)];\n }\n resampleMeans[r] = sum / n;\n }\n\n resampleMeans.sort((a, b) => a - b);\n\n const lowerIdx = Math.floor((alpha / 2) * nResamples);\n const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n // Count resamples below/above zero for directional probabilities\n // resampleMeans is already sorted, so binary search for zero\n let belowZero = 0;\n for (let i = 0; i < nResamples; i++) {\n if (resampleMeans[i] < 0) belowZero++;\n else break;\n }\n let aboveZero = 0;\n for (let i = nResamples - 1; i >= 0; i--) {\n if (resampleMeans[i] > 0) aboveZero++;\n else break;\n }\n\n return {\n lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n mean: round(mean),\n pRegression: round(belowZero / nResamples),\n pImprovement: round(aboveZero / nResamples),\n };\n}\n\nfunction round(n: number): number {\n return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n let state = seed | 0 || 1; // Ensure non-zero\n return () => {\n state ^= state << 13;\n state ^= state >> 17;\n state ^= state << 5;\n return (state >>> 0) / 0x100000000;\n };\n}\n","import type {\n EvalResult,\n EvalComparison,\n EvalCompareOptions,\n EvalRegression,\n EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\n\nfunction round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n scorerName: string,\n options: EvalCompareOptions | undefined,\n metadata: Record<string, unknown>,\n): number {\n // Explicit global threshold\n if (typeof options?.thresholds === 'number') return options.thresholds;\n\n // Per-scorer threshold map\n if (options?.thresholds && typeof options.thresholds === 'object') {\n const perScorer = options.thresholds[scorerName];\n if (perScorer != null) return perScorer;\n // Fall through to auto-calibration if scorer not in map\n }\n\n // Auto-calibrate from scorerTypes metadata\n const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n if (scorerTypes && scorerName in scorerTypes) {\n return scorerTypes[scorerName] === 'llm'\n ? DEFAULT_LLM_THRESHOLD\n : DEFAULT_DETERMINISTIC_THRESHOLD;\n }\n\n // Legacy fallback for results without scorerTypes\n return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n baseline: EvalResult | EvalResult[],\n candidate: EvalResult | EvalResult[],\n options?: EvalCompareOptions,\n): EvalComparison {\n const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];\n const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];\n\n if (baselineRuns.length === 0 || candidateRuns.length === 0) {\n throw new Error('Cannot compare empty eval result arrays');\n }\n\n // Use the first run as representative for metadata, dataset, scorers\n const baselineRef = baselineRuns[0];\n const candidateRef = candidateRuns[0];\n\n if (baselineRef.dataset !== candidateRef.dataset) {\n throw new Error(\n `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n );\n }\n\n const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n throw new Error(`Cannot compare evals with different scorers`);\n }\n\n // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n const pairedDiffs: Record<string, number[]> = {};\n for (const name of baselineScorerNames) {\n pairedDiffs[name] = [];\n }\n const runCount = Math.min(baselineRuns.length, candidateRuns.length);\n for (let r = 0; r < runCount; r++) {\n const bRun = baselineRuns[r];\n const cRun = candidateRuns[r];\n const minLength = Math.min(bRun.items.length, cRun.items.length);\n for (let i = 0; i < minLength; i++) {\n const bItem = bRun.items[i];\n const cItem = cRun.items[i];\n if (bItem.error || cItem.error) continue;\n for (const name of baselineScorerNames) {\n const bScore = bItem.scores[name];\n const cScore = cItem.scores[name];\n if (bScore != null && cScore != null) {\n pairedDiffs[name].push(cScore - bScore);\n }\n }\n }\n }\n\n // Compute aggregate means across runs\n const baselineMeans: Record<string, number> = {};\n const candidateMeans: Record<string, number> = {};\n for (const name of baselineScorerNames) {\n baselineMeans[name] =\n baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n baselineRuns.length;\n candidateMeans[name] =\n candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n candidateRuns.length;\n }\n\n const scorers: EvalComparison['scorers'] = {};\n for (const name of baselineScorerNames) {\n const bMeanRaw = baselineMeans[name];\n const cMeanRaw = candidateMeans[name];\n const deltaRaw = cMeanRaw - bMeanRaw;\n const bMean = round(bMeanRaw);\n const cMean = round(cMeanRaw);\n const delta = round(deltaRaw);\n const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n const entry: EvalComparison['scorers'][string] = {\n baselineMean: bMean,\n candidateMean: cMean,\n delta,\n deltaPercent,\n };\n\n // Compute bootstrap CI when we have enough paired data\n const diffs = pairedDiffs[name];\n if (diffs.length >= 2) {\n const ci = pairedBootstrapCI(diffs);\n entry.ci = { lower: ci.lower, upper: ci.upper };\n entry.pRegression = ci.pRegression;\n entry.pImprovement = ci.pImprovement;\n entry.n = diffs.length;\n // Significant when CI excludes zero AND delta exceeds practical threshold\n const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n }\n\n scorers[name] = entry;\n }\n\n // Per-item regressions/improvements.\n // For multi-run: average each item's score across runs to get a stable per-item comparison.\n const regressions: EvalRegression[] = [];\n const improvements: EvalImprovement[] = [];\n\n const itemCount = Math.min(\n ...baselineRuns.map((r) => r.items.length),\n ...candidateRuns.map((r) => r.items.length),\n );\n\n for (let i = 0; i < itemCount; i++) {\n // Check if any run has an error for this item\n const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n if (hasBaselineError || hasCandidateError) continue;\n\n for (const name of baselineScorerNames) {\n // Average this item's score across all runs on each side\n const bScores = baselineRuns\n .map((r) => r.items[i]?.scores[name])\n .filter((s): s is number => s != null);\n const cScores = candidateRuns\n .map((r) => r.items[i]?.scores[name])\n .filter((s): s is number => s != null);\n if (bScores.length === 0 || cScores.length === 0) continue;\n\n const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n const delta = round(cAvg - bAvg);\n const threshold = resolveThreshold(name, options, baselineRef.metadata);\n if (delta < -threshold)\n regressions.push({\n itemIndex: i,\n input: baselineRef.items[i]?.input,\n scorer: name,\n baselineScore: round(bAvg),\n candidateScore: round(cAvg),\n delta,\n });\n else if (delta > threshold)\n improvements.push({\n itemIndex: i,\n input: baselineRef.items[i]?.input,\n scorer: name,\n baselineScore: round(bAvg),\n candidateScore: round(cAvg),\n delta,\n });\n }\n }\n\n // Timing comparison (average across all runs for multi-run)\n let timing: EvalComparison['timing'];\n const baselineTimings = baselineRuns\n .filter((r) => r.summary.timing)\n .map((r) => r.summary.timing!.mean);\n const candidateTimings = candidateRuns\n .filter((r) => r.summary.timing)\n .map((r) => r.summary.timing!.mean);\n if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n }\n\n // Cost comparison (per-run average for multi-run)\n let cost: EvalComparison['cost'];\n const baselineAvgCost =\n baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n const candidateAvgCost =\n candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n const deltaRaw = candidateAvgCost - baselineAvgCost;\n const delta = round(deltaRaw);\n const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n cost = {\n baselineTotal: round(baselineAvgCost),\n candidateTotal: round(candidateAvgCost),\n delta,\n deltaPercent,\n };\n }\n\n const parts: string[] = [];\n for (const name of baselineScorerNames) {\n const s = scorers[name];\n if (Math.abs(s.delta) > 0.001) {\n const direction = s.delta > 0 ? 'improves' : 'regresses';\n const sign = s.delta > 0 ? '+' : '';\n let sigLabel = '';\n if (s.significant === true) sigLabel = ' (significant)';\n else if (s.significant === false) sigLabel = ' (not significant)';\n parts.push(\n `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n );\n }\n }\n if (timing && Math.abs(timing.deltaPercent) > 1) {\n const dir = timing.delta > 0 ? 'slower' : 'faster';\n parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n }\n if (cost && Math.abs(cost.deltaPercent) > 1) {\n const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n }\n const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n return {\n baseline: { id: baselineRef.id, metadata: baselineRef.metadata },\n candidate: { id: candidateRef.id, metadata: candidateRef.metadata },\n scorers,\n timing,\n cost,\n regressions,\n improvements,\n summary: summaryStr,\n };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n score: number;\n metadata?: Record<string, unknown>;\n cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n /** Resolve a provider:model URI to a provider instance and model name. */\n resolveProvider: (modelUri: string) => {\n provider: {\n chat(\n messages: { role: string; content: string }[],\n options: {\n model: string;\n temperature?: number;\n responseFormat?: { type: string; json_schema?: unknown };\n },\n ): Promise<{ content: string; cost?: number }>;\n };\n model: string;\n };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n name: string;\n description: string;\n score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly description: string;\n readonly isLlm: boolean;\n score(\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n context?: ScorerContext,\n ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n return {\n name: config.name,\n description: config.description,\n isLlm: false,\n score: config.score,\n };\n}\n","export function computeStats(scores: number[]): {\n mean: number;\n min: number;\n max: number;\n p50: number;\n p95: number;\n} {\n if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n const sorted = [...scores].sort((a, b) => a - b);\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n const min = sorted[0];\n const max = sorted[sorted.length - 1];\n const p50 = sorted[Math.floor(sorted.length * 0.5)];\n const p95 = sorted[Math.floor(sorted.length * 0.95)];\n return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n const match = cost.match(/^\\$?([\\d.]+)$/);\n if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n return parseFloat(match[1]);\n}\n\nexport async function runEval(\n config: EvalConfig,\n executeWorkflow: (\n input: unknown,\n runtime: AxlRuntime,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n runtime: AxlRuntime,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const id = randomUUID();\n const items = await config.dataset.getItems();\n const concurrency = config.concurrency ?? 5;\n const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n // Create a scorer context that LLM scorers use to resolve providers.\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const evalItems: EvalItem[] = new Array(items.length);\n let totalCost = 0;\n let budgetExceeded = false;\n\n async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n if (budgetExceeded) {\n evalItems[itemIndex] = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Budget exceeded',\n scores: {},\n };\n return;\n }\n\n const evalItem: EvalItem = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n scores: {},\n };\n const itemStart = Date.now();\n try {\n const result = await executeWorkflow(item.input, runtime);\n evalItem.duration = Date.now() - itemStart;\n evalItem.output = result.output;\n evalItem.cost = result.cost;\n if (result.metadata) evalItem.metadata = result.metadata;\n if (result.cost != null) {\n totalCost += result.cost;\n }\n } catch (err) {\n evalItem.duration = Date.now() - itemStart;\n evalItem.error = err instanceof Error ? err.message : String(err);\n evalItems[itemIndex] = evalItem;\n return;\n }\n\n if (budgetLimit != null && totalCost > budgetLimit) {\n budgetExceeded = true;\n }\n\n evalItem.scoreDetails = {};\n let itemScorerCost = 0;\n\n for (const scorer of config.scorers) {\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(\n evalItem.output,\n item.input,\n item.annotations,\n scorerContext,\n );\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n evalItem.scores[scorer.name] = round(scorerResult.score);\n evalItem.scoreDetails[scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n // Capture cost from error (LLM scorer attaches it)\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n evalItems[itemIndex] = evalItem;\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < items.length) {\n const currentIndex = index++;\n await processItem(items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n await Promise.all(workers);\n\n const failures = evalItems.filter((i) => i.error).length;\n const scorerNames = config.scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = evalItems\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n const scorerTypes: Record<string, string> = {};\n for (const s of config.scorers) {\n scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n }\n\n // Aggregate per-model LLM call counts across all items\n const totalModelCalls = new Map<string, number>();\n for (const item of evalItems) {\n const itemCounts = item.metadata?.modelCallCounts;\n if (itemCounts && typeof itemCounts === 'object') {\n for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n if (typeof count === 'number')\n totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n }\n } else {\n // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n const itemModels = item.metadata?.models;\n if (Array.isArray(itemModels)) {\n for (const m of itemModels) {\n if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n }\n }\n }\n }\n\n // models: unique list sorted by total calls (most-called first)\n // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n const modelsMeta: Record<string, unknown> = {};\n if (totalModelCalls.size > 0) {\n const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n modelsMeta.models = sorted.map(([m]) => m);\n modelsMeta.modelCounts = Object.fromEntries(sorted);\n }\n\n // Aggregate per-workflow call counts across all items (parallel to models).\n // Workflows come from trackExecution's trace-event collection — callers\n // don't specify workflow names anywhere, they just appear because the\n // runtime emits workflow_start events for every execute() call.\n const totalWorkflowCalls = new Map<string, number>();\n for (const item of evalItems) {\n const itemCounts = item.metadata?.workflowCallCounts;\n if (itemCounts && typeof itemCounts === 'object') {\n for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n if (typeof count === 'number')\n totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n }\n } else {\n // Fallback: count unique workflows per item\n const itemWorkflows = item.metadata?.workflows;\n if (Array.isArray(itemWorkflows)) {\n for (const w of itemWorkflows) {\n if (typeof w === 'string')\n totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n }\n }\n }\n }\n\n // Fall back to config.workflow when the callback bypassed the runtime's\n // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n // metadata.workflows array non-empty for the common test-harness case.\n const workflowsMeta: Record<string, unknown> = {};\n if (totalWorkflowCalls.size > 0) {\n const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n workflowsMeta.workflows = sorted.map(([w]) => w);\n workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n } else if (config.workflow) {\n workflowsMeta.workflows = [config.workflow];\n workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n }\n\n return {\n id,\n dataset: config.dataset.name,\n metadata: {\n ...config.metadata,\n scorerTypes,\n ...modelsMeta,\n ...workflowsMeta,\n },\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: evalItems,\n summary: { count: items.length, failures, scorers: scorerStats, timing },\n };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n concurrency?: number;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n result: EvalResult,\n scorers: Scorer[],\n runtime: AxlRuntime,\n options?: RescoreOptions,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const concurrency = options?.concurrency ?? 5;\n\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const rescored: EvalItem[] = new Array(result.items.length);\n let totalCost = 0;\n\n async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n // Pass through error items without scoring\n if (original.error) {\n rescored[itemIndex] = {\n input: original.input,\n annotations: original.annotations,\n output: original.output,\n error: original.error,\n scores: {},\n };\n return;\n }\n\n const item: EvalItem = {\n input: original.input,\n annotations: original.annotations,\n output: original.output,\n metadata: original.metadata,\n scores: {},\n scoreDetails: {},\n };\n\n let itemScorerCost = 0;\n\n for (const scorer of scorers) {\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!item.scorerErrors) item.scorerErrors = [];\n item.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n item.scores[scorer.name] = null;\n item.scoreDetails![scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n item.scores[scorer.name] = round(scorerResult.score);\n item.scoreDetails![scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!item.scorerErrors) item.scorerErrors = [];\n item.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n item.scores[scorer.name] = null;\n item.scoreDetails![scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n rescored[itemIndex] = item;\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < result.items.length) {\n const currentIndex = index++;\n await rescoreItem(result.items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, result.items.length) }, () =>\n runNext(),\n );\n await Promise.all(workers);\n\n const failures = rescored.filter((i) => i.error).length;\n const scorerNames = scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = rescored\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const scorerTypes: Record<string, string> = {};\n for (const s of scorers) {\n scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n }\n\n return {\n id: randomUUID(),\n dataset: result.dataset,\n metadata: (() => {\n // Strip run group membership — rescored results are independent evaluations.\n // metadata.workflows is preserved via ...rest so the rescored result keeps\n // the same workflow attribution as the original.\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { runGroupId: _, runIndex: __, ...rest } = result.metadata;\n const merged: Record<string, unknown> = {\n ...rest,\n rescored: true,\n originalId: result.id,\n scorerTypes,\n };\n // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n // as a top-level string field with no `metadata.workflows`. Migrate it\n // forward so rescored results from old artifacts retain their workflow\n // attribution under the modern shape.\n if (!Array.isArray(merged.workflows)) {\n const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n merged.workflows = [legacyWorkflow];\n merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n }\n }\n return merged;\n })(),\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: rescored,\n summary: { count: result.items.length, failures, scorers: scorerStats },\n };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n runGroupId: string;\n runCount: number;\n /**\n * Workflow names observed across all runs in the group, deduped and ordered\n * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n * multi-run group produced by `axl-eval --runs N` typically has one entry;\n * heterogeneous groups (custom callbacks) can have multiple.\n */\n workflows: string[];\n dataset: string;\n totalCost: number;\n totalDuration: number;\n scorers: Record<string, { mean: number; std: number; min: number; max: number }>;\n timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n if (values.length <= 1) return 0;\n const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n const { dataset } = runs[0];\n\n // Union workflow names across all runs in the group, first-seen first.\n // Most groups have one, but custom callbacks can produce heterogeneous ones.\n const seenWorkflows = new Set<string>();\n const workflows: string[] = [];\n for (const run of runs) {\n const list = run.metadata.workflows;\n if (Array.isArray(list)) {\n for (const w of list) {\n if (typeof w === 'string' && !seenWorkflows.has(w)) {\n seenWorkflows.add(w);\n workflows.push(w);\n }\n }\n }\n }\n\n const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n // Collect per-scorer means from each run\n const scorerNames = Object.keys(runs[0].summary.scorers);\n const scorers: MultiRunSummary['scorers'] = {};\n\n for (const name of scorerNames) {\n const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);\n const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n const minMean = Math.min(...means);\n const maxMean = Math.max(...means);\n\n scorers[name] = {\n mean: round(meanOfMeans),\n std: round(std(means, meanOfMeans)),\n min: round(minMean),\n max: round(maxMean),\n };\n }\n\n // Timing aggregation\n let timing: MultiRunSummary['timing'];\n const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n if (timingMeans.length === runs.length && timingMeans.length > 0) {\n const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n timing = {\n mean: round(meanTiming),\n std: round(std(timingMeans, meanTiming)),\n };\n }\n\n return {\n runGroupId,\n runCount: runs.length,\n workflows,\n dataset,\n totalCost: round(totalCost),\n totalDuration: round(totalDuration),\n scorers,\n timing,\n };\n}\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> | undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n | null\n | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,kBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACef,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAO,MAAM,IAAI;AAAA,MACjB,OAAO,MAAM,IAAI;AAAA,MACjB,MAAM,MAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAO,MAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAM,MAAM,IAAI;AAAA,IAChB,aAAa,MAAM,YAAY,UAAU;AAAA,IACzC,cAAc,MAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;AC5FA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACnE,QAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAEvE,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAGA,QAAM,cAAc,aAAa,CAAC;AAClC,QAAM,eAAe,cAAc,CAAC;AAEpC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,QAAM,WAAW,KAAK,IAAI,aAAa,QAAQ,cAAc,MAAM;AACnE,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAEtE,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,IACF;AAGA,UAAM,QAAQ,YAAY,IAAI;AAC9B,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AACxB,YAAM,IAAI,MAAM;AAEhB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAE/J,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,YAAY,IAAI,UAAU,YAAY,SAAS;AAAA,IAC/D,WAAW,EAAE,IAAI,aAAa,IAAI,UAAU,aAAa,SAAS;AAAA,IAClE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9PO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;;;ACVO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEO,SAASA,OAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;;;ACdA,yBAA2B;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,SAAS,OAAO;AACzB,eAAS,OAAO,OAAO;AACvB,UAAI,OAAO,SAAU,UAAS,WAAW,OAAO;AAChD,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,SAAS,IAAI;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAW,UAAU,OAAO,SAAS;AACnC,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAM,OAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAC/B,mBAAS,aAAa,OAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACvD,mBAAS,aAAa,OAAO,IAAI,IAAI;AAAA,YACnC,OAAOA,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAC/B,iBAAS,aAAa,OAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAGA,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;ACjQA,IAAAC,sBAA2B;AAU3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAE5C,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAE/E,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAEA,QAAI,iBAAiB;AAErB,eAAW,UAAU,SAAS;AAC5B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,eAAK,aAAa;AAAA,YAChB,WAAW,OAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,eAAK,OAAO,OAAO,IAAI,IAAI;AAC3B,eAAK,aAAc,OAAO,IAAI,IAAI;AAAA,YAChC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,eAAK,OAAO,OAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACnD,eAAK,aAAc,OAAO,IAAI,IAAI;AAAA,YAChC,OAAOA,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,aAAK,aAAa;AAAA,UAChB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,aAAK,OAAO,OAAO,IAAI,IAAI;AAC3B,aAAK,aAAc,OAAO,IAAI,IAAI;AAAA,UAChC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,aAAS,SAAS,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,OAAO,MAAM,QAAQ;AAClC,YAAM,eAAe;AACrB,YAAM,YAAY,OAAO,MAAM,YAAY,GAAG,YAAY;AAAA,IAC5D;AAAA,EACF;AAEA,QAAM,UAAU,MAAM;AAAA,IAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,MAAM,EAAE;AAAA,IAAG,MACjF,QAAQ;AAAA,EACV;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,QAAI,gCAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAKf,YAAM,EAAE,YAAY,GAAG,UAAU,IAAI,GAAG,KAAK,IAAI,OAAO;AACxD,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;ACxLA,IAAAC,sBAA2B;AAoB3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,kBAAyB,gCAAW;AACzE,QAAM,EAAE,QAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AAChE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAEjC,YAAQ,IAAI,IAAI;AAAA,MACd,MAAMC,OAAM,WAAW;AAAA,MACvB,KAAKA,OAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAKA,OAAM,OAAO;AAAA,MAClB,KAAKA,OAAM,OAAO;AAAA,IACpB;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAMA,OAAM,UAAU;AAAA,MACtB,KAAKA,OAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA,WAAWA,OAAM,SAAS;AAAA,IAC1B,eAAeA,OAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;;;ACtFA,uBAAwB;AACxB,qBAA2B;AAC3B,sBAA8B;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,QAAI,0BAAQ,KAAK,IAAI;AAC3B,YAAI,2BAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,eAAW,+BAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,WAAO,+BAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AR/GA;AAoBA,IAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAaZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,SAAS,kBAAkB,MAA6D;AACtF,QAAM,MAAM,KAAK,QAAQ,aAAa;AACtC,MAAI,QAAQ,MAAM,MAAM,KAAK,KAAK,OAAQ,QAAO;AACjD,QAAM,QAAQ,KAAK,MAAM,CAAC;AAG1B,MAAI,WAAW,KAAK,KAAK,EAAG,QAAO,WAAW,KAAK;AAGnD,QAAM,MAA8B,CAAC;AACrC,aAAW,QAAQ,MAAM,MAAM,GAAG,GAAG;AACnC,UAAM,CAAC,KAAK,GAAG,IAAI,KAAK,MAAM,GAAG;AACjC,QAAI,OAAO,OAAO,KAAM,KAAI,IAAI,KAAK,CAAC,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EACjE;AACA,SAAO,OAAO,KAAK,GAAG,EAAE,SAAS,IAAI,MAAM;AAC7C;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,aAAa,kBAAkB,IAAI;AACzC,QAAM,QAAQ,KAAK;AAAA,IACjB,CAAC,GAAG,MAAM,CAAC,EAAE,WAAW,IAAI,KAAK,EAAE,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM;AAAA,EAC9D;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,gFAAgF;AAC9F,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,cAAc,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACrE,QAAM,eAAe,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACtE,QAAM,WAAsC;AAC5C,QAAM,YAAuC;AAC7C,QAAM,iBAAiB,cAAc,OAAO,EAAE,WAAW,IAAI;AAC7D,QAAM,aAAa,YAAY,UAAU,WAAW,cAAc;AAElE,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAClE,QAAM,QAAQ,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,MAAM,IAAI;AAEtE,QAAM,WAAW,QAAQ,4BAA4B;AACrD,QAAM,SAAS,QAAQ,4BAA4B;AACnD,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,0CAA0C,QAAQ,EAAE;AAChG,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,0CAA0C,MAAM,EAAE;AAE7F,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,QAAI,OAAO,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AACjO,QAAI,OAAO;AACT,UAAI,EAAE,IAAI;AACR,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,cAAc,QAAQ;AACpC,gBAAQ,MAAM,EAAE,KAAK,EAAE,IAAI,GAAG;AAAA,MAChC,OAAO;AACL,gBAAQ,WAAM,OAAO,EAAE;AAAA,MACzB;AAAA,IACF;AACA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAEA,MAAI,WAAW,QAAQ;AACrB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN;AAAA,sBAAyB,EAAE,eAAe,KAAM,QAAQ,CAAC,CAAC,mBAAmB,EAAE,gBAAgB,KAAM,QAAQ,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACvJ;AAAA,EACF;AACA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,qBAAqB,EAAE,cAAc,QAAQ,CAAC,CAAC,kBAAkB,EAAE,eAAe,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACnI;AAAA,EACF;AAEA,QAAM,cAAc,MAAM,QAAQ,QAAQ,IAAI,SAAS,CAAC,IAAI;AAC5D,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,YAAY,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EACrF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AAEzD,UAAM,kBAAkB,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,eAAe,IAAI;AACzF,QAAI,iBAAiB;AACnB,YAAM,2BAA2B,YAAY;AAAA,QAC3C,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,gBAAgB,QAAQ,WAAW,QAAQ,CAAC,EAAE,QAAQ;AAAA,MACrF;AACA,UAAI,yBAA0B,SAAQ,KAAK,CAAC;AAAA,IAC9C,OAAO;AACL,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AACF;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,SAAS,GAAG;AACpB,YAAQ,MAAM,sEAAsE;AACpF,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,CAAC,aAAa,YAAY,IAAI;AACpC,QAAM,MAAM,KAAK,MAAM,UAAM,gBAAAA,UAAc,aAAa,OAAO,CAAC;AAChE,QAAM,UAAwB,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,GAAG;AAE7D,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,MAAI;AACF,UAAM,MAAM,MAAM,aAAkB,aAAQ,YAAY,GAAG,YAAY,GAAG;AAC1E,UAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,QAAI,CAAC,WAAW,WAAW,WAAW,QAAQ,WAAW,GAAG;AAC1D,cAAQ,MAAM,UAAU,YAAY,0BAA0B;AAC9D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,UAAM,WAAyB,CAAC;AAChC,eAAW,cAAc,SAAS;AAChC,eAAS,KAAK,MAAM,QAAQ,YAAY,WAAW,SAAS,OAAO,CAAC;AAAA,IACtE;AAEA,eAAW,KAAK,UAAU;AACxB,cAAQ,IAAI,OAAO,YAAY,CAAC,IAAI,IAAI;AAAA,IAC1C;AAEA,QAAI,YAAY;AACd,YAAM,SAAS,MAAM,QAAQ,GAAG,IAAI,WAAW,SAAS,CAAC;AACzD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,6BAA6B,UAAU,EAAE;AAAA,IACvD;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,0BAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,6BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAGA,SAAS,gBAAgB,WAA4B;AACnD,MAAI,CAAC,MAAM,QAAQ,SAAS,KAAK,UAAU,WAAW,EAAG,QAAO;AAChE,SAAQ,UAAwB,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,IAAI;AAC7F;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM;AAAA,IACJ,SAAS,gBAAgB,OAAO,SAAS,SAAS,CAAC,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK;AAAA,EAClG;AACA,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AAErC,UAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAAE;AACvF,QAAI,oBAAoB,GAAG;AACzB,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,MAC1K;AAAA,IACF,OAAO;AACL,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,MACvO;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,QAAQ,QAAQ;AACzB,UAAM,IAAI,OAAO,QAAQ;AACzB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,MAAM,EAAE,OAAO,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACtL;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,QAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,MAAM;AACzE,MAAI,gBAAgB,SAAS,GAAG;AAC9B,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,gBAAgB,QAAQ,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC;AACjF,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,oBAAoB,gBAAgB,MAAM,IAAI,OAAO,QAAQ,KAAK;AAAA,IACpE;AACA,eAAW,OAAO,aAAa,MAAM,GAAG,CAAC,GAAG;AAC1C,YAAM,KAAK,SAAS,GAAG,EAAE;AAAA,IAC3B;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,KAAK,eAAe,aAAa,SAAS,CAAC,OAAO;AAAA,IAC1D;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,oBAAoB,SAAkC;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,QAAQ,OAAO;AAC/C,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM;AAAA,IACJ,SAAS,gBAAgB,QAAQ,SAAS,CAAC,MAAM,QAAQ,OAAO,WAAW,QAAQ,QAAQ;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,gBAAkB,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC;AAAA,EACvH;AACA,QAAM,UAAU,aAAa,IAAI,WAAW,IAAI,IAAI,IAAI;AACxD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,QAAQ,QAAQ,IAAI;AAC9B,UAAM,UAAU,GAAG,EAAE,KAAK,QAAQ,CAAC,CAAC,SAAW,EAAE,IAAI,QAAQ,CAAC,CAAC;AAC/D,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,QAAQ,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,IAC/H;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ;AAClB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,OAAO,QAAQ,OAAO,OAAO,KAAM,QAAQ,CAAC,IAAI,YAAc,QAAQ,OAAO,MAAM,KAAM,QAAQ,CAAC,IAAI,KAAK,SAAS,QAAQ,CAAC;AAAA,IAC/J;AAAA,EACF;AAEA,QAAM,UAAU,QAAQ,YAAY,IAAI,IAAI,QAAQ,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC7E,QAAM,eAAe,QAAQ,gBAAgB,KAAM,QAAQ,CAAC;AAC5D,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,iBAAiB,OAAO,sBAAsB,WAAW,GAAG;AAEvE,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,WAAO,0BAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAMrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,MAAI,OAAO;AACX,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,kBAAkB,QAAQ,UAAU;AAC1F,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA,eAChC,QAAQ,SAAU,QAAO,KAAK,IAAI,GAAG,SAAS,OAAO,EAAE,KAAK,CAAC;AAAA;AAEpE,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM,MAAM;AAC1D;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,MAAM,IAAI,cAAc,IAAI;AAE7E,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM;AAAA,cACJ;AAAA,cACA,MAAM;AAAA,cACN;AAAA,YACF,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC3C,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO;AAAA,cACL,QAAQ,OAAO;AAAA,cACf,MAAM,OAAO,QAAQ;AAAA,cACrB,UAAU,OAAO,YAAY;AAAA,YAC/B;AAAA,UACF;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAQ,MAAM,SAAS,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC1E,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQ,QAAQ,MAAM,SAAS;AAAA,UAC1C;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,YAAI,OAAO,GAAG;AAEZ,gBAAM,EAAE,YAAAC,YAAW,IAAI,MAAM,OAAO,QAAa;AACjD,gBAAM,aAAaA,YAAW;AAC9B,gBAAM,aAA2B,CAAC;AAElC,mBAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,oBAAQ,MAAM,kBAAkB,IAAI,CAAC,IAAI,IAAI,KAAK;AAClD,kBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,OAAO;AACjE,mBAAO,SAAS,aAAa;AAC7B,mBAAO,SAAS,WAAW;AAC3B,uBAAW,KAAK,MAAM;AACtB,oBAAQ,KAAK,MAAM;AAAA,UACrB;AAEA,gBAAM,UAAU,cAAc,UAAU;AACxC,kBAAQ,IAAI,OAAO,oBAAoB,OAAO,IAAI,IAAI;AAAA,QACxD,OAAO;AACL,gBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,OAAO;AACjE,kBAAQ,KAAK,MAAM;AAEnB,kBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,QAC/C;AAAA,MACF,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAD,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["import_node_fs","round","round","round","import_node_crypto","round","import_node_crypto","round","readFileAsync","writeFileAsync","randomUUID"]}
1
+ {"version":3,"sources":["../src/cli.ts","../src/bootstrap.ts","../src/compare.ts","../src/scorer.ts","../src/utils.ts","../src/runner.ts","../src/rescore.ts","../src/multi-run.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport { rescore } from './rescore.js';\nimport { aggregateRuns } from './multi-run.js';\nimport type { MultiRunSummary } from './multi-run.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set([\n '--output',\n '--config',\n '--conditions',\n '--fail-on-regression',\n '--threshold',\n '--runs',\n '--capture-traces',\n]);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --runs <n> Run eval N times (multi-run)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval <path> --capture-traces Populate EvalItem.traces on every item\n (success + failure). Adds memory\n overhead proportional to dataset size\n x turns x agents; off by default.\n axl-eval rescore <results> <eval-file> Re-run scorers on saved outputs\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --threshold <v> Set regression threshold (global or per-scorer)\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n if (args[0] === 'rescore') {\n await runRescore(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nfunction parseThresholdArg(args: string[]): Record<string, number> | number | undefined {\n const idx = args.indexOf('--threshold');\n if (idx === -1 || idx + 1 >= args.length) return undefined;\n const value = args[idx + 1];\n\n // Global numeric threshold: --threshold 0.05\n if (/^[\\d.]+$/.test(value)) return parseFloat(value);\n\n // Per-scorer map: --threshold accuracy=0,tone=0.1\n const map: Record<string, number> = {};\n for (const pair of value.split(',')) {\n const [key, val] = pair.split('=');\n if (key && val != null) map[key.trim()] = parseFloat(val.trim());\n }\n return Object.keys(map).length > 0 ? map : undefined;\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const thresholds = parseThresholdArg(args);\n const files = args.filter(\n (a, i) => !a.startsWith('--') && !(i > 0 && args[i - 1] === '--threshold'),\n );\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]');\n process.exit(1);\n }\n\n const baselineRaw = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidateRaw = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const baseline: EvalResult | EvalResult[] = baselineRaw;\n const candidate: EvalResult | EvalResult[] = candidateRaw;\n const compareOptions = thresholds != null ? { thresholds } : undefined;\n const comparison = evalCompare(baseline, candidate, compareOptions);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n const hasCI = scorerNames.some((n) => comparison.scorers[n].ci != null);\n\n const ciHeader = hasCI ? ' CI 95% Sig' : '';\n const ciRule = hasCI ? ' ---------------- ---' : '';\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change${ciHeader}`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------${ciRule}`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n let line = ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`;\n if (hasCI) {\n if (s.ci) {\n const lo = (s.ci.lower >= 0 ? '+' : '') + s.ci.lower.toFixed(4);\n const hi = (s.ci.upper >= 0 ? '+' : '') + s.ci.upper.toFixed(4);\n const sig = s.significant ? ' *' : ' ';\n line += ` [${lo}, ${hi}]${sig}`;\n } else {\n line += ' —'.padEnd(22);\n }\n }\n console.log(line);\n }\n\n if (comparison.timing) {\n const t = comparison.timing;\n const sign = t.delta > 0 ? '+' : '';\n console.log(\n `\\n Timing: baseline ${(t.baselineMean / 1000).toFixed(2)}s -> candidate ${(t.candidateMean / 1000).toFixed(2)}s (${sign}${t.deltaPercent.toFixed(1)}%)`,\n );\n }\n if (comparison.cost) {\n const c = comparison.cost;\n const sign = c.delta > 0 ? '+' : '';\n console.log(\n ` Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`,\n );\n }\n\n const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;\n const stable = Math.max(\n 0,\n baselineRef.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n // When CI is available, only fail on significant regressions\n const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);\n if (hasSignificance) {\n const hasSignificantRegression = scorerNames.some(\n (n) => comparison.scorers[n].significant === true && comparison.scorers[n].delta < 0,\n );\n if (hasSignificantRegression) process.exit(1);\n } else {\n process.exit(1);\n }\n }\n}\n\nasync function runRescore(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length < 2) {\n console.error('Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]');\n process.exit(1);\n }\n\n const [resultsPath, evalFilePath] = paths;\n const raw = JSON.parse(await readFileAsync(resultsPath, 'utf-8'));\n const results: EvalResult[] = Array.isArray(raw) ? raw : [raw];\n\n const runtime = await getRuntime(configArg, conditions);\n try {\n const mod = await importModule(path.resolve(evalFilePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.scorers || evalConfig.scorers.length === 0) {\n console.error(`Error: ${evalFilePath} does not export scorers`);\n process.exit(1);\n }\n\n const rescored: EvalResult[] = [];\n for (const resultData of results) {\n rescored.push(await rescore(resultData, evalConfig.scorers, runtime));\n }\n\n for (const r of rescored) {\n console.log('\\n' + formatTable(r) + '\\n');\n }\n\n if (outputPath) {\n const output = Array.isArray(raw) ? rescored : rescored[0];\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Rescored results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\n/** Render observed workflows for a terse header line. Multiple → comma-separated. */\nfunction formatWorkflows(workflows: unknown): string {\n if (!Array.isArray(workflows) || workflows.length === 0) return '(unknown)';\n return (workflows as unknown[]).filter((w): w is string => typeof w === 'string').join(', ');\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(\n `Eval: ${formatWorkflows(result.metadata.workflows)} x ${result.dataset} (${result.summary.count} items)`,\n );\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n // Detect scorers with no valid scores (all items errored → computeStats([]) → all zeros)\n const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;\n if (validScoreCount === 0) {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)}`,\n );\n } else {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n }\n\n if (result.summary.timing) {\n const t = result.summary.timing;\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${(t.mean / 1000).toFixed(2).padStart(colWidth)}s ${(t.p50 / 1000).toFixed(2).padStart(colWidth)}s ${(t.p95 / 1000).toFixed(2).padStart(colWidth)}s`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n );\n\n const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);\n if (itemsWithErrors.length > 0) {\n const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors!))];\n lines.push('');\n lines.push(\n ` Scorer errors (${itemsWithErrors.length}/${result.summary.count} items affected):`,\n );\n for (const err of uniqueErrors.slice(0, 5)) {\n lines.push(` - ${err}`);\n }\n if (uniqueErrors.length > 5) {\n lines.push(` ... and ${uniqueErrors.length - 5} more`);\n }\n }\n\n return lines.join('\\n');\n}\n\nfunction formatMultiRunTable(summary: MultiRunSummary): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 16;\n\n lines.push(\n `Eval: ${formatWorkflows(summary.workflows)} x ${summary.dataset} \\u2014 ${summary.runCount} runs`,\n );\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean \\u00b1 Std'.padStart(colWidth)} ${'Min'.padStart(8)} ${'Max'.padStart(8)}`,\n );\n const ruleLen = maxNameLen + 2 + colWidth + 2 + 8 + 2 + 8;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = summary.scorers[name];\n const meanStd = `${s.mean.toFixed(3)} \\u00b1 ${s.std.toFixed(3)}`;\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${meanStd.padStart(colWidth)} ${s.min.toFixed(3).padStart(8)} ${s.max.toFixed(3).padStart(8)}`,\n );\n }\n\n if (summary.timing) {\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${((summary.timing.mean / 1000).toFixed(2) + ' \\u00b1 ' + (summary.timing.std / 1000).toFixed(2) + 's').padStart(colWidth)}`,\n );\n }\n\n const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : '$0.00';\n const durationStr = (summary.totalDuration / 1000).toFixed(1);\n lines.push('');\n lines.push(` Total Cost: ${costStr} | Total Duration: ${durationStr}s`);\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime || typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' || ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n runs: number;\n captureTraces: boolean;\n paths: string[];\n} {\n let outputPath: string | undefined;\n let configArg: string | undefined;\n let conditions: string[] = [];\n let runs = 1;\n let captureTraces = false;\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' || arg === '--config' || arg === '--conditions' || arg === '--runs') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else if (arg === '--runs') runs = Math.max(1, parseInt(value, 10) || 1);\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg === '--capture-traces') {\n // Boolean flag — no value consumed.\n captureTraces = true;\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, runs, captureTraces, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackExecution for cost + metadata attribution\n executeWorkflow = async (input, rt) => {\n const {\n result,\n cost: trackedCost,\n metadata,\n } = await runtime.trackExecution(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return {\n output: result.output,\n cost: result.cost ?? trackedCost,\n metadata: result.metadata ?? metadata,\n };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost, metadata } = await runtime.trackExecution(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost, metadata };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n // When --capture-traces is set, forward it to runEval so the runner\n // wraps each item's execution in a nested trackExecution({ captureTraces: true }).\n // This populates EvalItem.traces on both success (via tracked.traces)\n // and failure (via the axlCapturedTraces side-channel on the thrown\n // error). Nested trackExecution scopes walk the AsyncLocalStorage\n // parent chain, so the outer trackExecution above still observes\n // cost/metadata correctly.\n const runOptions = captureTraces ? { captureTraces: true } : undefined;\n\n if (runs > 1) {\n // Multi-run mode\n const { randomUUID } = await import('node:crypto');\n const runGroupId = randomUUID();\n const runResults: EvalResult[] = [];\n\n for (let r = 0; r < runs; r++) {\n console.error(`[axl-eval] Run ${r + 1}/${runs}...`);\n const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);\n result.metadata.runGroupId = runGroupId;\n result.metadata.runIndex = r;\n runResults.push(result);\n results.push(result);\n }\n\n const summary = aggregateRuns(runResults);\n console.log('\\n' + formatMultiRunTable(summary) + '\\n');\n } else {\n const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n }\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","export type BootstrapCIResult = {\n /** Lower bound of the confidence interval. */\n lower: number;\n /** Upper bound of the confidence interval. */\n upper: number;\n /** Mean of the original differences. */\n mean: number;\n /** Proportion of resamples where the mean difference is negative (probability of regression). */\n pRegression: number;\n /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n differences: number[],\n options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n const n = differences.length;\n if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n const nResamples = options?.nResamples ?? 1000;\n const alpha = options?.alpha ?? 0.05;\n\n const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n if (n === 1)\n return {\n lower: differences[0],\n upper: differences[0],\n mean,\n pRegression: mean < 0 ? 1 : 0,\n pImprovement: mean > 0 ? 1 : 0,\n };\n\n if (nResamples <= 0)\n return {\n lower: round(mean),\n upper: round(mean),\n mean: round(mean),\n pRegression: mean < 0 ? 1 : 0,\n pImprovement: mean > 0 ? 1 : 0,\n };\n\n // Seeded or random number generator\n const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n const resampleMeans: number[] = new Array(nResamples);\n for (let r = 0; r < nResamples; r++) {\n let sum = 0;\n for (let i = 0; i < n; i++) {\n sum += differences[Math.floor(rng() * n)];\n }\n resampleMeans[r] = sum / n;\n }\n\n resampleMeans.sort((a, b) => a - b);\n\n const lowerIdx = Math.floor((alpha / 2) * nResamples);\n const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n // Count resamples below/above zero for directional probabilities\n // resampleMeans is already sorted, so binary search for zero\n let belowZero = 0;\n for (let i = 0; i < nResamples; i++) {\n if (resampleMeans[i] < 0) belowZero++;\n else break;\n }\n let aboveZero = 0;\n for (let i = nResamples - 1; i >= 0; i--) {\n if (resampleMeans[i] > 0) aboveZero++;\n else break;\n }\n\n return {\n lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n mean: round(mean),\n pRegression: round(belowZero / nResamples),\n pImprovement: round(aboveZero / nResamples),\n };\n}\n\nfunction round(n: number): number {\n return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n let state = seed | 0 || 1; // Ensure non-zero\n return () => {\n state ^= state << 13;\n state ^= state >> 17;\n state ^= state << 5;\n return (state >>> 0) / 0x100000000;\n };\n}\n","import type {\n EvalResult,\n EvalComparison,\n EvalCompareOptions,\n EvalRegression,\n EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\n\nfunction round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n scorerName: string,\n options: EvalCompareOptions | undefined,\n metadata: Record<string, unknown>,\n): number {\n // Explicit global threshold\n if (typeof options?.thresholds === 'number') return options.thresholds;\n\n // Per-scorer threshold map\n if (options?.thresholds && typeof options.thresholds === 'object') {\n const perScorer = options.thresholds[scorerName];\n if (perScorer != null) return perScorer;\n // Fall through to auto-calibration if scorer not in map\n }\n\n // Auto-calibrate from scorerTypes metadata\n const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n if (scorerTypes && scorerName in scorerTypes) {\n return scorerTypes[scorerName] === 'llm'\n ? DEFAULT_LLM_THRESHOLD\n : DEFAULT_DETERMINISTIC_THRESHOLD;\n }\n\n // Legacy fallback for results without scorerTypes\n return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n baseline: EvalResult | EvalResult[],\n candidate: EvalResult | EvalResult[],\n options?: EvalCompareOptions,\n): EvalComparison {\n const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];\n const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];\n\n if (baselineRuns.length === 0 || candidateRuns.length === 0) {\n throw new Error('Cannot compare empty eval result arrays');\n }\n\n // Use the first run as representative for metadata, dataset, scorers\n const baselineRef = baselineRuns[0];\n const candidateRef = candidateRuns[0];\n\n if (baselineRef.dataset !== candidateRef.dataset) {\n throw new Error(\n `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n );\n }\n\n const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n throw new Error(`Cannot compare evals with different scorers`);\n }\n\n // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n const pairedDiffs: Record<string, number[]> = {};\n for (const name of baselineScorerNames) {\n pairedDiffs[name] = [];\n }\n const runCount = Math.min(baselineRuns.length, candidateRuns.length);\n for (let r = 0; r < runCount; r++) {\n const bRun = baselineRuns[r];\n const cRun = candidateRuns[r];\n const minLength = Math.min(bRun.items.length, cRun.items.length);\n for (let i = 0; i < minLength; i++) {\n const bItem = bRun.items[i];\n const cItem = cRun.items[i];\n if (bItem.error || cItem.error) continue;\n for (const name of baselineScorerNames) {\n const bScore = bItem.scores[name];\n const cScore = cItem.scores[name];\n if (bScore != null && cScore != null) {\n pairedDiffs[name].push(cScore - bScore);\n }\n }\n }\n }\n\n // Compute aggregate means across runs\n const baselineMeans: Record<string, number> = {};\n const candidateMeans: Record<string, number> = {};\n for (const name of baselineScorerNames) {\n baselineMeans[name] =\n baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n baselineRuns.length;\n candidateMeans[name] =\n candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n candidateRuns.length;\n }\n\n const scorers: EvalComparison['scorers'] = {};\n for (const name of baselineScorerNames) {\n const bMeanRaw = baselineMeans[name];\n const cMeanRaw = candidateMeans[name];\n const deltaRaw = cMeanRaw - bMeanRaw;\n const bMean = round(bMeanRaw);\n const cMean = round(cMeanRaw);\n const delta = round(deltaRaw);\n const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n const entry: EvalComparison['scorers'][string] = {\n baselineMean: bMean,\n candidateMean: cMean,\n delta,\n deltaPercent,\n };\n\n // Compute bootstrap CI when we have enough paired data\n const diffs = pairedDiffs[name];\n if (diffs.length >= 2) {\n const ci = pairedBootstrapCI(diffs);\n entry.ci = { lower: ci.lower, upper: ci.upper };\n entry.pRegression = ci.pRegression;\n entry.pImprovement = ci.pImprovement;\n entry.n = diffs.length;\n // Significant when CI excludes zero AND delta exceeds practical threshold\n const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n }\n\n scorers[name] = entry;\n }\n\n // Per-item regressions/improvements.\n // For multi-run: average each item's score across runs to get a stable per-item comparison.\n const regressions: EvalRegression[] = [];\n const improvements: EvalImprovement[] = [];\n\n const itemCount = Math.min(\n ...baselineRuns.map((r) => r.items.length),\n ...candidateRuns.map((r) => r.items.length),\n );\n\n for (let i = 0; i < itemCount; i++) {\n // Check if any run has an error for this item\n const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n if (hasBaselineError || hasCandidateError) continue;\n\n for (const name of baselineScorerNames) {\n // Average this item's score across all runs on each side\n const bScores = baselineRuns\n .map((r) => r.items[i]?.scores[name])\n .filter((s): s is number => s != null);\n const cScores = candidateRuns\n .map((r) => r.items[i]?.scores[name])\n .filter((s): s is number => s != null);\n if (bScores.length === 0 || cScores.length === 0) continue;\n\n const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n const delta = round(cAvg - bAvg);\n const threshold = resolveThreshold(name, options, baselineRef.metadata);\n if (delta < -threshold)\n regressions.push({\n itemIndex: i,\n input: baselineRef.items[i]?.input,\n scorer: name,\n baselineScore: round(bAvg),\n candidateScore: round(cAvg),\n delta,\n });\n else if (delta > threshold)\n improvements.push({\n itemIndex: i,\n input: baselineRef.items[i]?.input,\n scorer: name,\n baselineScore: round(bAvg),\n candidateScore: round(cAvg),\n delta,\n });\n }\n }\n\n // Timing comparison (average across all runs for multi-run)\n let timing: EvalComparison['timing'];\n const baselineTimings = baselineRuns\n .filter((r) => r.summary.timing)\n .map((r) => r.summary.timing!.mean);\n const candidateTimings = candidateRuns\n .filter((r) => r.summary.timing)\n .map((r) => r.summary.timing!.mean);\n if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n }\n\n // Cost comparison (per-run average for multi-run)\n let cost: EvalComparison['cost'];\n const baselineAvgCost =\n baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n const candidateAvgCost =\n candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n const deltaRaw = candidateAvgCost - baselineAvgCost;\n const delta = round(deltaRaw);\n const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n cost = {\n baselineTotal: round(baselineAvgCost),\n candidateTotal: round(candidateAvgCost),\n delta,\n deltaPercent,\n };\n }\n\n const parts: string[] = [];\n for (const name of baselineScorerNames) {\n const s = scorers[name];\n if (Math.abs(s.delta) > 0.001) {\n const direction = s.delta > 0 ? 'improves' : 'regresses';\n const sign = s.delta > 0 ? '+' : '';\n let sigLabel = '';\n if (s.significant === true) sigLabel = ' (significant)';\n else if (s.significant === false) sigLabel = ' (not significant)';\n parts.push(\n `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n );\n }\n }\n if (timing && Math.abs(timing.deltaPercent) > 1) {\n const dir = timing.delta > 0 ? 'slower' : 'faster';\n parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n }\n if (cost && Math.abs(cost.deltaPercent) > 1) {\n const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n }\n const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n return {\n baseline: { id: baselineRef.id, metadata: baselineRef.metadata },\n candidate: { id: candidateRef.id, metadata: candidateRef.metadata },\n scorers,\n timing,\n cost,\n regressions,\n improvements,\n summary: summaryStr,\n };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n score: number;\n metadata?: Record<string, unknown>;\n cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n /** Resolve a provider:model URI to a provider instance and model name. */\n resolveProvider: (modelUri: string) => {\n provider: {\n chat(\n messages: { role: string; content: string }[],\n options: {\n model: string;\n temperature?: number;\n responseFormat?: { type: string; json_schema?: unknown };\n },\n ): Promise<{ content: string; cost?: number }>;\n };\n model: string;\n };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n name: string;\n description: string;\n score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly description: string;\n readonly isLlm: boolean;\n score(\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n context?: ScorerContext,\n ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n return {\n name: config.name,\n description: config.description,\n isLlm: false,\n score: config.score,\n };\n}\n","export function computeStats(scores: number[]): {\n mean: number;\n min: number;\n max: number;\n p50: number;\n p95: number;\n} {\n if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n const sorted = [...scores].sort((a, b) => a - b);\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n const min = sorted[0];\n const max = sorted[sorted.length - 1];\n const p50 = sorted[Math.floor(sorted.length * 0.5)];\n const p95 = sorted[Math.floor(sorted.length * 0.95)];\n return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary, RunEvalOptions } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n const match = cost.match(/^\\$?([\\d.]+)$/);\n if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n return parseFloat(match[1]);\n}\n\n/**\n * Extract a user-returned cost only if it's a non-negative finite number.\n * Guards against workflows that return `{ cost: 'free' }`, `{ cost: NaN }`,\n * `{ cost: -1 }`, `{ cost: Infinity }`, etc. — the TS type says `cost?: number`\n * but at runtime we can't trust that.\n *\n * Negative costs are rejected because (a) cost is a USD amount and negative\n * values are nonsensical, and (b) a negative cost would silently shrink\n * `totalCost` below the budget limit check at the processItem level,\n * letting a buggy/malicious workflow run unbounded. `0` is preserved\n * because free operations on paid models are valid.\n *\n * When a user-supplied value is rejected (present but invalid), we log\n * once per item via `console.warn` so the type violation is visible\n * rather than silently swallowed.\n */\nfunction extractUserCost(result: unknown, label = 'executeWorkflow'): number | undefined {\n if (result === null || typeof result !== 'object') return undefined;\n const raw = (result as { cost?: unknown }).cost;\n if (raw === undefined) return undefined;\n if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) return raw;\n\n console.warn(\n `[axl-eval] Ignoring invalid \\`cost\\` from ${label} return: expected non-negative finite number, got ${typeof raw === 'number' ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`,\n );\n return undefined;\n}\n\n/**\n * Extract a user-returned metadata record only if it's a plain object.\n * Rejects arrays, null, scalars, and exotic object types (Date, Map, Set,\n * Error, class instances) that would satisfy a loose `typeof === 'object'`\n * check but break `Record<string, unknown>` assumptions in downstream\n * consumers (spread, Object.entries, property access).\n */\nfunction extractUserMetadata(\n result: unknown,\n label = 'executeWorkflow',\n): Record<string, unknown> | undefined {\n if (result === null || typeof result !== 'object') return undefined;\n const meta = (result as { metadata?: unknown }).metadata;\n if (meta === undefined) return undefined;\n if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) {\n console.warn(\n `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${meta === null ? 'null' : Array.isArray(meta) ? 'array' : typeof meta}.`,\n );\n return undefined;\n }\n // Reject exotic objects (Date, Map, Set, Error, class instances) whose\n // prototype chain differs from Object.prototype. Those pass `typeof ===\n // 'object'` but don't behave like `Record<string, unknown>`.\n const proto = Object.getPrototypeOf(meta);\n if (proto !== Object.prototype && proto !== null) {\n console.warn(\n `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${(proto?.constructor?.name as string | undefined) ?? 'exotic object'}.`,\n );\n return undefined;\n }\n return meta as Record<string, unknown>;\n}\n\nexport async function runEval(\n config: EvalConfig,\n executeWorkflow: (\n input: unknown,\n runtime: AxlRuntime,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n runtime: AxlRuntime,\n options?: RunEvalOptions,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const id = randomUUID();\n const items = await config.dataset.getItems();\n const concurrency = config.concurrency ?? 5;\n const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n // Create a scorer context that LLM scorers use to resolve providers.\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const evalItems: EvalItem[] = new Array(items.length);\n let totalCost = 0;\n let budgetExceeded = false;\n\n async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n if (options?.signal?.aborted) {\n evalItems[itemIndex] = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Cancelled',\n scores: {},\n };\n options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n return;\n }\n\n if (budgetExceeded) {\n evalItems[itemIndex] = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Budget exceeded',\n scores: {},\n };\n options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n return;\n }\n\n const evalItem: EvalItem = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n scores: {},\n };\n const itemStart = Date.now();\n try {\n // When captureTraces is on, wrap the user's executeWorkflow in\n // trackExecution so we get a per-item TraceEvent[] scoped via\n // AsyncLocalStorage. trackExecution's scope walks the parent chain, so\n // any nested trackExecution (e.g. from the CLI) continues to work.\n if (options?.captureTraces) {\n const tracked = await runtime.trackExecution(\n async () => executeWorkflow(item.input, runtime),\n { captureTraces: true },\n );\n evalItem.duration = Date.now() - itemStart;\n evalItem.output = tracked.result.output;\n // Prefer user-returned cost/metadata; fall back to tracked values.\n // Type-guarded so a non-number cost (e.g. `{ cost: 'free' }`) or a\n // non-object metadata (e.g. an array or scalar) doesn't silently\n // corrupt downstream math or shape expectations.\n evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;\n evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;\n if (tracked.traces && tracked.traces.length > 0) {\n evalItem.traces = tracked.traces;\n }\n if (evalItem.cost != null) {\n totalCost += evalItem.cost;\n }\n } else {\n const result = await executeWorkflow(item.input, runtime);\n evalItem.duration = Date.now() - itemStart;\n evalItem.output = result.output;\n evalItem.cost = extractUserCost(result);\n const meta = extractUserMetadata(result);\n if (meta) evalItem.metadata = meta;\n if (evalItem.cost != null) {\n totalCost += evalItem.cost;\n }\n }\n } catch (err) {\n evalItem.duration = Date.now() - itemStart;\n evalItem.error = err instanceof Error ? err.message : String(err);\n // Failed items are exactly when per-item traces are most valuable —\n // recover them from the captured-traces side-channel `trackExecution`\n // attaches to the error on the failure path.\n if (options?.captureTraces) {\n const captured = (err as { axlCapturedTraces?: unknown }).axlCapturedTraces;\n if (Array.isArray(captured) && captured.length > 0) {\n evalItem.traces = captured as EvalItem['traces'];\n }\n }\n evalItems[itemIndex] = evalItem;\n options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n return;\n }\n\n if (budgetLimit != null && totalCost > budgetLimit) {\n budgetExceeded = true;\n }\n\n evalItem.scoreDetails = {};\n let itemScorerCost = 0;\n\n for (const scorer of config.scorers) {\n if (options?.signal?.aborted) break;\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(\n evalItem.output,\n item.input,\n item.annotations,\n scorerContext,\n );\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n evalItem.scores[scorer.name] = round(scorerResult.score);\n evalItem.scoreDetails[scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n // Capture cost from error (LLM scorer attaches it)\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n evalItems[itemIndex] = evalItem;\n options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < items.length) {\n const currentIndex = index++;\n await processItem(items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n await Promise.all(workers);\n\n const failures = evalItems.filter((i) => i.error).length;\n const scorerNames = config.scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = evalItems\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n const scorerTypes: Record<string, string> = {};\n for (const s of config.scorers) {\n scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n }\n\n // Aggregate per-model LLM call counts across all items\n const totalModelCalls = new Map<string, number>();\n for (const item of evalItems) {\n const itemCounts = item.metadata?.modelCallCounts;\n if (itemCounts && typeof itemCounts === 'object') {\n for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n if (typeof count === 'number')\n totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n }\n } else {\n // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n const itemModels = item.metadata?.models;\n if (Array.isArray(itemModels)) {\n for (const m of itemModels) {\n if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n }\n }\n }\n }\n\n // models: unique list sorted by total calls (most-called first)\n // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n const modelsMeta: Record<string, unknown> = {};\n if (totalModelCalls.size > 0) {\n const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n modelsMeta.models = sorted.map(([m]) => m);\n modelsMeta.modelCounts = Object.fromEntries(sorted);\n }\n\n // Aggregate per-workflow call counts across all items (parallel to models).\n // Workflows come from trackExecution's trace-event collection — callers\n // don't specify workflow names anywhere, they just appear because the\n // runtime emits workflow_start events for every execute() call.\n const totalWorkflowCalls = new Map<string, number>();\n for (const item of evalItems) {\n const itemCounts = item.metadata?.workflowCallCounts;\n if (itemCounts && typeof itemCounts === 'object') {\n for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n if (typeof count === 'number')\n totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n }\n } else {\n // Fallback: count unique workflows per item\n const itemWorkflows = item.metadata?.workflows;\n if (Array.isArray(itemWorkflows)) {\n for (const w of itemWorkflows) {\n if (typeof w === 'string')\n totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n }\n }\n }\n }\n\n // Fall back to config.workflow when the callback bypassed the runtime's\n // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n // metadata.workflows array non-empty for the common test-harness case.\n const workflowsMeta: Record<string, unknown> = {};\n if (totalWorkflowCalls.size > 0) {\n const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n workflowsMeta.workflows = sorted.map(([w]) => w);\n workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n } else if (config.workflow) {\n workflowsMeta.workflows = [config.workflow];\n workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n }\n\n options?.onProgress?.({ type: 'run_done', totalItems: items.length, failures });\n\n return {\n id,\n dataset: config.dataset.name,\n metadata: {\n ...config.metadata,\n scorerTypes,\n ...modelsMeta,\n ...workflowsMeta,\n },\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: evalItems,\n summary: { count: items.length, failures, scorers: scorerStats, timing },\n };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n concurrency?: number;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n result: EvalResult,\n scorers: Scorer[],\n runtime: AxlRuntime,\n options?: RescoreOptions,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const concurrency = options?.concurrency ?? 5;\n\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const rescored: EvalItem[] = new Array(result.items.length);\n let totalCost = 0;\n\n async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n // Pass through error items without scoring\n if (original.error) {\n rescored[itemIndex] = {\n input: original.input,\n annotations: original.annotations,\n output: original.output,\n error: original.error,\n scores: {},\n };\n return;\n }\n\n // Preserve per-item traces from the original run: rescore re-runs scorers\n // but leaves the workflow untouched, so the original execution traces\n // remain accurate and useful for diagnosing score changes.\n const item: EvalItem = {\n input: original.input,\n annotations: original.annotations,\n output: original.output,\n metadata: original.metadata,\n traces: original.traces,\n scores: {},\n scoreDetails: {},\n };\n\n let itemScorerCost = 0;\n\n for (const scorer of scorers) {\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!item.scorerErrors) item.scorerErrors = [];\n item.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n item.scores[scorer.name] = null;\n item.scoreDetails![scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n item.scores[scorer.name] = round(scorerResult.score);\n item.scoreDetails![scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!item.scorerErrors) item.scorerErrors = [];\n item.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n item.scores[scorer.name] = null;\n item.scoreDetails![scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n rescored[itemIndex] = item;\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < result.items.length) {\n const currentIndex = index++;\n await rescoreItem(result.items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, result.items.length) }, () =>\n runNext(),\n );\n await Promise.all(workers);\n\n const failures = rescored.filter((i) => i.error).length;\n const scorerNames = scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = rescored\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const scorerTypes: Record<string, string> = {};\n for (const s of scorers) {\n scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n }\n\n return {\n id: randomUUID(),\n dataset: result.dataset,\n metadata: (() => {\n // Strip run group membership — rescored results are independent evaluations.\n // metadata.workflows is preserved via ...rest so the rescored result keeps\n // the same workflow attribution as the original.\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { runGroupId: _, runIndex: __, ...rest } = result.metadata;\n const merged: Record<string, unknown> = {\n ...rest,\n rescored: true,\n originalId: result.id,\n scorerTypes,\n };\n // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n // as a top-level string field with no `metadata.workflows`. Migrate it\n // forward so rescored results from old artifacts retain their workflow\n // attribution under the modern shape.\n if (!Array.isArray(merged.workflows)) {\n const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n merged.workflows = [legacyWorkflow];\n merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n }\n }\n return merged;\n })(),\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: rescored,\n summary: { count: result.items.length, failures, scorers: scorerStats },\n };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n runGroupId: string;\n runCount: number;\n /**\n * Workflow names observed across all runs in the group, deduped and ordered\n * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n * multi-run group produced by `axl-eval --runs N` typically has one entry;\n * heterogeneous groups (custom callbacks) can have multiple.\n */\n workflows: string[];\n dataset: string;\n totalCost: number;\n totalDuration: number;\n scorers: Record<string, { mean: number; std: number; min: number; max: number }>;\n timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n if (values.length <= 1) return 0;\n const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n const { dataset } = runs[0];\n\n // Union workflow names across all runs in the group, first-seen first.\n // Most groups have one, but custom callbacks can produce heterogeneous ones.\n const seenWorkflows = new Set<string>();\n const workflows: string[] = [];\n for (const run of runs) {\n const list = run.metadata.workflows;\n if (Array.isArray(list)) {\n for (const w of list) {\n if (typeof w === 'string' && !seenWorkflows.has(w)) {\n seenWorkflows.add(w);\n workflows.push(w);\n }\n }\n }\n }\n\n const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n // Collect per-scorer means from each run\n const scorerNames = Object.keys(runs[0].summary.scorers);\n const scorers: MultiRunSummary['scorers'] = {};\n\n for (const name of scorerNames) {\n const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);\n const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n const minMean = Math.min(...means);\n const maxMean = Math.max(...means);\n\n scorers[name] = {\n mean: round(meanOfMeans),\n std: round(std(means, meanOfMeans)),\n min: round(minMean),\n max: round(maxMean),\n };\n }\n\n // Timing aggregation\n let timing: MultiRunSummary['timing'];\n const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n if (timingMeans.length === runs.length && timingMeans.length > 0) {\n const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n timing = {\n mean: round(meanTiming),\n std: round(std(timingMeans, meanTiming)),\n };\n }\n\n return {\n runGroupId,\n runCount: runs.length,\n workflows,\n dataset,\n totalCost: round(totalCost),\n totalDuration: round(totalDuration),\n scorers,\n timing,\n };\n}\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> | undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n | null\n | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,kBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACef,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAO,MAAM,IAAI;AAAA,MACjB,OAAO,MAAM,IAAI;AAAA,MACjB,MAAM,MAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAO,MAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAM,MAAM,IAAI;AAAA,IAChB,aAAa,MAAM,YAAY,UAAU;AAAA,IACzC,cAAc,MAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;AC5FA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACnE,QAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAEvE,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAGA,QAAM,cAAc,aAAa,CAAC;AAClC,QAAM,eAAe,cAAc,CAAC;AAEpC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,QAAM,WAAW,KAAK,IAAI,aAAa,QAAQ,cAAc,MAAM;AACnE,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAEtE,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,IACF;AAGA,UAAM,QAAQ,YAAY,IAAI;AAC9B,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AACxB,YAAM,IAAI,MAAM;AAEhB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAE/J,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,YAAY,IAAI,UAAU,YAAY,SAAS;AAAA,IAC/D,WAAW,EAAE,IAAI,aAAa,IAAI,UAAU,aAAa,SAAS;AAAA,IAClE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9PO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;;;ACVO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEO,SAASA,OAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;;;ACdA,yBAA2B;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAkBA,SAAS,gBAAgB,QAAiB,QAAQ,mBAAuC;AACvF,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,MAAO,OAA8B;AAC3C,MAAI,QAAQ,OAAW,QAAO;AAC9B,MAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,KAAK,OAAO,EAAG,QAAO;AAExE,UAAQ;AAAA,IACN,6CAA6C,KAAK,qDAAqD,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI,OAAO,GAAG;AAAA,EAC3J;AACA,SAAO;AACT;AASA,SAAS,oBACP,QACA,QAAQ,mBAC6B;AACrC,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,OAAQ,OAAkC;AAChD,MAAI,SAAS,OAAW,QAAO;AAC/B,MAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;AACpE,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAuC,SAAS,OAAO,SAAS,MAAM,QAAQ,IAAI,IAAI,UAAU,OAAO,IAAI;AAAA,IACnK;AACA,WAAO;AAAA,EACT;AAIA,QAAM,QAAQ,OAAO,eAAe,IAAI;AACxC,MAAI,UAAU,OAAO,aAAa,UAAU,MAAM;AAChD,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAwC,OAAO,aAAa,QAA+B,eAAe;AAAA,IAClK;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,eAAsB,QACpB,QACA,iBAIA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,SAAS,QAAQ,SAAS;AAC5B,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AAKF,UAAI,SAAS,eAAe;AAC1B,cAAM,UAAU,MAAM,QAAQ;AAAA,UAC5B,YAAY,gBAAgB,KAAK,OAAO,OAAO;AAAA,UAC/C,EAAE,eAAe,KAAK;AAAA,QACxB;AACA,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,QAAQ,OAAO;AAKjC,iBAAS,OAAO,gBAAgB,QAAQ,MAAM,KAAK,QAAQ;AAC3D,iBAAS,WAAW,oBAAoB,QAAQ,MAAM,KAAK,QAAQ;AACnE,YAAI,QAAQ,UAAU,QAAQ,OAAO,SAAS,GAAG;AAC/C,mBAAS,SAAS,QAAQ;AAAA,QAC5B;AACA,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF,OAAO;AACL,cAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,OAAO;AACzB,iBAAS,OAAO,gBAAgB,MAAM;AACtC,cAAM,OAAO,oBAAoB,MAAM;AACvC,YAAI,KAAM,UAAS,WAAW;AAC9B,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAIhE,UAAI,SAAS,eAAe;AAC1B,cAAM,WAAY,IAAwC;AAC1D,YAAI,MAAM,QAAQ,QAAQ,KAAK,SAAS,SAAS,GAAG;AAClD,mBAAS,SAAS;AAAA,QACpB;AAAA,MACF;AACA,gBAAU,SAAS,IAAI;AACvB,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,SAAS,QAAQ,QAAS;AAC9B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAM,OAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAC/B,mBAAS,aAAa,OAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACvD,mBAAS,aAAa,OAAO,IAAI,IAAI;AAAA,YACnC,OAAOA,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAC/B,iBAAS,aAAa,OAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AACvB,aAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAAA,EAClF;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAGA,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,WAAS,aAAa,EAAE,MAAM,YAAY,YAAY,MAAM,QAAQ,SAAS,CAAC;AAE9E,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,IACL;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;ACpXA,IAAAC,sBAA2B;AAU3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAE5C,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAE/E,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAKA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,SAAS;AAAA,MACjB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAEA,QAAI,iBAAiB;AAErB,eAAW,UAAU,SAAS;AAC5B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,eAAK,aAAa;AAAA,YAChB,WAAW,OAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,eAAK,OAAO,OAAO,IAAI,IAAI;AAC3B,eAAK,aAAc,OAAO,IAAI,IAAI;AAAA,YAChC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,eAAK,OAAO,OAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACnD,eAAK,aAAc,OAAO,IAAI,IAAI;AAAA,YAChC,OAAOA,OAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,aAAK,aAAa;AAAA,UAChB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,aAAK,OAAO,OAAO,IAAI,IAAI;AAC3B,aAAK,aAAc,OAAO,IAAI,IAAI;AAAA,UAChC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,aAAS,SAAS,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,OAAO,MAAM,QAAQ;AAClC,YAAM,eAAe;AACrB,YAAM,YAAY,OAAO,MAAM,YAAY,GAAG,YAAY;AAAA,IAC5D;AAAA,EACF;AAEA,QAAM,UAAU,MAAM;AAAA,IAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,MAAM,EAAE;AAAA,IAAG,MACjF,QAAQ;AAAA,EACV;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,QAAI,gCAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAKf,YAAM,EAAE,YAAY,GAAG,UAAU,IAAI,GAAG,KAAK,IAAI,OAAO;AACxD,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;AC5LA,IAAAC,sBAA2B;AAoB3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,kBAAyB,gCAAW;AACzE,QAAM,EAAE,QAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AAChE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAEjC,YAAQ,IAAI,IAAI;AAAA,MACd,MAAMC,OAAM,WAAW;AAAA,MACvB,KAAKA,OAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAKA,OAAM,OAAO;AAAA,MAClB,KAAKA,OAAM,OAAO;AAAA,IACpB;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAMA,OAAM,UAAU;AAAA,MACtB,KAAKA,OAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA,WAAWA,OAAM,SAAS;AAAA,IAC1B,eAAeA,OAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;;;ACtFA,uBAAwB;AACxB,qBAA2B;AAC3B,sBAA8B;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,QAAI,0BAAQ,KAAK,IAAI;AAC3B,YAAI,2BAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,eAAW,+BAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,WAAO,+BAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AR/GA;AAoBA,IAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAiBZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,SAAS,kBAAkB,MAA6D;AACtF,QAAM,MAAM,KAAK,QAAQ,aAAa;AACtC,MAAI,QAAQ,MAAM,MAAM,KAAK,KAAK,OAAQ,QAAO;AACjD,QAAM,QAAQ,KAAK,MAAM,CAAC;AAG1B,MAAI,WAAW,KAAK,KAAK,EAAG,QAAO,WAAW,KAAK;AAGnD,QAAM,MAA8B,CAAC;AACrC,aAAW,QAAQ,MAAM,MAAM,GAAG,GAAG;AACnC,UAAM,CAAC,KAAK,GAAG,IAAI,KAAK,MAAM,GAAG;AACjC,QAAI,OAAO,OAAO,KAAM,KAAI,IAAI,KAAK,CAAC,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EACjE;AACA,SAAO,OAAO,KAAK,GAAG,EAAE,SAAS,IAAI,MAAM;AAC7C;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,aAAa,kBAAkB,IAAI;AACzC,QAAM,QAAQ,KAAK;AAAA,IACjB,CAAC,GAAG,MAAM,CAAC,EAAE,WAAW,IAAI,KAAK,EAAE,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM;AAAA,EAC9D;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,gFAAgF;AAC9F,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,cAAc,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACrE,QAAM,eAAe,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACtE,QAAM,WAAsC;AAC5C,QAAM,YAAuC;AAC7C,QAAM,iBAAiB,cAAc,OAAO,EAAE,WAAW,IAAI;AAC7D,QAAM,aAAa,YAAY,UAAU,WAAW,cAAc;AAElE,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAClE,QAAM,QAAQ,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,MAAM,IAAI;AAEtE,QAAM,WAAW,QAAQ,4BAA4B;AACrD,QAAM,SAAS,QAAQ,4BAA4B;AACnD,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,0CAA0C,QAAQ,EAAE;AAChG,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,0CAA0C,MAAM,EAAE;AAE7F,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,QAAI,OAAO,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AACjO,QAAI,OAAO;AACT,UAAI,EAAE,IAAI;AACR,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,cAAc,QAAQ;AACpC,gBAAQ,MAAM,EAAE,KAAK,EAAE,IAAI,GAAG;AAAA,MAChC,OAAO;AACL,gBAAQ,WAAM,OAAO,EAAE;AAAA,MACzB;AAAA,IACF;AACA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAEA,MAAI,WAAW,QAAQ;AACrB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN;AAAA,sBAAyB,EAAE,eAAe,KAAM,QAAQ,CAAC,CAAC,mBAAmB,EAAE,gBAAgB,KAAM,QAAQ,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACvJ;AAAA,EACF;AACA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,qBAAqB,EAAE,cAAc,QAAQ,CAAC,CAAC,kBAAkB,EAAE,eAAe,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACnI;AAAA,EACF;AAEA,QAAM,cAAc,MAAM,QAAQ,QAAQ,IAAI,SAAS,CAAC,IAAI;AAC5D,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,YAAY,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EACrF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AAEzD,UAAM,kBAAkB,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,eAAe,IAAI;AACzF,QAAI,iBAAiB;AACnB,YAAM,2BAA2B,YAAY;AAAA,QAC3C,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,gBAAgB,QAAQ,WAAW,QAAQ,CAAC,EAAE,QAAQ;AAAA,MACrF;AACA,UAAI,yBAA0B,SAAQ,KAAK,CAAC;AAAA,IAC9C,OAAO;AACL,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AACF;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,SAAS,GAAG;AACpB,YAAQ,MAAM,sEAAsE;AACpF,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,CAAC,aAAa,YAAY,IAAI;AACpC,QAAM,MAAM,KAAK,MAAM,UAAM,gBAAAA,UAAc,aAAa,OAAO,CAAC;AAChE,QAAM,UAAwB,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,GAAG;AAE7D,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,MAAI;AACF,UAAM,MAAM,MAAM,aAAkB,aAAQ,YAAY,GAAG,YAAY,GAAG;AAC1E,UAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,QAAI,CAAC,WAAW,WAAW,WAAW,QAAQ,WAAW,GAAG;AAC1D,cAAQ,MAAM,UAAU,YAAY,0BAA0B;AAC9D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,UAAM,WAAyB,CAAC;AAChC,eAAW,cAAc,SAAS;AAChC,eAAS,KAAK,MAAM,QAAQ,YAAY,WAAW,SAAS,OAAO,CAAC;AAAA,IACtE;AAEA,eAAW,KAAK,UAAU;AACxB,cAAQ,IAAI,OAAO,YAAY,CAAC,IAAI,IAAI;AAAA,IAC1C;AAEA,QAAI,YAAY;AACd,YAAM,SAAS,MAAM,QAAQ,GAAG,IAAI,WAAW,SAAS,CAAC;AACzD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,6BAA6B,UAAU,EAAE;AAAA,IACvD;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,0BAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,6BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAGA,SAAS,gBAAgB,WAA4B;AACnD,MAAI,CAAC,MAAM,QAAQ,SAAS,KAAK,UAAU,WAAW,EAAG,QAAO;AAChE,SAAQ,UAAwB,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,IAAI;AAC7F;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM;AAAA,IACJ,SAAS,gBAAgB,OAAO,SAAS,SAAS,CAAC,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK;AAAA,EAClG;AACA,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AAErC,UAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAAE;AACvF,QAAI,oBAAoB,GAAG;AACzB,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,MAC1K;AAAA,IACF,OAAO;AACL,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,MACvO;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,QAAQ,QAAQ;AACzB,UAAM,IAAI,OAAO,QAAQ;AACzB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,MAAM,EAAE,OAAO,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACtL;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,QAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,MAAM;AACzE,MAAI,gBAAgB,SAAS,GAAG;AAC9B,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,gBAAgB,QAAQ,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC;AACjF,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,oBAAoB,gBAAgB,MAAM,IAAI,OAAO,QAAQ,KAAK;AAAA,IACpE;AACA,eAAW,OAAO,aAAa,MAAM,GAAG,CAAC,GAAG;AAC1C,YAAM,KAAK,SAAS,GAAG,EAAE;AAAA,IAC3B;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,KAAK,eAAe,aAAa,SAAS,CAAC,OAAO;AAAA,IAC1D;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,oBAAoB,SAAkC;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,QAAQ,OAAO;AAC/C,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM;AAAA,IACJ,SAAS,gBAAgB,QAAQ,SAAS,CAAC,MAAM,QAAQ,OAAO,WAAW,QAAQ,QAAQ;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,gBAAkB,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC;AAAA,EACvH;AACA,QAAM,UAAU,aAAa,IAAI,WAAW,IAAI,IAAI,IAAI;AACxD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,QAAQ,QAAQ,IAAI;AAC9B,UAAM,UAAU,GAAG,EAAE,KAAK,QAAQ,CAAC,CAAC,SAAW,EAAE,IAAI,QAAQ,CAAC,CAAC;AAC/D,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,QAAQ,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,IAC/H;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ;AAClB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,OAAO,QAAQ,OAAO,OAAO,KAAM,QAAQ,CAAC,IAAI,YAAc,QAAQ,OAAO,MAAM,KAAM,QAAQ,CAAC,IAAI,KAAK,SAAS,QAAQ,CAAC;AAAA,IAC/J;AAAA,EACF;AAEA,QAAM,UAAU,QAAQ,YAAY,IAAI,IAAI,QAAQ,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC7E,QAAM,eAAe,QAAQ,gBAAgB,KAAM,QAAQ,CAAC;AAC5D,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,iBAAiB,OAAO,sBAAsB,WAAW,GAAG;AAEvE,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,WAAO,0BAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAOrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,MAAI,OAAO;AACX,MAAI,gBAAgB;AACpB,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,kBAAkB,QAAQ,UAAU;AAC1F,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA,eAChC,QAAQ,SAAU,QAAO,KAAK,IAAI,GAAG,SAAS,OAAO,EAAE,KAAK,CAAC;AAAA;AAEpE,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,QAAQ,oBAAoB;AAErC,sBAAgB;AAAA,IAClB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM,eAAe,MAAM;AACzE;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,eAAe,MAAM,IAAI,cAAc,IAAI;AAE5F,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM;AAAA,cACJ;AAAA,cACA,MAAM;AAAA,cACN;AAAA,YACF,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC3C,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO;AAAA,cACL,QAAQ,OAAO;AAAA,cACf,MAAM,OAAO,QAAQ;AAAA,cACrB,UAAU,OAAO,YAAY;AAAA,YAC/B;AAAA,UACF;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAQ,MAAM,SAAS,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC1E,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQ,QAAQ,MAAM,SAAS;AAAA,UAC1C;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AASA,cAAM,aAAa,gBAAgB,EAAE,eAAe,KAAK,IAAI;AAE7D,YAAI,OAAO,GAAG;AAEZ,gBAAM,EAAE,YAAAC,YAAW,IAAI,MAAM,OAAO,QAAa;AACjD,gBAAM,aAAaA,YAAW;AAC9B,gBAAM,aAA2B,CAAC;AAElC,mBAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,oBAAQ,MAAM,kBAAkB,IAAI,CAAC,IAAI,IAAI,KAAK;AAClD,kBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,SAAS,UAAU;AAC7E,mBAAO,SAAS,aAAa;AAC7B,mBAAO,SAAS,WAAW;AAC3B,uBAAW,KAAK,MAAM;AACtB,oBAAQ,KAAK,MAAM;AAAA,UACrB;AAEA,gBAAM,UAAU,cAAc,UAAU;AACxC,kBAAQ,IAAI,OAAO,oBAAoB,OAAO,IAAI,IAAI;AAAA,QACxD,OAAO;AACL,gBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,SAAS,UAAU;AAC7E,kBAAQ,KAAK,MAAM;AAEnB,kBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,QAC/C;AAAA,MACF,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAD,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["import_node_fs","round","round","round","import_node_crypto","round","import_node_crypto","round","readFileAsync","writeFileAsync","randomUUID"]}
package/dist/cli.js CHANGED
@@ -4,7 +4,7 @@ import {
4
4
  evalCompare,
5
5
  rescore,
6
6
  runEval
7
- } from "./chunk-T4OFZQCJ.js";
7
+ } from "./chunk-CV6VE5VT.js";
8
8
 
9
9
  // src/cli.ts
10
10
  import { readdirSync, statSync } from "fs";
@@ -80,7 +80,8 @@ var KNOWN_FLAGS = /* @__PURE__ */ new Set([
80
80
  "--conditions",
81
81
  "--fail-on-regression",
82
82
  "--threshold",
83
- "--runs"
83
+ "--runs",
84
+ "--capture-traces"
84
85
  ]);
85
86
  async function main() {
86
87
  const args = process.argv.slice(2);
@@ -92,6 +93,10 @@ Usage:
92
93
  axl-eval <path> --output <file> Save results to JSON
93
94
  axl-eval <path> --config <file> Use config file for runtime
94
95
  axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)
96
+ axl-eval <path> --capture-traces Populate EvalItem.traces on every item
97
+ (success + failure). Adds memory
98
+ overhead proportional to dataset size
99
+ x turns x agents; off by default.
95
100
  axl-eval rescore <results> <eval-file> Re-run scorers on saved outputs
96
101
  axl-eval compare <a> <b> Compare two eval result files
97
102
  axl-eval compare <a> <b> --threshold <v> Set regression threshold (global or per-scorer)
@@ -412,6 +417,7 @@ function parseEvalArgs(args) {
412
417
  let configArg;
413
418
  let conditions = [];
414
419
  let runs = 1;
420
+ let captureTraces = false;
415
421
  const paths = [];
416
422
  for (let i = 0; i < args.length; i++) {
417
423
  const arg = args[i];
@@ -426,6 +432,8 @@ function parseEvalArgs(args) {
426
432
  else if (arg === "--runs") runs = Math.max(1, parseInt(value, 10) || 1);
427
433
  else
428
434
  conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
435
+ } else if (arg === "--capture-traces") {
436
+ captureTraces = true;
429
437
  } else if (arg.startsWith("--")) {
430
438
  if (!KNOWN_FLAGS.has(arg)) {
431
439
  console.error(`Unknown flag: ${arg}`);
@@ -435,10 +443,10 @@ function parseEvalArgs(args) {
435
443
  paths.push(arg);
436
444
  }
437
445
  }
438
- return { outputPath, configArg, conditions, runs, paths };
446
+ return { outputPath, configArg, conditions, runs, captureTraces, paths };
439
447
  }
440
448
  async function runEvalCommand(args) {
441
- const { outputPath, configArg, conditions, runs, paths } = parseEvalArgs(args);
449
+ const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);
442
450
  if (paths.length === 0) {
443
451
  console.error("Error: No eval file path provided");
444
452
  process.exit(1);
@@ -493,13 +501,14 @@ async function runEvalCommand(args) {
493
501
  );
494
502
  executeWorkflow = async (input) => ({ output: input });
495
503
  }
504
+ const runOptions = captureTraces ? { captureTraces: true } : void 0;
496
505
  if (runs > 1) {
497
506
  const { randomUUID } = await import("crypto");
498
507
  const runGroupId = randomUUID();
499
508
  const runResults = [];
500
509
  for (let r = 0; r < runs; r++) {
501
510
  console.error(`[axl-eval] Run ${r + 1}/${runs}...`);
502
- const result = await runEval(evalConfig, executeWorkflow, runtime);
511
+ const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
503
512
  result.metadata.runGroupId = runGroupId;
504
513
  result.metadata.runIndex = r;
505
514
  runResults.push(result);
@@ -508,7 +517,7 @@ async function runEvalCommand(args) {
508
517
  const summary = aggregateRuns(runResults);
509
518
  console.log("\n" + formatMultiRunTable(summary) + "\n");
510
519
  } else {
511
- const result = await runEval(evalConfig, executeWorkflow, runtime);
520
+ const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
512
521
  results.push(result);
513
522
  console.log("\n" + formatTable(result) + "\n");
514
523
  }