npm - @axlsdk/eval - Versions diffs - 0.13.8 → 0.15.0 - Mend

@axlsdk/eval 0.13.8 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

package/README.md +46 -2
package/dist/{chunk-CMWRFU3P.js → chunk-CV6VE5VT.js} +137 -15
package/dist/chunk-CV6VE5VT.js.map +1 -0
package/dist/cli.cjs +160 -21
package/dist/cli.cjs.map +1 -1
package/dist/cli.js +25 -8
package/dist/cli.js.map +1 -1
package/dist/index.cjs +136 -14
package/dist/index.cjs.map +1 -1
package/dist/index.d.cts +72 -5
package/dist/index.d.ts +72 -5
package/dist/index.js +1 -1
package/package.json +2 -2
package/dist/chunk-CMWRFU3P.js.map +0 -1

package/dist/cli.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"sources":["../src/cli.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport { rescore } from './rescore.js';\nimport { aggregateRuns } from './multi-run.js';\nimport type { MultiRunSummary } from './multi-run.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set([\n '--output',\n '--config',\n '--conditions',\n '--fail-on-regression',\n '--threshold',\n '--runs',\n]);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 \|\| args[0] === '--help' \|\| args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --runs <n> Run eval N times (multi-run)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval rescore <results> <eval-file> Re-run scorers on saved outputs\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --threshold <v> Set regression threshold (global or per-scorer)\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n if (args[0] === 'rescore') {\n await runRescore(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nfunction parseThresholdArg(args: string[]): Record<string, number> \| number \| undefined {\n const idx = args.indexOf('--threshold');\n if (idx === -1 \|\| idx + 1 >= args.length) return undefined;\n const value = args[idx + 1];\n\n // Global numeric threshold: --threshold 0.05\n if (/^[\\d.]+$/.test(value)) return parseFloat(value);\n\n // Per-scorer map: --threshold accuracy=0,tone=0.1\n const map: Record<string, number> = {};\n for (const pair of value.split(',')) {\n const [key, val] = pair.split('=');\n if (key && val != null) map[key.trim()] = parseFloat(val.trim());\n }\n return Object.keys(map).length > 0 ? map : undefined;\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const thresholds = parseThresholdArg(args);\n const files = args.filter(\n (a, i) => !a.startsWith('--') && !(i > 0 && args[i - 1] === '--threshold'),\n );\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]');\n process.exit(1);\n }\n\n const baselineRaw = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidateRaw = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const baseline: EvalResult \| EvalResult[] = baselineRaw;\n const candidate: EvalResult \| EvalResult[] = candidateRaw;\n const compareOptions = thresholds != null ? { thresholds } : undefined;\n const comparison = evalCompare(baseline, candidate, compareOptions);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n const hasCI = scorerNames.some((n) => comparison.scorers[n].ci != null);\n\n const ciHeader = hasCI ? ' CI 95% Sig' : '';\n const ciRule = hasCI ? ' ---------------- ---' : '';\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change${ciHeader}`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------${ciRule}`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n let line = ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`;\n if (hasCI) {\n if (s.ci) {\n const lo = (s.ci.lower >= 0 ? '+' : '') + s.ci.lower.toFixed(4);\n const hi = (s.ci.upper >= 0 ? '+' : '') + s.ci.upper.toFixed(4);\n const sig = s.significant ? ' ' : ' ';\n line += ` [${lo}, ${hi}]${sig}`;\n } else {\n line += ' —'.padEnd(22);\n }\n }\n console.log(line);\n }\n\n if (comparison.timing) {\n const t = comparison.timing;\n const sign = t.delta > 0 ? '+' : '';\n console.log(\n `\\n Timing: baseline ${(t.baselineMean / 1000).toFixed(2)}s -> candidate ${(t.candidateMean / 1000).toFixed(2)}s (${sign}${t.deltaPercent.toFixed(1)}%)`,\n );\n }\n if (comparison.cost) {\n const c = comparison.cost;\n const sign = c.delta > 0 ? '+' : '';\n console.log(\n ` Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`,\n );\n }\n\n const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;\n const stable = Math.max(\n 0,\n baselineRef.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} \| Improvements: ${comparison.improvements.length} \| Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n // When CI is available, only fail on significant regressions\n const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);\n if (hasSignificance) {\n const hasSignificantRegression = scorerNames.some(\n (n) => comparison.scorers[n].significant === true && comparison.scorers[n].delta < 0,\n );\n if (hasSignificantRegression) process.exit(1);\n } else {\n process.exit(1);\n }\n }\n}\n\nasync function runRescore(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length < 2) {\n console.error('Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]');\n process.exit(1);\n }\n\n const [resultsPath, evalFilePath] = paths;\n const raw = JSON.parse(await readFileAsync(resultsPath, 'utf-8'));\n const results: EvalResult[] = Array.isArray(raw) ? raw : [raw];\n\n const runtime = await getRuntime(configArg, conditions);\n try {\n const mod = await importModule(path.resolve(evalFilePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.scorers \|\| evalConfig.scorers.length === 0) {\n console.error(`Error: ${evalFilePath} does not export scorers`);\n process.exit(1);\n }\n\n const rescored: EvalResult[] = [];\n for (const resultData of results) {\n rescored.push(await rescore(resultData, evalConfig.scorers, runtime));\n }\n\n for (const r of rescored) {\n console.log('\\n' + formatTable(r) + '\\n');\n }\n\n if (outputPath) {\n const output = Array.isArray(raw) ? rescored : rescored[0];\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Rescored results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n // Detect scorers with no valid scores (all items errored → computeStats([]) → all zeros)\n const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;\n if (validScoreCount === 0) {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)}`,\n );\n } else {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n }\n\n if (result.summary.timing) {\n const t = result.summary.timing;\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${(t.mean / 1000).toFixed(2).padStart(colWidth)}s ${(t.p50 / 1000).toFixed(2).padStart(colWidth)}s ${(t.p95 / 1000).toFixed(2).padStart(colWidth)}s`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} \| Cost: ${costStr} \| Duration: ${durationSec}s`,\n );\n\n const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);\n if (itemsWithErrors.length > 0) {\n const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors!))];\n lines.push('');\n lines.push(\n ` Scorer errors (${itemsWithErrors.length}/${result.summary.count} items affected):`,\n );\n for (const err of uniqueErrors.slice(0, 5)) {\n lines.push(` - ${err}`);\n }\n if (uniqueErrors.length > 5) {\n lines.push(` ... and ${uniqueErrors.length - 5} more`);\n }\n }\n\n return lines.join('\\n');\n}\n\nfunction formatMultiRunTable(summary: MultiRunSummary): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 16;\n\n lines.push(`Eval: ${summary.workflow} x ${summary.dataset} \\u2014 ${summary.runCount} runs`);\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean \\u00b1 Std'.padStart(colWidth)} ${'Min'.padStart(8)} ${'Max'.padStart(8)}`,\n );\n const ruleLen = maxNameLen + 2 + colWidth + 2 + 8 + 2 + 8;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = summary.scorers[name];\n const meanStd = `${s.mean.toFixed(3)} \\u00b1 ${s.std.toFixed(3)}`;\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${meanStd.padStart(colWidth)} ${s.min.toFixed(3).padStart(8)} ${s.max.toFixed(3).padStart(8)}`,\n );\n }\n\n if (summary.timing) {\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${((summary.timing.mean / 1000).toFixed(2) + ' \\u00b1 ' + (summary.timing.std / 1000).toFixed(2) + 's').padStart(colWidth)}`,\n );\n }\n\n const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : '$0.00';\n const durationStr = (summary.totalDuration / 1000).toFixed(1);\n lines.push('');\n lines.push(` Total Cost: ${costStr} \| Total Duration: ${durationStr}s`);\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime \|\| typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement\|Unexpected reserved word\|top-level await\|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' \|\| ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n runs: number;\n paths: string[];\n} {\n let outputPath: string \| undefined;\n let configArg: string \| undefined;\n let conditions: string[] = [];\n let runs = 1;\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' \|\| arg === '--config' \|\| arg === '--conditions' \|\| arg === '--runs') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else if (arg === '--runs') runs = Math.max(1, parseInt(value, 10) \|\| 1);\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, runs, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, runs, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow \|\| !evalConfig.dataset \|\| !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackExecution for cost + metadata attribution\n executeWorkflow = async (input, rt) => {\n const {\n result,\n cost: trackedCost,\n metadata,\n } = await runtime.trackExecution(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return {\n output: result.output,\n cost: result.cost ?? trackedCost,\n metadata: result.metadata ?? metadata,\n };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost, metadata } = await runtime.trackExecution(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost, metadata };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n if (runs > 1) {\n // Multi-run mode\n const { randomUUID } = await import('node:crypto');\n const runGroupId = randomUUID();\n const runResults: EvalResult[] = [];\n\n for (let r = 0; r < runs; r++) {\n console.error(`[axl-eval] Run ${r + 1}/${runs}...`);\n const result = await runEval(evalConfig, executeWorkflow, runtime);\n result.metadata.runGroupId = runGroupId;\n result.metadata.runIndex = r;\n runResults.push(result);\n results.push(result);\n }\n\n const summary = aggregateRuns(runResults);\n console.log('\\n' + formatMultiRunTable(summary) + '\\n');\n } else {\n const result = await runEval(evalConfig, executeWorkflow, runtime);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n }\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","/*\n CLI utilities for config detection, runtime resolution, and loader registration.\n \n These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n /\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string \| undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/* Returns true if the file is TypeScript and needs tsx to load. /\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/\n Resolve the AxlRuntime from a dynamically imported config module.\n \n Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n /\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> \| undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n \| ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n \| null\n \| undefined;\n\n/\n Import a module, using tsx's `tsImport()` for TypeScript files.\n \n `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;;;AAEA,SAAS,aAAa,gBAAgB;AACtC,SAAS,YAAY,eAAe,aAAa,gBAAgB,aAAa;AAC9E,YAAY,UAAU;;;ACItB,SAAS,eAAe;AACxB,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,IAAI,QAAQ,KAAK,IAAI;AAC3B,QAAI,WAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,WAAW,cAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,OAAO,cAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AD3FA,IAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAaZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,SAAS,kBAAkB,MAA6D;AACtF,QAAM,MAAM,KAAK,QAAQ,aAAa;AACtC,MAAI,QAAQ,MAAM,MAAM,KAAK,KAAK,OAAQ,QAAO;AACjD,QAAM,QAAQ,KAAK,MAAM,CAAC;AAG1B,MAAI,WAAW,KAAK,KAAK,EAAG,QAAO,WAAW,KAAK;AAGnD,QAAM,MAA8B,CAAC;AACrC,aAAW,QAAQ,MAAM,MAAM,GAAG,GAAG;AACnC,UAAM,CAAC,KAAK,GAAG,IAAI,KAAK,MAAM,GAAG;AACjC,QAAI,OAAO,OAAO,KAAM,KAAI,IAAI,KAAK,CAAC,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EACjE;AACA,SAAO,OAAO,KAAK,GAAG,EAAE,SAAS,IAAI,MAAM;AAC7C;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,aAAa,kBAAkB,IAAI;AACzC,QAAM,QAAQ,KAAK;AAAA,IACjB,CAAC,GAAG,MAAM,CAAC,EAAE,WAAW,IAAI,KAAK,EAAE,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM;AAAA,EAC9D;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,gFAAgF;AAC9F,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,cAAc,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACrE,QAAM,eAAe,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACtE,QAAM,WAAsC;AAC5C,QAAM,YAAuC;AAC7C,QAAM,iBAAiB,cAAc,OAAO,EAAE,WAAW,IAAI;AAC7D,QAAM,aAAa,YAAY,UAAU,WAAW,cAAc;AAElE,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAClE,QAAM,QAAQ,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,MAAM,IAAI;AAEtE,QAAM,WAAW,QAAQ,4BAA4B;AACrD,QAAM,SAAS,QAAQ,4BAA4B;AACnD,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,0CAA0C,QAAQ,EAAE;AAChG,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,0CAA0C,MAAM,EAAE;AAE7F,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,QAAI,OAAO,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AACjO,QAAI,OAAO;AACT,UAAI,EAAE,IAAI;AACR,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,cAAc,QAAQ;AACpC,gBAAQ,MAAM,EAAE,KAAK,EAAE,IAAI,GAAG;AAAA,MAChC,OAAO;AACL,gBAAQ,WAAM,OAAO,EAAE;AAAA,MACzB;AAAA,IACF;AACA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAEA,MAAI,WAAW,QAAQ;AACrB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN;AAAA,sBAAyB,EAAE,eAAe,KAAM,QAAQ,CAAC,CAAC,mBAAmB,EAAE,gBAAgB,KAAM,QAAQ,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACvJ;AAAA,EACF;AACA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,qBAAqB,EAAE,cAAc,QAAQ,CAAC,CAAC,kBAAkB,EAAE,eAAe,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACnI;AAAA,EACF;AAEA,QAAM,cAAc,MAAM,QAAQ,QAAQ,IAAI,SAAS,CAAC,IAAI;AAC5D,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,YAAY,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EACrF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AAEzD,UAAM,kBAAkB,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,eAAe,IAAI;AACzF,QAAI,iBAAiB;AACnB,YAAM,2BAA2B,YAAY;AAAA,QAC3C,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,gBAAgB,QAAQ,WAAW,QAAQ,CAAC,EAAE,QAAQ;AAAA,MACrF;AACA,UAAI,yBAA0B,SAAQ,KAAK,CAAC;AAAA,IAC9C,OAAO;AACL,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AACF;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,SAAS,GAAG;AACpB,YAAQ,MAAM,sEAAsE;AACpF,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,CAAC,aAAa,YAAY,IAAI;AACpC,QAAM,MAAM,KAAK,MAAM,MAAM,cAAc,aAAa,OAAO,CAAC;AAChE,QAAM,UAAwB,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,GAAG;AAE7D,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,MAAI;AACF,UAAM,MAAM,MAAM,aAAkB,aAAQ,YAAY,GAAG,YAAY,GAAG;AAC1E,UAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,QAAI,CAAC,WAAW,WAAW,WAAW,QAAQ,WAAW,GAAG;AAC1D,cAAQ,MAAM,UAAU,YAAY,0BAA0B;AAC9D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,UAAM,WAAyB,CAAC;AAChC,eAAW,cAAc,SAAS;AAChC,eAAS,KAAK,MAAM,QAAQ,YAAY,WAAW,SAAS,OAAO,CAAC;AAAA,IACtE;AAEA,eAAW,KAAK,UAAU;AACxB,cAAQ,IAAI,OAAO,YAAY,CAAC,IAAI,IAAI;AAAA,IAC1C;AAEA,QAAI,YAAY;AACd,YAAM,SAAS,MAAM,QAAQ,GAAG,IAAI,WAAW,SAAS,CAAC;AACzD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,6BAA6B,UAAU,EAAE;AAAA,IACvD;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,UAAU,YAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AAErC,UAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAAE;AACvF,QAAI,oBAAoB,GAAG;AACzB,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,MAC1K;AAAA,IACF,OAAO;AACL,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,MACvO;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,QAAQ,QAAQ;AACzB,UAAM,IAAI,OAAO,QAAQ;AACzB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,MAAM,EAAE,OAAO,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACtL;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,QAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,MAAM;AACzE,MAAI,gBAAgB,SAAS,GAAG;AAC9B,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,gBAAgB,QAAQ,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC;AACjF,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,oBAAoB,gBAAgB,MAAM,IAAI,OAAO,QAAQ,KAAK;AAAA,IACpE;AACA,eAAW,OAAO,aAAa,MAAM,GAAG,CAAC,GAAG;AAC1C,YAAM,KAAK,SAAS,GAAG,EAAE;AAAA,IAC3B;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,KAAK,eAAe,aAAa,SAAS,CAAC,OAAO;AAAA,IAC1D;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,oBAAoB,SAAkC;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,QAAQ,OAAO;AAC/C,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,QAAQ,QAAQ,MAAM,QAAQ,OAAO,WAAW,QAAQ,QAAQ,OAAO;AAC3F,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,gBAAkB,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC;AAAA,EACvH;AACA,QAAM,UAAU,aAAa,IAAI,WAAW,IAAI,IAAI,IAAI;AACxD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,QAAQ,QAAQ,IAAI;AAC9B,UAAM,UAAU,GAAG,EAAE,KAAK,QAAQ,CAAC,CAAC,SAAW,EAAE,IAAI,QAAQ,CAAC,CAAC;AAC/D,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,QAAQ,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,IAC/H;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ;AAClB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,OAAO,QAAQ,OAAO,OAAO,KAAM,QAAQ,CAAC,IAAI,YAAc,QAAQ,OAAO,MAAM,KAAM,QAAQ,CAAC,IAAI,KAAK,SAAS,QAAQ,CAAC;AAAA,IAC/J;AAAA,EACF;AAEA,QAAM,UAAU,QAAQ,YAAY,IAAI,IAAI,QAAQ,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC7E,QAAM,eAAe,QAAQ,gBAAgB,KAAM,QAAQ,CAAC;AAC5D,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,iBAAiB,OAAO,sBAAsB,WAAW,GAAG;AAEvE,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,OAAO,SAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAMrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,MAAI,OAAO;AACX,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,kBAAkB,QAAQ,UAAU;AAC1F,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA,eAChC,QAAQ,SAAU,QAAO,KAAK,IAAI,GAAG,SAAS,OAAO,EAAE,KAAK,CAAC;AAAA;AAEpE,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM,MAAM;AAC1D;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,MAAM,IAAI,cAAc,IAAI;AAE7E,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM;AAAA,cACJ;AAAA,cACA,MAAM;AAAA,cACN;AAAA,YACF,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC3C,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO;AAAA,cACL,QAAQ,OAAO;AAAA,cACf,MAAM,OAAO,QAAQ;AAAA,cACrB,UAAU,OAAO,YAAY;AAAA,YAC/B;AAAA,UACF;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAQ,MAAM,SAAS,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC1E,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQ,QAAQ,MAAM,SAAS;AAAA,UAC1C;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,YAAI,OAAO,GAAG;AAEZ,gBAAM,EAAE,WAAW,IAAI,MAAM,OAAO,QAAa;AACjD,gBAAM,aAAa,WAAW;AAC9B,gBAAM,aAA2B,CAAC;AAElC,mBAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,oBAAQ,MAAM,kBAAkB,IAAI,CAAC,IAAI,IAAI,KAAK;AAClD,kBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,OAAO;AACjE,mBAAO,SAAS,aAAa;AAC7B,mBAAO,SAAS,WAAW;AAC3B,uBAAW,KAAK,MAAM;AACtB,oBAAQ,KAAK,MAAM;AAAA,UACrB;AAEA,gBAAM,UAAU,cAAc,UAAU;AACxC,kBAAQ,IAAI,OAAO,oBAAoB,OAAO,IAAI,IAAI;AAAA,QACxD,OAAO;AACL,gBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,OAAO;AACjE,kBAAQ,KAAK,MAAM;AAEnB,kBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,QAC/C;AAAA,MACF,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
1	+ {"version":3,"sources":["../src/cli.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport { rescore } from './rescore.js';\nimport { aggregateRuns } from './multi-run.js';\nimport type { MultiRunSummary } from './multi-run.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set([\n '--output',\n '--config',\n '--conditions',\n '--fail-on-regression',\n '--threshold',\n '--runs',\n '--capture-traces',\n]);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 \|\| args[0] === '--help' \|\| args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --runs <n> Run eval N times (multi-run)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval <path> --capture-traces Populate EvalItem.traces on every item\n (success + failure). Adds memory\n overhead proportional to dataset size\n x turns x agents; off by default.\n axl-eval rescore <results> <eval-file> Re-run scorers on saved outputs\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --threshold <v> Set regression threshold (global or per-scorer)\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n if (args[0] === 'rescore') {\n await runRescore(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nfunction parseThresholdArg(args: string[]): Record<string, number> \| number \| undefined {\n const idx = args.indexOf('--threshold');\n if (idx === -1 \|\| idx + 1 >= args.length) return undefined;\n const value = args[idx + 1];\n\n // Global numeric threshold: --threshold 0.05\n if (/^[\\d.]+$/.test(value)) return parseFloat(value);\n\n // Per-scorer map: --threshold accuracy=0,tone=0.1\n const map: Record<string, number> = {};\n for (const pair of value.split(',')) {\n const [key, val] = pair.split('=');\n if (key && val != null) map[key.trim()] = parseFloat(val.trim());\n }\n return Object.keys(map).length > 0 ? map : undefined;\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const thresholds = parseThresholdArg(args);\n const files = args.filter(\n (a, i) => !a.startsWith('--') && !(i > 0 && args[i - 1] === '--threshold'),\n );\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json> [--threshold <value>]');\n process.exit(1);\n }\n\n const baselineRaw = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidateRaw = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const baseline: EvalResult \| EvalResult[] = baselineRaw;\n const candidate: EvalResult \| EvalResult[] = candidateRaw;\n const compareOptions = thresholds != null ? { thresholds } : undefined;\n const comparison = evalCompare(baseline, candidate, compareOptions);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n const hasCI = scorerNames.some((n) => comparison.scorers[n].ci != null);\n\n const ciHeader = hasCI ? ' CI 95% Sig' : '';\n const ciRule = hasCI ? ' ---------------- ---' : '';\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change${ciHeader}`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------${ciRule}`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n let line = ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`;\n if (hasCI) {\n if (s.ci) {\n const lo = (s.ci.lower >= 0 ? '+' : '') + s.ci.lower.toFixed(4);\n const hi = (s.ci.upper >= 0 ? '+' : '') + s.ci.upper.toFixed(4);\n const sig = s.significant ? ' ' : ' ';\n line += ` [${lo}, ${hi}]${sig}`;\n } else {\n line += ' —'.padEnd(22);\n }\n }\n console.log(line);\n }\n\n if (comparison.timing) {\n const t = comparison.timing;\n const sign = t.delta > 0 ? '+' : '';\n console.log(\n `\\n Timing: baseline ${(t.baselineMean / 1000).toFixed(2)}s -> candidate ${(t.candidateMean / 1000).toFixed(2)}s (${sign}${t.deltaPercent.toFixed(1)}%)`,\n );\n }\n if (comparison.cost) {\n const c = comparison.cost;\n const sign = c.delta > 0 ? '+' : '';\n console.log(\n ` Cost: baseline $${c.baselineTotal.toFixed(2)} -> candidate $${c.candidateTotal.toFixed(2)} (${sign}${c.deltaPercent.toFixed(1)}%)`,\n );\n }\n\n const baselineRef = Array.isArray(baseline) ? baseline[0] : baseline;\n const stable = Math.max(\n 0,\n baselineRef.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} \| Improvements: ${comparison.improvements.length} \| Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n // When CI is available, only fail on significant regressions\n const hasSignificance = scorerNames.some((n) => comparison.scorers[n].significant != null);\n if (hasSignificance) {\n const hasSignificantRegression = scorerNames.some(\n (n) => comparison.scorers[n].significant === true && comparison.scorers[n].delta < 0,\n );\n if (hasSignificantRegression) process.exit(1);\n } else {\n process.exit(1);\n }\n }\n}\n\nasync function runRescore(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length < 2) {\n console.error('Usage: axl-eval rescore <results.json> <eval-file> [--output <file>]');\n process.exit(1);\n }\n\n const [resultsPath, evalFilePath] = paths;\n const raw = JSON.parse(await readFileAsync(resultsPath, 'utf-8'));\n const results: EvalResult[] = Array.isArray(raw) ? raw : [raw];\n\n const runtime = await getRuntime(configArg, conditions);\n try {\n const mod = await importModule(path.resolve(evalFilePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.scorers \|\| evalConfig.scorers.length === 0) {\n console.error(`Error: ${evalFilePath} does not export scorers`);\n process.exit(1);\n }\n\n const rescored: EvalResult[] = [];\n for (const resultData of results) {\n rescored.push(await rescore(resultData, evalConfig.scorers, runtime));\n }\n\n for (const r of rescored) {\n console.log('\\n' + formatTable(r) + '\\n');\n }\n\n if (outputPath) {\n const output = Array.isArray(raw) ? rescored : rescored[0];\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Rescored results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\n/* Render observed workflows for a terse header line. Multiple → comma-separated. /\nfunction formatWorkflows(workflows: unknown): string {\n if (!Array.isArray(workflows) \|\| workflows.length === 0) return '(unknown)';\n return (workflows as unknown[]).filter((w): w is string => typeof w === 'string').join(', ');\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(\n `Eval: ${formatWorkflows(result.metadata.workflows)} x ${result.dataset} (${result.summary.count} items)`,\n );\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n // Detect scorers with no valid scores (all items errored → computeStats([]) → all zeros)\n const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;\n if (validScoreCount === 0) {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)} ${'--'.padStart(colWidth)}`,\n );\n } else {\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n }\n\n if (result.summary.timing) {\n const t = result.summary.timing;\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${(t.mean / 1000).toFixed(2).padStart(colWidth)}s ${(t.p50 / 1000).toFixed(2).padStart(colWidth)}s ${(t.p95 / 1000).toFixed(2).padStart(colWidth)}s`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} \| Cost: ${costStr} \| Duration: ${durationSec}s`,\n );\n\n const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);\n if (itemsWithErrors.length > 0) {\n const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors!))];\n lines.push('');\n lines.push(\n ` Scorer errors (${itemsWithErrors.length}/${result.summary.count} items affected):`,\n );\n for (const err of uniqueErrors.slice(0, 5)) {\n lines.push(` - ${err}`);\n }\n if (uniqueErrors.length > 5) {\n lines.push(` ... and ${uniqueErrors.length - 5} more`);\n }\n }\n\n return lines.join('\\n');\n}\n\nfunction formatMultiRunTable(summary: MultiRunSummary): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 16;\n\n lines.push(\n `Eval: ${formatWorkflows(summary.workflows)} x ${summary.dataset} \\u2014 ${summary.runCount} runs`,\n );\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean \\u00b1 Std'.padStart(colWidth)} ${'Min'.padStart(8)} ${'Max'.padStart(8)}`,\n );\n const ruleLen = maxNameLen + 2 + colWidth + 2 + 8 + 2 + 8;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = summary.scorers[name];\n const meanStd = `${s.mean.toFixed(3)} \\u00b1 ${s.std.toFixed(3)}`;\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${meanStd.padStart(colWidth)} ${s.min.toFixed(3).padStart(8)} ${s.max.toFixed(3).padStart(8)}`,\n );\n }\n\n if (summary.timing) {\n lines.push(\n ` ${'Timing'.padEnd(maxNameLen)} ${((summary.timing.mean / 1000).toFixed(2) + ' \\u00b1 ' + (summary.timing.std / 1000).toFixed(2) + 's').padStart(colWidth)}`,\n );\n }\n\n const costStr = summary.totalCost > 0 ? `$${summary.totalCost.toFixed(2)}` : '$0.00';\n const durationStr = (summary.totalDuration / 1000).toFixed(1);\n lines.push('');\n lines.push(` Total Cost: ${costStr} \| Total Duration: ${durationStr}s`);\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime \|\| typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement\|Unexpected reserved word\|top-level await\|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' \|\| ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n runs: number;\n captureTraces: boolean;\n paths: string[];\n} {\n let outputPath: string \| undefined;\n let configArg: string \| undefined;\n let conditions: string[] = [];\n let runs = 1;\n let captureTraces = false;\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' \|\| arg === '--config' \|\| arg === '--conditions' \|\| arg === '--runs') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else if (arg === '--runs') runs = Math.max(1, parseInt(value, 10) \|\| 1);\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg === '--capture-traces') {\n // Boolean flag — no value consumed.\n captureTraces = true;\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, runs, captureTraces, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, runs, captureTraces, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow \|\| !evalConfig.dataset \|\| !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackExecution for cost + metadata attribution\n executeWorkflow = async (input, rt) => {\n const {\n result,\n cost: trackedCost,\n metadata,\n } = await runtime.trackExecution(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return {\n output: result.output,\n cost: result.cost ?? trackedCost,\n metadata: result.metadata ?? metadata,\n };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost, metadata } = await runtime.trackExecution(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost, metadata };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n // When --capture-traces is set, forward it to runEval so the runner\n // wraps each item's execution in a nested trackExecution({ captureTraces: true }).\n // This populates EvalItem.traces on both success (via tracked.traces)\n // and failure (via the axlCapturedTraces side-channel on the thrown\n // error). Nested trackExecution scopes walk the AsyncLocalStorage\n // parent chain, so the outer trackExecution above still observes\n // cost/metadata correctly.\n const runOptions = captureTraces ? { captureTraces: true } : undefined;\n\n if (runs > 1) {\n // Multi-run mode\n const { randomUUID } = await import('node:crypto');\n const runGroupId = randomUUID();\n const runResults: EvalResult[] = [];\n\n for (let r = 0; r < runs; r++) {\n console.error(`[axl-eval] Run ${r + 1}/${runs}...`);\n const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);\n result.metadata.runGroupId = runGroupId;\n result.metadata.runIndex = r;\n runResults.push(result);\n results.push(result);\n }\n\n const summary = aggregateRuns(runResults);\n console.log('\\n' + formatMultiRunTable(summary) + '\\n');\n } else {\n const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n }\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","/*\n CLI utilities for config detection, runtime resolution, and loader registration.\n \n These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n /\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string \| undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/* Returns true if the file is TypeScript and needs tsx to load. /\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/\n Resolve the AxlRuntime from a dynamically imported config module.\n \n Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n /\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> \| undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n \| ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n \| null\n \| undefined;\n\n/\n Import a module, using tsx's `tsImport()` for TypeScript files.\n \n `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;;;AAEA,SAAS,aAAa,gBAAgB;AACtC,SAAS,YAAY,eAAe,aAAa,gBAAgB,aAAa;AAC9E,YAAY,UAAU;;;ACItB,SAAS,eAAe;AACxB,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,IAAI,QAAQ,KAAK,IAAI;AAC3B,QAAI,WAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,WAAW,cAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,OAAO,cAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AD3FA,IAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAiBZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,SAAS,kBAAkB,MAA6D;AACtF,QAAM,MAAM,KAAK,QAAQ,aAAa;AACtC,MAAI,QAAQ,MAAM,MAAM,KAAK,KAAK,OAAQ,QAAO;AACjD,QAAM,QAAQ,KAAK,MAAM,CAAC;AAG1B,MAAI,WAAW,KAAK,KAAK,EAAG,QAAO,WAAW,KAAK;AAGnD,QAAM,MAA8B,CAAC;AACrC,aAAW,QAAQ,MAAM,MAAM,GAAG,GAAG;AACnC,UAAM,CAAC,KAAK,GAAG,IAAI,KAAK,MAAM,GAAG;AACjC,QAAI,OAAO,OAAO,KAAM,KAAI,IAAI,KAAK,CAAC,IAAI,WAAW,IAAI,KAAK,CAAC;AAAA,EACjE;AACA,SAAO,OAAO,KAAK,GAAG,EAAE,SAAS,IAAI,MAAM;AAC7C;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,aAAa,kBAAkB,IAAI;AACzC,QAAM,QAAQ,KAAK;AAAA,IACjB,CAAC,GAAG,MAAM,CAAC,EAAE,WAAW,IAAI,KAAK,EAAE,IAAI,KAAK,KAAK,IAAI,CAAC,MAAM;AAAA,EAC9D;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,gFAAgF;AAC9F,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,cAAc,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACrE,QAAM,eAAe,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AACtE,QAAM,WAAsC;AAC5C,QAAM,YAAuC;AAC7C,QAAM,iBAAiB,cAAc,OAAO,EAAE,WAAW,IAAI;AAC7D,QAAM,aAAa,YAAY,UAAU,WAAW,cAAc;AAElE,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAClE,QAAM,QAAQ,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,MAAM,IAAI;AAEtE,QAAM,WAAW,QAAQ,4BAA4B;AACrD,QAAM,SAAS,QAAQ,4BAA4B;AACnD,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,0CAA0C,QAAQ,EAAE;AAChG,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,0CAA0C,MAAM,EAAE;AAE7F,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,QAAI,OAAO,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AACjO,QAAI,OAAO;AACT,UAAI,EAAE,IAAI;AACR,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,GAAG,SAAS,IAAI,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC9D,cAAM,MAAM,EAAE,cAAc,QAAQ;AACpC,gBAAQ,MAAM,EAAE,KAAK,EAAE,IAAI,GAAG;AAAA,MAChC,OAAO;AACL,gBAAQ,WAAM,OAAO,EAAE;AAAA,MACzB;AAAA,IACF;AACA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAEA,MAAI,WAAW,QAAQ;AACrB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN;AAAA,sBAAyB,EAAE,eAAe,KAAM,QAAQ,CAAC,CAAC,mBAAmB,EAAE,gBAAgB,KAAM,QAAQ,CAAC,CAAC,MAAM,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACvJ;AAAA,EACF;AACA,MAAI,WAAW,MAAM;AACnB,UAAM,IAAI,WAAW;AACrB,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,qBAAqB,EAAE,cAAc,QAAQ,CAAC,CAAC,kBAAkB,EAAE,eAAe,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC;AAAA,IACnI;AAAA,EACF;AAEA,QAAM,cAAc,MAAM,QAAQ,QAAQ,IAAI,SAAS,CAAC,IAAI;AAC5D,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,YAAY,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EACrF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AAEzD,UAAM,kBAAkB,YAAY,KAAK,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,eAAe,IAAI;AACzF,QAAI,iBAAiB;AACnB,YAAM,2BAA2B,YAAY;AAAA,QAC3C,CAAC,MAAM,WAAW,QAAQ,CAAC,EAAE,gBAAgB,QAAQ,WAAW,QAAQ,CAAC,EAAE,QAAQ;AAAA,MACrF;AACA,UAAI,yBAA0B,SAAQ,KAAK,CAAC;AAAA,IAC9C,OAAO;AACL,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AACF;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,SAAS,GAAG;AACpB,YAAQ,MAAM,sEAAsE;AACpF,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,CAAC,aAAa,YAAY,IAAI;AACpC,QAAM,MAAM,KAAK,MAAM,MAAM,cAAc,aAAa,OAAO,CAAC;AAChE,QAAM,UAAwB,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,GAAG;AAE7D,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,MAAI;AACF,UAAM,MAAM,MAAM,aAAkB,aAAQ,YAAY,GAAG,YAAY,GAAG;AAC1E,UAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,QAAI,CAAC,WAAW,WAAW,WAAW,QAAQ,WAAW,GAAG;AAC1D,cAAQ,MAAM,UAAU,YAAY,0BAA0B;AAC9D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,UAAM,WAAyB,CAAC;AAChC,eAAW,cAAc,SAAS;AAChC,eAAS,KAAK,MAAM,QAAQ,YAAY,WAAW,SAAS,OAAO,CAAC;AAAA,IACtE;AAEA,eAAW,KAAK,UAAU;AACxB,cAAQ,IAAI,OAAO,YAAY,CAAC,IAAI,IAAI;AAAA,IAC1C;AAEA,QAAI,YAAY;AACd,YAAM,SAAS,MAAM,QAAQ,GAAG,IAAI,WAAW,SAAS,CAAC;AACzD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,6BAA6B,UAAU,EAAE;AAAA,IACvD;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,UAAU,YAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAGA,SAAS,gBAAgB,WAA4B;AACnD,MAAI,CAAC,MAAM,QAAQ,SAAS,KAAK,UAAU,WAAW,EAAG,QAAO;AAChE,SAAQ,UAAwB,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,EAAE,KAAK,IAAI;AAC7F;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM;AAAA,IACJ,SAAS,gBAAgB,OAAO,SAAS,SAAS,CAAC,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK;AAAA,EAClG;AACA,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AAErC,UAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAAE;AACvF,QAAI,oBAAoB,GAAG;AACzB,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,MAC1K;AAAA,IACF,OAAO;AACL,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,MACvO;AAAA,IACF;AAAA,EACF;AAEA,MAAI,OAAO,QAAQ,QAAQ;AACzB,UAAM,IAAI,OAAO,QAAQ;AACzB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,MAAM,EAAE,OAAO,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,MAAM,EAAE,MAAM,KAAM,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACtL;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,QAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,MAAM;AACzE,MAAI,gBAAgB,SAAS,GAAG;AAC9B,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,gBAAgB,QAAQ,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC;AACjF,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,oBAAoB,gBAAgB,MAAM,IAAI,OAAO,QAAQ,KAAK;AAAA,IACpE;AACA,eAAW,OAAO,aAAa,MAAM,GAAG,CAAC,GAAG;AAC1C,YAAM,KAAK,SAAS,GAAG,EAAE;AAAA,IAC3B;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,KAAK,eAAe,aAAa,SAAS,CAAC,OAAO;AAAA,IAC1D;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,SAAS,oBAAoB,SAAkC;AAC7D,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,QAAQ,OAAO;AAC/C,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM;AAAA,IACJ,SAAS,gBAAgB,QAAQ,SAAS,CAAC,MAAM,QAAQ,OAAO,WAAW,QAAQ,QAAQ;AAAA,EAC7F;AACA,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,gBAAkB,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC,KAAK,MAAM,SAAS,CAAC,CAAC;AAAA,EACvH;AACA,QAAM,UAAU,aAAa,IAAI,WAAW,IAAI,IAAI,IAAI;AACxD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,QAAQ,QAAQ,IAAI;AAC9B,UAAM,UAAU,GAAG,EAAE,KAAK,QAAQ,CAAC,CAAC,SAAW,EAAE,IAAI,QAAQ,CAAC,CAAC;AAC/D,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,QAAQ,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAAA,IAC/H;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ;AAClB,UAAM;AAAA,MACJ,KAAK,SAAS,OAAO,UAAU,CAAC,OAAO,QAAQ,OAAO,OAAO,KAAM,QAAQ,CAAC,IAAI,YAAc,QAAQ,OAAO,MAAM,KAAM,QAAQ,CAAC,IAAI,KAAK,SAAS,QAAQ,CAAC;AAAA,IAC/J;AAAA,EACF;AAEA,QAAM,UAAU,QAAQ,YAAY,IAAI,IAAI,QAAQ,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC7E,QAAM,eAAe,QAAQ,gBAAgB,KAAM,QAAQ,CAAC;AAC5D,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,iBAAiB,OAAO,sBAAsB,WAAW,GAAG;AAEvE,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,OAAO,SAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAOrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,MAAI,OAAO;AACX,MAAI,gBAAgB;AACpB,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,kBAAkB,QAAQ,UAAU;AAC1F,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA,eAChC,QAAQ,SAAU,QAAO,KAAK,IAAI,GAAG,SAAS,OAAO,EAAE,KAAK,CAAC;AAAA;AAEpE,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,QAAQ,oBAAoB;AAErC,sBAAgB;AAAA,IAClB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM,eAAe,MAAM;AACzE;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,eAAe,MAAM,IAAI,cAAc,IAAI;AAE5F,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM;AAAA,cACJ;AAAA,cACA,MAAM;AAAA,cACN;AAAA,YACF,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC3C,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO;AAAA,cACL,QAAQ,OAAO;AAAA,cACf,MAAM,OAAO,QAAQ;AAAA,cACrB,UAAU,OAAO,YAAY;AAAA,YAC/B;AAAA,UACF;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAQ,MAAM,SAAS,IAAI,MAAM,QAAQ,eAAe,YAAY;AAC1E,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQ,QAAQ,MAAM,SAAS;AAAA,UAC1C;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AASA,cAAM,aAAa,gBAAgB,EAAE,eAAe,KAAK,IAAI;AAE7D,YAAI,OAAO,GAAG;AAEZ,gBAAM,EAAE,WAAW,IAAI,MAAM,OAAO,QAAa;AACjD,gBAAM,aAAa,WAAW;AAC9B,gBAAM,aAA2B,CAAC;AAElC,mBAAS,IAAI,GAAG,IAAI,MAAM,KAAK;AAC7B,oBAAQ,MAAM,kBAAkB,IAAI,CAAC,IAAI,IAAI,KAAK;AAClD,kBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,SAAS,UAAU;AAC7E,mBAAO,SAAS,aAAa;AAC7B,mBAAO,SAAS,WAAW;AAC3B,uBAAW,KAAK,MAAM;AACtB,oBAAQ,KAAK,MAAM;AAAA,UACrB;AAEA,gBAAM,UAAU,cAAc,UAAU;AACxC,kBAAQ,IAAI,OAAO,oBAAoB,OAAO,IAAI,IAAI;AAAA,QACxD,OAAO;AACL,gBAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,SAAS,UAAU;AAC7E,kBAAQ,KAAK,MAAM;AAEnB,kBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,QAC/C;AAAA,MACF,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/index.cjs CHANGED Viewed

@@ -184,7 +184,36 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, runtime) {
+function extractUserCost(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const raw = result.cost;
+  if (raw === void 0) return void 0;
+  if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) return raw;
+  console.warn(
+    `[axl-eval] Ignoring invalid \`cost\` from ${label} return: expected non-negative finite number, got ${typeof raw === "number" ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`
+  );
+  return void 0;
+}
+function extractUserMetadata(result, label = "executeWorkflow") {
+  if (result === null || typeof result !== "object") return void 0;
+  const meta = result.metadata;
+  if (meta === void 0) return void 0;
+  if (meta === null || typeof meta !== "object" || Array.isArray(meta)) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${meta === null ? "null" : Array.isArray(meta) ? "array" : typeof meta}.`
+    );
+    return void 0;
+  }
+  const proto = Object.getPrototypeOf(meta);
+  if (proto !== Object.prototype && proto !== null) {
+    console.warn(
+      `[axl-eval] Ignoring invalid \`metadata\` from ${label} return: expected plain object, got ${proto?.constructor?.name ?? "exotic object"}.`
+    );
+    return void 0;
+  }
+  return meta;
+}
+async function runEval(config, executeWorkflow, runtime, options) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
@@ -204,6 +233,17 @@ async function runEval(config, executeWorkflow, runtime) {
   let totalCost = 0;
   let budgetExceeded = false;
   async function processItem(item, itemIndex) {
+    if (options?.signal?.aborted) {
+      evalItems[itemIndex] = {
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Cancelled",
+        scores: {}
+      };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
+      return;
+    }
     if (budgetExceeded) {
       evalItems[itemIndex] = {
         input: item.input,
@@ -212,6 +252,7 @@ async function runEval(config, executeWorkflow, runtime) {
         error: "Budget exceeded",
         scores: {}
       };
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     const evalItem = {
@@ -222,18 +263,43 @@ async function runEval(config, executeWorkflow, runtime) {
     };
     const itemStart = Date.now();
     try {
-      const result = await executeWorkflow(item.input, runtime);
-      evalItem.duration = Date.now() - itemStart;
-      evalItem.output = result.output;
-      evalItem.cost = result.cost;
-      if (result.metadata) evalItem.metadata = result.metadata;
-      if (result.cost != null) {
-        totalCost += result.cost;
+      if (options?.captureTraces) {
+        const tracked = await runtime.trackExecution(
+          async () => executeWorkflow(item.input, runtime),
+          { captureTraces: true }
+        );
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = tracked.result.output;
+        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;
+        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;
+        if (tracked.traces && tracked.traces.length > 0) {
+          evalItem.traces = tracked.traces;
+        }
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
+      } else {
+        const result = await executeWorkflow(item.input, runtime);
+        evalItem.duration = Date.now() - itemStart;
+        evalItem.output = result.output;
+        evalItem.cost = extractUserCost(result);
+        const meta = extractUserMetadata(result);
+        if (meta) evalItem.metadata = meta;
+        if (evalItem.cost != null) {
+          totalCost += evalItem.cost;
+        }
       }
     } catch (err) {
       evalItem.duration = Date.now() - itemStart;
       evalItem.error = err instanceof Error ? err.message : String(err);
+      if (options?.captureTraces) {
+        const captured = err.axlCapturedTraces;
+        if (Array.isArray(captured) && captured.length > 0) {
+          evalItem.traces = captured;
+        }
+      }
       evalItems[itemIndex] = evalItem;
+      options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
       return;
     }
     if (budgetLimit != null && totalCost > budgetLimit) {
@@ -242,6 +308,7 @@ async function runEval(config, executeWorkflow, runtime) {
     evalItem.scoreDetails = {};
     let itemScorerCost = 0;
     for (const scorer2 of config.scorers) {
+      if (options?.signal?.aborted) break;
       const scorerStart = Date.now();
       try {
         const raw = await scorer2.score(
@@ -297,6 +364,7 @@ async function runEval(config, executeWorkflow, runtime) {
     }
     evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : void 0;
     evalItems[itemIndex] = evalItem;
+    options?.onProgress?.({ type: "item_done", itemIndex, totalItems: items.length });
   }
   let index = 0;
   async function runNext() {
@@ -343,14 +411,42 @@ async function runEval(config, executeWorkflow, runtime) {
     modelsMeta.models = sorted.map(([m]) => m);
     modelsMeta.modelCounts = Object.fromEntries(sorted);
   }
+  const totalWorkflowCalls = /* @__PURE__ */ new Map();
+  for (const item of evalItems) {
+    const itemCounts = item.metadata?.workflowCallCounts;
+    if (itemCounts && typeof itemCounts === "object") {
+      for (const [w, count] of Object.entries(itemCounts)) {
+        if (typeof count === "number")
+          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);
+      }
+    } else {
+      const itemWorkflows = item.metadata?.workflows;
+      if (Array.isArray(itemWorkflows)) {
+        for (const w of itemWorkflows) {
+          if (typeof w === "string")
+            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);
+        }
+      }
+    }
+  }
+  const workflowsMeta = {};
+  if (totalWorkflowCalls.size > 0) {
+    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);
+    workflowsMeta.workflows = sorted.map(([w]) => w);
+    workflowsMeta.workflowCounts = Object.fromEntries(sorted);
+  } else if (config.workflow) {
+    workflowsMeta.workflows = [config.workflow];
+    workflowsMeta.workflowCounts = { [config.workflow]: items.length };
+  }
+  options?.onProgress?.({ type: "run_done", totalItems: items.length, failures });
   return {
     id,
-    workflow: config.workflow,
     dataset: config.dataset.name,
     metadata: {
       ...config.metadata,
       scorerTypes,
-      ...modelsMeta
+      ...modelsMeta,
+      ...workflowsMeta
     },
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
@@ -649,6 +745,7 @@ async function rescore(result, scorers, runtime, options) {
       annotations: original.annotations,
       output: original.output,
       metadata: original.metadata,
+      traces: original.traces,
       scores: {},
       scoreDetails: {}
     };
@@ -730,11 +827,23 @@ async function rescore(result, scorers, runtime, options) {
   }
   return {
     id: (0, import_node_crypto2.randomUUID)(),
-    workflow: result.workflow,
     dataset: result.dataset,
     metadata: (() => {
       const { runGroupId: _, runIndex: __, ...rest } = result.metadata;
-      return { ...rest, rescored: true, originalId: result.id, scorerTypes };
+      const merged = {
+        ...rest,
+        rescored: true,
+        originalId: result.id,
+        scorerTypes
+      };
+      if (!Array.isArray(merged.workflows)) {
+        const legacyWorkflow = result.workflow;
+        if (typeof legacyWorkflow === "string" && legacyWorkflow) {
+          merged.workflows = [legacyWorkflow];
+          merged.workflowCounts = { [legacyWorkflow]: result.items.length };
+        }
+      }
+      return merged;
     })(),
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     totalCost,
@@ -754,7 +863,20 @@ function std(values, mean) {
 function aggregateRuns(runs) {
   if (runs.length === 0) throw new Error("Cannot aggregate zero runs");
   const runGroupId = runs[0].metadata.runGroupId ?? (0, import_node_crypto3.randomUUID)();
-  const { workflow, dataset: dataset2 } = runs[0];
+  const { dataset: dataset2 } = runs[0];
+  const seenWorkflows = /* @__PURE__ */ new Set();
+  const workflows = [];
+  for (const run of runs) {
+    const list = run.metadata.workflows;
+    if (Array.isArray(list)) {
+      for (const w of list) {
+        if (typeof w === "string" && !seenWorkflows.has(w)) {
+          seenWorkflows.add(w);
+          workflows.push(w);
+        }
+      }
+    }
+  }
   const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);
   const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);
   const scorerNames = Object.keys(runs[0].summary.scorers);
@@ -783,7 +905,7 @@ function aggregateRuns(runs) {
   return {
     runGroupId,
     runCount: runs.length,
-    workflow,
+    workflows,
     dataset: dataset2,
     totalCost: round(totalCost),
     totalDuration: round(totalDuration),