@axlsdk/eval 0.11.1 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1 -1
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.js +1 -1
- package/dist/cli.js.map +1 -1
- package/package.json +2 -2
package/dist/cli.cjs
CHANGED
|
@@ -498,7 +498,7 @@ async function runEvalCommand(args) {
|
|
|
498
498
|
for (const filePath of evalFiles) {
|
|
499
499
|
try {
|
|
500
500
|
const mod = await importModule(path.resolve(filePath), import_meta.url);
|
|
501
|
-
const evalConfig = mod.default ?? mod.config ?? mod;
|
|
501
|
+
const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
|
|
502
502
|
if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
|
|
503
503
|
console.error(
|
|
504
504
|
`Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
|
package/dist/cli.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set(['--output', '--config', '--conditions', '--fail-on-regression']);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const files = args.filter((a) => !a.startsWith('--'));\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json>');\n process.exit(1);\n }\n\n const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const comparison = evalCompare(baseline, candidate);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n console.log(\n ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n );\n }\n\n const stable = Math.max(\n 0,\n baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n process.exit(1);\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n );\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime || typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' || ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n paths: string[];\n} {\n let outputPath: string | undefined;\n let configArg: string | undefined;\n let conditions: string[] = [];\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' || arg === '--config' || arg === '--conditions') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackCost for automatic cost attribution\n executeWorkflow = async (input, rt) => {\n const { result, cost: trackedCost } = await runtime.trackCost(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return { output: result.output, cost: result.cost ?? trackedCost };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost } = await runtime.trackCost(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n const result = await runEval(evalConfig, executeWorkflow, mod.provider, runtime);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n if (baseline.dataset !== candidate.dataset) {\n throw new Error(\n `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n );\n }\n\n const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n throw new Error(`Cannot compare evals with different scorers`);\n }\n\n const scorers: EvalComparison['scorers'] = {};\n for (const name of baselineScorerNames) {\n const bMean = baseline.summary.scorers[name].mean;\n const cMean = candidate.summary.scorers[name].mean;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n }\n\n const regressions: EvalRegression[] = [];\n const improvements: EvalImprovement[] = [];\n\n const minLength = Math.min(baseline.items.length, candidate.items.length);\n for (let i = 0; i < minLength; i++) {\n const bItem = baseline.items[i];\n const cItem = candidate.items[i];\n if (bItem.error || cItem.error) continue;\n for (const name of baselineScorerNames) {\n const bScore = bItem.scores[name];\n const cScore = cItem.scores[name];\n if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n const delta = round(cScore - bScore);\n if (delta < -0.1)\n regressions.push({\n input: bItem.input,\n scorer: name,\n baselineScore: bScore,\n candidateScore: cScore,\n delta,\n });\n else if (delta > 0.1)\n improvements.push({\n input: bItem.input,\n scorer: name,\n baselineScore: bScore,\n candidateScore: cScore,\n delta,\n });\n }\n }\n\n const parts: string[] = [];\n for (const name of baselineScorerNames) {\n const s = scorers[name];\n if (Math.abs(s.delta) > 0.001) {\n const direction = s.delta > 0 ? 'improves' : 'regresses';\n const sign = s.delta > 0 ? '+' : '';\n parts.push(\n `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n );\n }\n }\n const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n return {\n baseline: { id: baseline.id, metadata: baseline.metadata },\n candidate: { id: candidate.id, metadata: candidate.metadata },\n scorers,\n regressions,\n improvements,\n summary: summaryStr,\n };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n mean: number;\n min: number;\n max: number;\n p50: number;\n p95: number;\n} {\n if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n const sorted = [...scores].sort((a, b) => a - b);\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n const min = sorted[0];\n const max = sorted[sorted.length - 1];\n const p50 = sorted[Math.floor(sorted.length * 0.5)];\n const p95 = sorted[Math.floor(sorted.length * 0.95)];\n return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n const match = cost.match(/^\\$?([\\d.]+)$/);\n if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n chat(\n messages: { role: string; content: string }[],\n options: { model: string; temperature?: number },\n ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n config: EvalConfig,\n executeWorkflow: (\n input: unknown,\n runtime: AxlRuntime,\n ) => Promise<{ output: unknown; cost?: number }>,\n provider: LlmScorerProvider | undefined,\n runtime: AxlRuntime,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const id = randomUUID();\n const items = await config.dataset.getItems();\n const concurrency = config.concurrency ?? 5;\n const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n if (provider) {\n for (const scorer of config.scorers) {\n if (scorer.isLlm) {\n (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n }\n }\n }\n\n const evalItems: EvalItem[] = [];\n let totalCost = 0;\n let budgetExceeded = false;\n\n async function processItem(item: (typeof items)[0]): Promise<void> {\n if (budgetExceeded) {\n evalItems.push({\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Budget exceeded',\n scores: {},\n });\n return;\n }\n\n const evalItem: EvalItem = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n scores: {},\n };\n try {\n const result = await executeWorkflow(item.input, runtime);\n evalItem.output = result.output;\n if (result.cost != null) {\n totalCost += result.cost;\n }\n } catch (err) {\n evalItem.error = err instanceof Error ? err.message : String(err);\n evalItems.push(evalItem);\n return;\n }\n\n if (budgetLimit != null && totalCost > budgetLimit) {\n budgetExceeded = true;\n }\n\n for (const scorer of config.scorers) {\n try {\n const score = await scorer.score(evalItem.output, item.input, item.annotations);\n if (score < 0 || score > 1) {\n if (!evalItem.errors) evalItem.errors = [];\n evalItem.errors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n );\n evalItem.scores[scorer.name] = -1;\n } else {\n evalItem.scores[scorer.name] = round(score);\n }\n } catch (err) {\n if (!evalItem.errors) evalItem.errors = [];\n evalItem.errors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n evalItem.scores[scorer.name] = -1;\n }\n }\n evalItems.push(evalItem);\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < items.length) {\n const currentIndex = index++;\n await processItem(items[currentIndex]);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n await Promise.all(workers);\n\n const failures = evalItems.filter((i) => i.error).length;\n const scorerNames = config.scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = evalItems\n .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n .map((i) => i.scores[name]);\n scorerStats[name] = computeStats(scores);\n }\n\n return {\n id,\n workflow: config.workflow,\n dataset: config.dataset.name,\n metadata: config.metadata ?? {},\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: evalItems,\n summary: { count: items.length, failures, scorers: scorerStats },\n };\n}\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> | undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n | null\n | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,kBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC/EA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBAIA,UACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;ACnJA,uBAAwB;AACxB,qBAA2B;AAC3B,sBAA8B;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,QAAI,0BAAQ,KAAK,IAAI;AAC3B,YAAI,2BAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,eAAW,+BAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,WAAO,+BAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AH/GA;AAiBA,IAAM,cAAc,oBAAI,IAAI,CAAC,YAAY,YAAY,gBAAgB,sBAAsB,CAAC;AAE5F,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,0DAA0D;AACxE,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,0BAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,6BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,WAAO,0BAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAKrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,gBAAgB;AACtE,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA;AAEvC,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM;AACpD;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,WAAW,IAAI,UAAU;AAE5D,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM,EAAE,QAAAC,SAAQ,MAAM,YAAY,IAAI,MAAM,QAAQ,UAAU,YAAY;AACxE,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO,EAAE,QAAQA,QAAO,QAAQ,MAAMA,QAAO,QAAQ,YAAY;AAAA,UACnE;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAAA,SAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU,YAAY;AAC3D,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQA,SAAQ,KAAK;AAAA,UAChC;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,cAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,UAAU,OAAO;AAC/E,gBAAQ,KAAK,MAAM;AAEnB,gBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,MAC/C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["import_node_fs","round","readFileAsync","result","writeFileAsync"]}
|
|
1
|
+
{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set(['--output', '--config', '--conditions', '--fail-on-regression']);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const files = args.filter((a) => !a.startsWith('--'));\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json>');\n process.exit(1);\n }\n\n const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const comparison = evalCompare(baseline, candidate);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n console.log(\n ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n );\n }\n\n const stable = Math.max(\n 0,\n baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n process.exit(1);\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n );\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime || typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' || ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n paths: string[];\n} {\n let outputPath: string | undefined;\n let configArg: string | undefined;\n let conditions: string[] = [];\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' || arg === '--config' || arg === '--conditions') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackCost for automatic cost attribution\n executeWorkflow = async (input, rt) => {\n const { result, cost: trackedCost } = await runtime.trackCost(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return { output: result.output, cost: result.cost ?? trackedCost };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost } = await runtime.trackCost(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n const result = await runEval(evalConfig, executeWorkflow, mod.provider, runtime);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n if (baseline.dataset !== candidate.dataset) {\n throw new Error(\n `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n );\n }\n\n const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n throw new Error(`Cannot compare evals with different scorers`);\n }\n\n const scorers: EvalComparison['scorers'] = {};\n for (const name of baselineScorerNames) {\n const bMean = baseline.summary.scorers[name].mean;\n const cMean = candidate.summary.scorers[name].mean;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n }\n\n const regressions: EvalRegression[] = [];\n const improvements: EvalImprovement[] = [];\n\n const minLength = Math.min(baseline.items.length, candidate.items.length);\n for (let i = 0; i < minLength; i++) {\n const bItem = baseline.items[i];\n const cItem = candidate.items[i];\n if (bItem.error || cItem.error) continue;\n for (const name of baselineScorerNames) {\n const bScore = bItem.scores[name];\n const cScore = cItem.scores[name];\n if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n const delta = round(cScore - bScore);\n if (delta < -0.1)\n regressions.push({\n input: bItem.input,\n scorer: name,\n baselineScore: bScore,\n candidateScore: cScore,\n delta,\n });\n else if (delta > 0.1)\n improvements.push({\n input: bItem.input,\n scorer: name,\n baselineScore: bScore,\n candidateScore: cScore,\n delta,\n });\n }\n }\n\n const parts: string[] = [];\n for (const name of baselineScorerNames) {\n const s = scorers[name];\n if (Math.abs(s.delta) > 0.001) {\n const direction = s.delta > 0 ? 'improves' : 'regresses';\n const sign = s.delta > 0 ? '+' : '';\n parts.push(\n `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n );\n }\n }\n const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n return {\n baseline: { id: baseline.id, metadata: baseline.metadata },\n candidate: { id: candidate.id, metadata: candidate.metadata },\n scorers,\n regressions,\n improvements,\n summary: summaryStr,\n };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n mean: number;\n min: number;\n max: number;\n p50: number;\n p95: number;\n} {\n if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n const sorted = [...scores].sort((a, b) => a - b);\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n const min = sorted[0];\n const max = sorted[sorted.length - 1];\n const p50 = sorted[Math.floor(sorted.length * 0.5)];\n const p95 = sorted[Math.floor(sorted.length * 0.95)];\n return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n const match = cost.match(/^\\$?([\\d.]+)$/);\n if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n chat(\n messages: { role: string; content: string }[],\n options: { model: string; temperature?: number },\n ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n config: EvalConfig,\n executeWorkflow: (\n input: unknown,\n runtime: AxlRuntime,\n ) => Promise<{ output: unknown; cost?: number }>,\n provider: LlmScorerProvider | undefined,\n runtime: AxlRuntime,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const id = randomUUID();\n const items = await config.dataset.getItems();\n const concurrency = config.concurrency ?? 5;\n const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n if (provider) {\n for (const scorer of config.scorers) {\n if (scorer.isLlm) {\n (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n }\n }\n }\n\n const evalItems: EvalItem[] = [];\n let totalCost = 0;\n let budgetExceeded = false;\n\n async function processItem(item: (typeof items)[0]): Promise<void> {\n if (budgetExceeded) {\n evalItems.push({\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Budget exceeded',\n scores: {},\n });\n return;\n }\n\n const evalItem: EvalItem = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n scores: {},\n };\n try {\n const result = await executeWorkflow(item.input, runtime);\n evalItem.output = result.output;\n if (result.cost != null) {\n totalCost += result.cost;\n }\n } catch (err) {\n evalItem.error = err instanceof Error ? err.message : String(err);\n evalItems.push(evalItem);\n return;\n }\n\n if (budgetLimit != null && totalCost > budgetLimit) {\n budgetExceeded = true;\n }\n\n for (const scorer of config.scorers) {\n try {\n const score = await scorer.score(evalItem.output, item.input, item.annotations);\n if (score < 0 || score > 1) {\n if (!evalItem.errors) evalItem.errors = [];\n evalItem.errors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n );\n evalItem.scores[scorer.name] = -1;\n } else {\n evalItem.scores[scorer.name] = round(score);\n }\n } catch (err) {\n if (!evalItem.errors) evalItem.errors = [];\n evalItem.errors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n evalItem.scores[scorer.name] = -1;\n }\n }\n evalItems.push(evalItem);\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < items.length) {\n const currentIndex = index++;\n await processItem(items[currentIndex]);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n await Promise.all(workers);\n\n const failures = evalItems.filter((i) => i.error).length;\n const scorerNames = config.scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = evalItems\n .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n .map((i) => i.scores[name]);\n scorerStats[name] = computeStats(scores);\n }\n\n return {\n id,\n workflow: config.workflow,\n dataset: config.dataset.name,\n metadata: config.metadata ?? {},\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: evalItems,\n summary: { count: items.length, failures, scorers: scorerStats },\n };\n}\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> | undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n | null\n | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,kBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC/EA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBAIA,UACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;ACnJA,uBAAwB;AACxB,qBAA2B;AAC3B,sBAA8B;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,QAAI,0BAAQ,KAAK,IAAI;AAC3B,YAAI,2BAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,eAAW,+BAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,WAAO,+BAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AH/GA;AAiBA,IAAM,cAAc,oBAAI,IAAI,CAAC,YAAY,YAAY,gBAAgB,sBAAsB,CAAC;AAE5F,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,0DAA0D;AACxE,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,0BAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,6BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,WAAO,0BAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAKrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,gBAAgB;AACtE,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA;AAEvC,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM;AACpD;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM,EAAE,QAAAC,SAAQ,MAAM,YAAY,IAAI,MAAM,QAAQ,UAAU,YAAY;AACxE,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO,EAAE,QAAQA,QAAO,QAAQ,MAAMA,QAAO,QAAQ,YAAY;AAAA,UACnE;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAAA,SAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU,YAAY;AAC3D,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQA,SAAQ,KAAK;AAAA,UAChC;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,cAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,UAAU,OAAO;AAC/E,gBAAQ,KAAK,MAAM;AAEnB,gBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,MAC/C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["import_node_fs","round","readFileAsync","result","writeFileAsync"]}
|
package/dist/cli.js
CHANGED
|
@@ -285,7 +285,7 @@ async function runEvalCommand(args) {
|
|
|
285
285
|
for (const filePath of evalFiles) {
|
|
286
286
|
try {
|
|
287
287
|
const mod = await importModule(path.resolve(filePath), import.meta.url);
|
|
288
|
-
const evalConfig = mod.default ?? mod.config ?? mod;
|
|
288
|
+
const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
|
|
289
289
|
if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
|
|
290
290
|
console.error(
|
|
291
291
|
`Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cli.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set(['--output', '--config', '--conditions', '--fail-on-regression']);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const files = args.filter((a) => !a.startsWith('--'));\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json>');\n process.exit(1);\n }\n\n const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const comparison = evalCompare(baseline, candidate);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n console.log(\n ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n );\n }\n\n const stable = Math.max(\n 0,\n baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n process.exit(1);\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n );\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime || typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' || ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n paths: string[];\n} {\n let outputPath: string | undefined;\n let configArg: string | undefined;\n let conditions: string[] = [];\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' || arg === '--config' || arg === '--conditions') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackCost for automatic cost attribution\n executeWorkflow = async (input, rt) => {\n const { result, cost: trackedCost } = await runtime.trackCost(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return { output: result.output, cost: result.cost ?? trackedCost };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost } = await runtime.trackCost(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n const result = await runEval(evalConfig, executeWorkflow, mod.provider, runtime);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> | undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n | null\n | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;AAEA,SAAS,aAAa,gBAAgB;AACtC,SAAS,YAAY,eAAe,aAAa,gBAAgB,aAAa;AAC9E,YAAY,UAAU;;;ACItB,SAAS,eAAe;AACxB,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,IAAI,QAAQ,KAAK,IAAI;AAC3B,QAAI,WAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,WAAW,cAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,OAAO,cAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AD9FA,IAAM,cAAc,oBAAI,IAAI,CAAC,YAAY,YAAY,gBAAgB,sBAAsB,CAAC;AAE5F,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,0DAA0D;AACxE,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,UAAU,YAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,OAAO,SAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAKrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,gBAAgB;AACtE,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA;AAEvC,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM;AACpD;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,WAAW,IAAI,UAAU;AAE5D,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM,EAAE,QAAAA,SAAQ,MAAM,YAAY,IAAI,MAAM,QAAQ,UAAU,YAAY;AACxE,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO,EAAE,QAAQA,QAAO,QAAQ,MAAMA,QAAO,QAAQ,YAAY;AAAA,UACnE;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAAA,SAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU,YAAY;AAC3D,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQA,SAAQ,KAAK;AAAA,UAChC;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,cAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,UAAU,OAAO;AAC/E,gBAAQ,KAAK,MAAM;AAEnB,gBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,MAC/C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["result"]}
|
|
1
|
+
{"version":3,"sources":["../src/cli.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n findConfig,\n resolveRuntime,\n importModule,\n registerConditions,\n CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set(['--output', '--config', '--conditions', '--fail-on-regression']);\n\nasync function main() {\n const args = process.argv.slice(2);\n\n if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n console.log(`\nUsage:\n axl-eval <path> Run eval file(s)\n axl-eval <path> --output <file> Save results to JSON\n axl-eval <path> --config <file> Use config file for runtime\n axl-eval <path> --conditions <list> Node.js import conditions (comma-separated)\n axl-eval compare <a> <b> Compare two eval result files\n axl-eval compare <a> <b> --fail-on-regression Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n process.exit(0);\n }\n\n if (args[0] === 'compare') {\n await runCompare(args.slice(1));\n return;\n }\n\n await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n const failOnRegression = args.includes('--fail-on-regression');\n const files = args.filter((a) => !a.startsWith('--'));\n\n if (files.length !== 2) {\n console.error('Usage: axl-eval compare <baseline.json> <candidate.json>');\n process.exit(1);\n }\n\n const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n const comparison = evalCompare(baseline, candidate);\n\n console.log(\n `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n );\n\n const scorerNames = Object.keys(comparison.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n console.log(` ${'Scorer'.padEnd(maxNameLen)} Baseline Candidate Delta Change`);\n console.log(` ${''.padEnd(maxNameLen, '-')} -------- --------- -------- ------`);\n\n for (const name of scorerNames) {\n const s = comparison.scorers[name];\n const sign = s.delta > 0 ? '+' : '';\n console.log(\n ` ${name.padEnd(maxNameLen)} ${s.baselineMean.toFixed(3).padStart(8)} ${s.candidateMean.toFixed(3).padStart(9)} ${(sign + s.delta.toFixed(3)).padStart(8)} ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n );\n }\n\n const stable = Math.max(\n 0,\n baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n );\n console.log(\n `\\n Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n );\n\n if (failOnRegression && comparison.regressions.length > 0) {\n process.exit(1);\n }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n const resolved = path.resolve(p);\n try {\n const stat = statSync(resolved);\n if (stat.isDirectory()) {\n const entries = readdirSync(resolved);\n return entries\n .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n .map((e) => path.join(resolved, e));\n }\n return [resolved];\n } catch {\n return [resolved];\n }\n}\n\nfunction formatTable(result: EvalResult): string {\n const lines: string[] = [];\n const scorerNames = Object.keys(result.summary.scorers);\n const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n const colWidth = 8;\n\n lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n lines.push(\n ` ${'Scorer'.padEnd(maxNameLen)} ${'Mean'.padStart(colWidth)} ${'Min'.padStart(colWidth)} ${'Max'.padStart(colWidth)} ${'p50'.padStart(colWidth)} ${'p95'.padStart(colWidth)}`,\n );\n\n const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n lines.push(' ' + '\\u2500'.repeat(ruleLen));\n\n for (const name of scorerNames) {\n const s = result.summary.scorers[name];\n lines.push(\n ` ${name.padEnd(maxNameLen)} ${s.mean.toFixed(2).padStart(colWidth)} ${s.min.toFixed(2).padStart(colWidth)} ${s.max.toFixed(2).padStart(colWidth)} ${s.p50.toFixed(2).padStart(colWidth)} ${s.p95.toFixed(2).padStart(colWidth)}`,\n );\n }\n\n const durationSec = (result.duration / 1000).toFixed(1);\n const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n lines.push('');\n lines.push(\n ` Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n );\n\n return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n try {\n const mod = await importModule(configPath, import.meta.url);\n const runtime = resolveRuntime(mod) as AxlRuntime;\n\n if (!runtime || typeof runtime.execute !== 'function') {\n console.error(`Config must export a default AxlRuntime instance.`);\n if (runtime) {\n const keys = Object.keys(runtime as object)\n .slice(0, 5)\n .join(', ');\n console.error(` Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n }\n console.error(\n `Example:\\n import { AxlRuntime } from '@axlsdk/axl';\\n export default new AxlRuntime({ ... });`,\n );\n process.exit(1);\n }\n\n return runtime;\n } catch (err) {\n const msg = err instanceof Error ? err.message : String(err);\n if (\n /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n msg,\n )\n ) {\n const ext = path.extname(configPath);\n console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n if (ext === '.ts' || ext === '.tsx') {\n console.error(\n ` Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n );\n } else {\n console.error(` Tip: add \"type\": \"module\" to your package.json.`);\n }\n console.error();\n }\n console.error(`Failed to load config:`, err);\n process.exit(1);\n }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n // Register import conditions before any config loading\n if (conditions && conditions.length > 0) {\n await registerConditions(conditions);\n }\n\n // 1. Explicit --config\n if (configArg) {\n const configPath = path.resolve(process.cwd(), configArg);\n const stat = statSync(configPath, { throwIfNoEntry: false });\n if (!stat?.isFile()) {\n console.error(`Config file not found: ${configPath}`);\n process.exit(1);\n }\n console.error(`[axl-eval] Loading config from ${configPath}`);\n return resolveRuntimeFromConfig(configPath);\n }\n\n // 2. Auto-detect\n const found = findConfig(process.cwd());\n if (found) {\n console.error(`[axl-eval] Auto-detected config: ${found}`);\n return resolveRuntimeFromConfig(found);\n }\n\n // 3. Bare runtime (providers from env vars)\n const { AxlRuntime } = await import('@axlsdk/axl');\n return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n outputPath?: string;\n configArg?: string;\n conditions: string[];\n paths: string[];\n} {\n let outputPath: string | undefined;\n let configArg: string | undefined;\n let conditions: string[] = [];\n const paths: string[] = [];\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (arg === '--output' || arg === '--config' || arg === '--conditions') {\n if (i + 1 >= args.length) {\n console.error(`Error: ${arg} requires a value`);\n process.exit(1);\n }\n const value = args[++i];\n if (arg === '--output') outputPath = value;\n else if (arg === '--config') configArg = value;\n else\n conditions = value\n .split(',')\n .map((c) => c.trim())\n .filter(Boolean);\n } else if (arg.startsWith('--')) {\n if (!KNOWN_FLAGS.has(arg)) {\n console.error(`Unknown flag: ${arg}`);\n process.exit(1);\n }\n } else {\n paths.push(arg);\n }\n }\n\n return { outputPath, configArg, conditions, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n if (paths.length === 0) {\n console.error('Error: No eval file path provided');\n process.exit(1);\n }\n\n const evalFiles: string[] = [];\n for (const p of paths) {\n evalFiles.push(...collectEvalFiles(p));\n }\n\n if (evalFiles.length === 0) {\n console.error('Error: No eval files found');\n process.exit(1);\n }\n\n const runtime = await getRuntime(configArg, conditions);\n const results: EvalResult[] = [];\n\n try {\n for (const filePath of evalFiles) {\n try {\n const mod = await importModule(path.resolve(filePath), import.meta.url);\n const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n console.error(\n `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n );\n continue;\n }\n\n // Resolve executeWorkflow: custom export > registered workflow > passthrough\n let executeWorkflow: (\n input: unknown,\n rt?: unknown,\n ) => Promise<{ output: unknown; cost?: number }>;\n\n if (mod.executeWorkflow) {\n // Wrap custom executeWorkflow with trackCost for automatic cost attribution\n executeWorkflow = async (input, rt) => {\n const { result, cost: trackedCost } = await runtime.trackCost(async () => {\n return mod.executeWorkflow(input, rt);\n });\n return { output: result.output, cost: result.cost ?? trackedCost };\n };\n } else if (runtime.getWorkflow(evalConfig.workflow)) {\n // No executeWorkflow exported but workflow is registered — use runtime.execute()\n executeWorkflow = async (input) => {\n const { result, cost } = await runtime.trackCost(async () => {\n return runtime.execute(evalConfig.workflow, input);\n });\n return { output: result, cost };\n };\n } else {\n console.warn(\n `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n );\n executeWorkflow = async (input) => ({ output: input });\n }\n\n const result = await runEval(evalConfig, executeWorkflow, mod.provider, runtime);\n results.push(result);\n\n console.log('\\n' + formatTable(result) + '\\n');\n } catch (err) {\n console.error(\n `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n\n if (outputPath && results.length > 0) {\n const output = results.length === 1 ? results[0] : results;\n const outputDir = path.dirname(path.resolve(outputPath));\n await mkdir(outputDir, { recursive: true });\n await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n console.log(`Results saved to ${outputPath}`);\n }\n } finally {\n await runtime.shutdown().catch(() => {});\n }\n\n // Exit with non-zero code if no evals succeeded\n if (results.length === 0) {\n process.exit(1);\n }\n}\n\nmain().catch((err) => {\n console.error(err);\n process.exit(1);\n});\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n 'axl.config.mts',\n 'axl.config.ts',\n 'axl.config.mjs',\n 'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n for (const name of CONFIG_CANDIDATES) {\n const p = resolve(cwd, name);\n if (existsSync(p)) return p;\n }\n return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n const def = mod.default as Record<string, any> | undefined;\n return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n | null\n | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n filePath: string,\n parentURL: string,\n): Promise<Record<string, any>> {\n if (needsTsxLoader(filePath)) {\n if (tsImportFn === undefined) {\n try {\n // @ts-expect-error — tsx is an optional runtime dependency\n const mod = await import('tsx/esm/api');\n tsImportFn = mod.tsImport ?? null;\n } catch {\n tsImportFn = null;\n console.warn(\n '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n ' Install it with: npm install -D tsx',\n );\n }\n }\n if (tsImportFn) {\n return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n }\n }\n return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n try {\n const nodeModule = await import('node:module');\n const hookCode = [\n `const extra = ${JSON.stringify(conditions)};`,\n `export async function resolve(specifier, context, nextResolve) {`,\n ` return nextResolve(specifier, {`,\n ` ...context,`,\n ` conditions: [...new Set([...context.conditions, ...extra])],`,\n ` });`,\n `}`,\n ].join('\\n');\n nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n } catch {\n console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n }\n}\n"],"mappings":";;;;;;;AAEA,SAAS,aAAa,gBAAgB;AACtC,SAAS,YAAY,eAAe,aAAa,gBAAgB,aAAa;AAC9E,YAAY,UAAU;;;ACItB,SAAS,eAAe;AACxB,SAAS,kBAAkB;AAC3B,SAAS,qBAAqB;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,IAAI,QAAQ,KAAK,IAAI;AAC3B,QAAI,WAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,WAAW,cAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,OAAO,cAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AD9FA,IAAM,cAAc,oBAAI,IAAI,CAAC,YAAY,YAAY,gBAAgB,sBAAsB,CAAC;AAE5F,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,0DAA0D;AACxE,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,UAAU,YAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,OAAO,SAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAKrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,gBAAgB;AACtE,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA;AAEvC,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM;AACpD;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM,EAAE,QAAAA,SAAQ,MAAM,YAAY,IAAI,MAAM,QAAQ,UAAU,YAAY;AACxE,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO,EAAE,QAAQA,QAAO,QAAQ,MAAMA,QAAO,QAAQ,YAAY;AAAA,UACnE;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAAA,SAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU,YAAY;AAC3D,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQA,SAAQ,KAAK;AAAA,UAChC;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,cAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,UAAU,OAAO;AAC/E,gBAAQ,KAAK,MAAM;AAEnB,gBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,MAC/C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,YAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,YAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["result"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@axlsdk/eval",
|
|
3
|
-
"version": "0.11.
|
|
3
|
+
"version": "0.11.3",
|
|
4
4
|
"description": "Evaluation framework for Axl agentic workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.cjs",
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
"dist"
|
|
26
26
|
],
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@axlsdk/axl": "0.11.
|
|
28
|
+
"@axlsdk/axl": "0.11.3"
|
|
29
29
|
},
|
|
30
30
|
"devDependencies": {
|
|
31
31
|
"@types/node": "^25.2.3",
|