@axlsdk/eval 0.1.0

package/dist/cli.cjs

@@ -0,0 +1,375 @@
+#!/usr/bin/env node
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/cli.ts
+var import_node_fs = require("fs");
+var import_promises = require("fs/promises");
+var path = __toESM(require("path"), 1);
+
+// src/compare.ts
+function round(n) {
+  return Math.round(n * 100) / 100;
+}
+function evalCompare(baseline, candidate) {
+  if (baseline.dataset !== candidate.dataset) {
+    throw new Error(
+      `Cannot compare evals from different datasets: "${baseline.dataset}" vs "${candidate.dataset}"`
+    );
+  }
+  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();
+  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();
+  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
+    throw new Error(`Cannot compare evals with different scorers`);
+  }
+  const scorers = {};
+  for (const name of baselineScorerNames) {
+    const bMean = baseline.summary.scorers[name].mean;
+    const cMean = candidate.summary.scorers[name].mean;
+    const delta = round(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round(delta / bMean * 100) : 0;
+    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+  }
+  const regressions = [];
+  const improvements = [];
+  const minLength = Math.min(baseline.items.length, candidate.items.length);
+  for (let i = 0; i < minLength; i++) {
+    const bItem = baseline.items[i];
+    const cItem = candidate.items[i];
+    if (bItem.error || cItem.error) continue;
+    for (const name of baselineScorerNames) {
+      const bScore = bItem.scores[name];
+      const cScore = cItem.scores[name];
+      if (bScore === void 0 || cScore === void 0 || bScore < 0 || cScore < 0) continue;
+      const delta = round(cScore - bScore);
+      if (delta < -0.1)
+        regressions.push({
+          input: bItem.input,
+          scorer: name,
+          baselineScore: bScore,
+          candidateScore: cScore,
+          delta
+        });
+      else if (delta > 0.1)
+        improvements.push({
+          input: bItem.input,
+          scorer: name,
+          baselineScore: bScore,
+          candidateScore: cScore,
+          delta
+        });
+    }
+  }
+  const parts = [];
+  for (const name of baselineScorerNames) {
+    const s = scorers[name];
+    if (Math.abs(s.delta) > 1e-3) {
+      const direction = s.delta > 0 ? "improves" : "regresses";
+      const sign = s.delta > 0 ? "+" : "";
+      parts.push(
+        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`
+      );
+    }
+  }
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no significant changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  return {
+    baseline: { id: baseline.id, metadata: baseline.metadata },
+    candidate: { id: candidate.id, metadata: candidate.metadata },
+    scorers,
+    regressions,
+    improvements,
+    summary: summaryStr
+  };
+}
+
+// src/runner.ts
+var import_node_crypto = require("crypto");
+function computeStats(scores) {
+  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  const p50 = sorted[Math.floor(sorted.length * 0.5)];
+  const p95 = sorted[Math.floor(sorted.length * 0.95)];
+  return { mean: round2(mean), min: round2(min), max: round2(max), p50: round2(p50), p95: round2(p95) };
+}
+function round2(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+function parseCost(cost) {
+  const match = cost.match(/^\$?([\d.]+)$/);
+  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
+  return parseFloat(match[1]);
+}
+async function runEval(config, executeWorkflow, provider) {
+  const startTime = Date.now();
+  const id = (0, import_node_crypto.randomUUID)();
+  const items = await config.dataset.getItems();
+  const concurrency = config.concurrency ?? 5;
+  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
+  if (provider) {
+    for (const scorer of config.scorers) {
+      if (scorer.isLlm) {
+        scorer._provider = provider;
+      }
+    }
+  }
+  const evalItems = [];
+  let totalCost = 0;
+  let budgetExceeded = false;
+  async function processItem(item) {
+    if (budgetExceeded) {
+      evalItems.push({
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Budget exceeded",
+        scores: {}
+      });
+      return;
+    }
+    const evalItem = {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      scores: {}
+    };
+    try {
+      const result = await executeWorkflow(item.input);
+      evalItem.output = result.output;
+      if (result.cost != null) {
+        totalCost += result.cost;
+      }
+    } catch (err) {
+      evalItem.error = err instanceof Error ? err.message : String(err);
+      evalItems.push(evalItem);
+      return;
+    }
+    if (budgetLimit != null && totalCost > budgetLimit) {
+      budgetExceeded = true;
+    }
+    for (const scorer of config.scorers) {
+      try {
+        const score = await scorer.score(evalItem.output, item.input, item.annotations);
+        if (score < 0 || score > 1) {
+          if (!evalItem.errors) evalItem.errors = [];
+          evalItem.errors.push(
+            `Scorer "${scorer.name}" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`
+          );
+          evalItem.scores[scorer.name] = -1;
+        } else {
+          evalItem.scores[scorer.name] = round2(score);
+        }
+      } catch (err) {
+        if (!evalItem.errors) evalItem.errors = [];
+        evalItem.errors.push(
+          `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+        evalItem.scores[scorer.name] = -1;
+      }
+    }
+    evalItems.push(evalItem);
+  }
+  let index = 0;
+  async function runNext() {
+    while (index < items.length) {
+      const currentIndex = index++;
+      await processItem(items[currentIndex]);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
+  await Promise.all(workers);
+  const failures = evalItems.filter((i) => i.error).length;
+  const scorerNames = config.scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = evalItems.filter((i) => !i.error && i.scores[name] !== void 0 && i.scores[name] >= 0).map((i) => i.scores[name]);
+    scorerStats[name] = computeStats(scores);
+  }
+  return {
+    id,
+    workflow: config.workflow,
+    dataset: config.dataset.name,
+    metadata: config.metadata ?? {},
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost,
+    duration: Date.now() - startTime,
+    items: evalItems,
+    summary: { count: items.length, failures, scorers: scorerStats }
+  };
+}
+
+// src/cli.ts
+async function main() {
+  const args = process.argv.slice(2);
+  if (args.length === 0 || args[0] === "--help") {
+    console.log(`
+Usage:
+  npx axl eval <path>              Run eval file(s)
+  npx axl eval <path> --output <f> Run eval and save results to JSON
+  npx axl eval compare <a> <b>     Compare two eval result files
+  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
+`);
+    process.exit(0);
+  }
+  if (args[0] === "compare") {
+    await runCompare(args.slice(1));
+    return;
+  }
+  await runEvalCommand(args);
+}
+async function runCompare(args) {
+  const failOnRegression = args.includes("--fail-on-regression");
+  const files = args.filter((a) => !a.startsWith("--"));
+  if (files.length !== 2) {
+    console.error("Usage: npx axl eval compare <baseline.json> <candidate.json>");
+    process.exit(1);
+  }
+  const baseline = JSON.parse(await (0, import_promises.readFile)(files[0], "utf-8"));
+  const candidate = JSON.parse(await (0, import_promises.readFile)(files[1], "utf-8"));
+  const comparison = evalCompare(baseline, candidate);
+  console.log(
+    `
+Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})
+`
+  );
+  const scorerNames = Object.keys(comparison.scorers);
+  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);
+  console.log(`  ${"Scorer".padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);
+  console.log(`  ${"".padEnd(maxNameLen, "-")}  --------  ---------  --------  ------`);
+  for (const name of scorerNames) {
+    const s = comparison.scorers[name];
+    const sign = s.delta > 0 ? "+" : "";
+    console.log(
+      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + "%").padStart(6)}`
+    );
+  }
+  const stable = Math.max(
+    0,
+    baseline.items.length - comparison.regressions.length - comparison.improvements.length
+  );
+  console.log(
+    `
+  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}
+`
+  );
+  if (failOnRegression && comparison.regressions.length > 0) {
+    process.exit(1);
+  }
+}
+function collectEvalFiles(p) {
+  const resolved = path.resolve(p);
+  try {
+    const stat = (0, import_node_fs.statSync)(resolved);
+    if (stat.isDirectory()) {
+      const entries = (0, import_node_fs.readdirSync)(resolved);
+      return entries.filter((e) => e.endsWith(".eval.ts") || e.endsWith(".eval.js")).map((e) => path.join(resolved, e));
+    }
+    return [resolved];
+  } catch {
+    return [resolved];
+  }
+}
+function formatTable(result) {
+  const lines = [];
+  const scorerNames = Object.keys(result.summary.scorers);
+  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), "Scorer".length);
+  const colWidth = 8;
+  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);
+  lines.push(
+    `  ${"Scorer".padEnd(maxNameLen)}  ${"Mean".padStart(colWidth)}  ${"Min".padStart(colWidth)}  ${"Max".padStart(colWidth)}  ${"p50".padStart(colWidth)}  ${"p95".padStart(colWidth)}`
+  );
+  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;
+  lines.push("  " + "\u2500".repeat(ruleLen));
+  for (const name of scorerNames) {
+    const s = result.summary.scorers[name];
+    lines.push(
+      `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`
+    );
+  }
+  const durationSec = (result.duration / 1e3).toFixed(1);
+  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : "$0.00";
+  lines.push("");
+  lines.push(
+    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`
+  );
+  return lines.join("\n");
+}
+async function runEvalCommand(args) {
+  let outputPath;
+  const paths = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === "--output" && i + 1 < args.length) {
+      outputPath = args[++i];
+    } else if (!args[i].startsWith("--")) {
+      paths.push(args[i]);
+    }
+  }
+  if (paths.length === 0) {
+    console.error("Error: No eval file path provided");
+    process.exit(1);
+  }
+  const evalFiles = [];
+  for (const p of paths) {
+    evalFiles.push(...collectEvalFiles(p));
+  }
+  if (evalFiles.length === 0) {
+    console.error("Error: No eval files found");
+    process.exit(1);
+  }
+  const results = [];
+  for (const filePath of evalFiles) {
+    try {
+      const mod = await import(path.resolve(filePath));
+      const evalConfig = mod.default ?? mod.config ?? mod;
+      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+        console.error(
+          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
+        );
+        continue;
+      }
+      const executeWorkflow = mod.executeWorkflow ?? (async (input) => ({ output: input }));
+      const result = await runEval(evalConfig, executeWorkflow, mod.provider);
+      results.push(result);
+      console.log("\n" + formatTable(result) + "\n");
+    } catch (err) {
+      console.error(
+        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+  if (outputPath && results.length > 0) {
+    const output = results.length === 1 ? results[0] : results;
+    const outputDir = path.dirname(path.resolve(outputPath));
+    await (0, import_promises.mkdir)(outputDir, { recursive: true });
+    await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
+    console.log(`Results saved to ${outputPath}`);
+  }
+}
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+//# sourceMappingURL=cli.cjs.map

package/dist/cli.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\n\nasync function main() {\n  const args = process.argv.slice(2);\n\n  if (args.length === 0 || args[0] === '--help') {\n    console.log(`\nUsage:\n  npx axl eval <path>              Run eval file(s)\n  npx axl eval <path> --output <f> Run eval and save results to JSON\n  npx axl eval compare <a> <b>     Compare two eval result files\n  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === 'compare') {\n    await runCompare(args.slice(1));\n    return;\n  }\n\n  await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n  const failOnRegression = args.includes('--fail-on-regression');\n  const files = args.filter((a) => !a.startsWith('--'));\n\n  if (files.length !== 2) {\n    console.error('Usage: npx axl eval compare <baseline.json> <candidate.json>');\n    process.exit(1);\n  }\n\n  const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n  const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n  const comparison = evalCompare(baseline, candidate);\n\n  console.log(\n    `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n  );\n\n  const scorerNames = Object.keys(comparison.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n  console.log(`  ${'Scorer'.padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);\n  console.log(`  ${''.padEnd(maxNameLen, '-')}  --------  ---------  --------  ------`);\n\n  for (const name of scorerNames) {\n    const s = comparison.scorers[name];\n    const sign = s.delta > 0 ? '+' : '';\n    console.log(\n      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n    );\n  }\n\n  const stable = Math.max(\n    0,\n    baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n  );\n  console.log(\n    `\\n  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n  );\n\n  if (failOnRegression && comparison.regressions.length > 0) {\n    process.exit(1);\n  }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n  const resolved = path.resolve(p);\n  try {\n    const stat = statSync(resolved);\n    if (stat.isDirectory()) {\n      const entries = readdirSync(resolved);\n      return entries\n        .filter((e) => e.endsWith('.eval.ts') || e.endsWith('.eval.js'))\n        .map((e) => path.join(resolved, e));\n    }\n    return [resolved];\n  } catch {\n    return [resolved];\n  }\n}\n\nfunction formatTable(result: EvalResult): string {\n  const lines: string[] = [];\n  const scorerNames = Object.keys(result.summary.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n  const colWidth = 8;\n\n  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n  lines.push(\n    `  ${'Scorer'.padEnd(maxNameLen)}  ${'Mean'.padStart(colWidth)}  ${'Min'.padStart(colWidth)}  ${'Max'.padStart(colWidth)}  ${'p50'.padStart(colWidth)}  ${'p95'.padStart(colWidth)}`,\n  );\n\n  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n  lines.push('  ' + '\\u2500'.repeat(ruleLen));\n\n  for (const name of scorerNames) {\n    const s = result.summary.scorers[name];\n    lines.push(\n      `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`,\n    );\n  }\n\n  const durationSec = (result.duration / 1000).toFixed(1);\n  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n  lines.push('');\n  lines.push(\n    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n  );\n\n  return lines.join('\\n');\n}\n\nasync function runEvalCommand(args: string[]) {\n  let outputPath: string | undefined;\n  const paths: string[] = [];\n  for (let i = 0; i < args.length; i++) {\n    if (args[i] === '--output' && i + 1 < args.length) {\n      outputPath = args[++i];\n    } else if (!args[i].startsWith('--')) {\n      paths.push(args[i]);\n    }\n  }\n\n  if (paths.length === 0) {\n    console.error('Error: No eval file path provided');\n    process.exit(1);\n  }\n\n  const evalFiles: string[] = [];\n  for (const p of paths) {\n    evalFiles.push(...collectEvalFiles(p));\n  }\n\n  if (evalFiles.length === 0) {\n    console.error('Error: No eval files found');\n    process.exit(1);\n  }\n\n  const results: EvalResult[] = [];\n\n  for (const filePath of evalFiles) {\n    try {\n      const mod = await import(path.resolve(filePath));\n      const evalConfig: EvalConfig = mod.default ?? mod.config ?? mod;\n\n      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n        console.error(\n          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n        );\n        continue;\n      }\n\n      const executeWorkflow =\n        mod.executeWorkflow ?? (async (input: unknown) => ({ output: input }));\n\n      const result = await runEval(evalConfig, executeWorkflow, mod.provider);\n      results.push(result);\n\n      console.log('\\n' + formatTable(result) + '\\n');\n    } catch (err) {\n      console.error(\n        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n      );\n    }\n  }\n\n  if (outputPath && results.length > 0) {\n    const output = results.length === 1 ? results[0] : results;\n    // Ensure output directory exists\n    const outputDir = path.dirname(path.resolve(outputPath));\n    await mkdir(outputDir, { recursive: true });\n    await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n    console.log(`Results saved to ${outputPath}`);\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (input: unknown) => Promise<{ output: unknown; cost?: number }>,\n  provider?: any,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as any)._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,qBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AChFA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBACA,UACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAe,YAAY;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK;AAC/C,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;AFrIA,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,UAAU;AAC7C,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAMf;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,8DAA8D;AAC5E,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,yBAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,4BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC,EAC9D,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,eAAe,eAAe,MAAgB;AAC5C,MAAI;AACJ,QAAM,QAAkB,CAAC;AACzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,QAAI,KAAK,CAAC,MAAM,cAAc,IAAI,IAAI,KAAK,QAAQ;AACjD,mBAAa,KAAK,EAAE,CAAC;AAAA,IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,GAAG;AACpC,YAAM,KAAK,KAAK,CAAC,CAAC;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAwB,CAAC;AAE/B,aAAW,YAAY,WAAW;AAChC,QAAI;AACF,YAAM,MAAM,MAAM,OAAY,aAAQ,QAAQ;AAC9C,YAAM,aAAyB,IAAI,WAAW,IAAI,UAAU;AAE5D,UAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,gBAAQ;AAAA,UACN,UAAU,QAAQ;AAAA,QACpB;AACA;AAAA,MACF;AAEA,YAAM,kBACJ,IAAI,oBAAoB,OAAO,WAAoB,EAAE,QAAQ,MAAM;AAErE,YAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,QAAQ;AACtE,cAAQ,KAAK,MAAM;AAEnB,cAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,IAC/C,SAAS,KAAK;AACZ,cAAQ;AAAA,QACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MACrF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,UAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AAEnD,UAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,cAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,cAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,YAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,EAC9C;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["round","readFileAsync","writeFileAsync"]}

package/dist/cli.d.cts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.js

@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+import {
+  evalCompare,
+  runEval
+} from "./chunk-F5UZX6FP.js";
+
+// src/cli.ts
+import { readdirSync, statSync } from "fs";
+import { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from "fs/promises";
+import * as path from "path";
+async function main() {
+  const args = process.argv.slice(2);
+  if (args.length === 0 || args[0] === "--help") {
+    console.log(`
+Usage:
+  npx axl eval <path>              Run eval file(s)
+  npx axl eval <path> --output <f> Run eval and save results to JSON
+  npx axl eval compare <a> <b>     Compare two eval result files
+  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
+`);
+    process.exit(0);
+  }
+  if (args[0] === "compare") {
+    await runCompare(args.slice(1));
+    return;
+  }
+  await runEvalCommand(args);
+}
+async function runCompare(args) {
+  const failOnRegression = args.includes("--fail-on-regression");
+  const files = args.filter((a) => !a.startsWith("--"));
+  if (files.length !== 2) {
+    console.error("Usage: npx axl eval compare <baseline.json> <candidate.json>");
+    process.exit(1);
+  }
+  const baseline = JSON.parse(await readFileAsync(files[0], "utf-8"));
+  const candidate = JSON.parse(await readFileAsync(files[1], "utf-8"));
+  const comparison = evalCompare(baseline, candidate);
+  console.log(
+    `
+Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})
+`
+  );
+  const scorerNames = Object.keys(comparison.scorers);
+  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);
+  console.log(`  ${"Scorer".padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);
+  console.log(`  ${"".padEnd(maxNameLen, "-")}  --------  ---------  --------  ------`);
+  for (const name of scorerNames) {
+    const s = comparison.scorers[name];
+    const sign = s.delta > 0 ? "+" : "";
+    console.log(
+      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + "%").padStart(6)}`
+    );
+  }
+  const stable = Math.max(
+    0,
+    baseline.items.length - comparison.regressions.length - comparison.improvements.length
+  );
+  console.log(
+    `
+  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}
+`
+  );
+  if (failOnRegression && comparison.regressions.length > 0) {
+    process.exit(1);
+  }
+}
+function collectEvalFiles(p) {
+  const resolved = path.resolve(p);
+  try {
+    const stat = statSync(resolved);
+    if (stat.isDirectory()) {
+      const entries = readdirSync(resolved);
+      return entries.filter((e) => e.endsWith(".eval.ts") || e.endsWith(".eval.js")).map((e) => path.join(resolved, e));
+    }
+    return [resolved];
+  } catch {
+    return [resolved];
+  }
+}
+function formatTable(result) {
+  const lines = [];
+  const scorerNames = Object.keys(result.summary.scorers);
+  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), "Scorer".length);
+  const colWidth = 8;
+  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);
+  lines.push(
+    `  ${"Scorer".padEnd(maxNameLen)}  ${"Mean".padStart(colWidth)}  ${"Min".padStart(colWidth)}  ${"Max".padStart(colWidth)}  ${"p50".padStart(colWidth)}  ${"p95".padStart(colWidth)}`
+  );
+  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;
+  lines.push("  " + "\u2500".repeat(ruleLen));
+  for (const name of scorerNames) {
+    const s = result.summary.scorers[name];
+    lines.push(
+      `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`
+    );
+  }
+  const durationSec = (result.duration / 1e3).toFixed(1);
+  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : "$0.00";
+  lines.push("");
+  lines.push(
+    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`
+  );
+  return lines.join("\n");
+}
+async function runEvalCommand(args) {
+  let outputPath;
+  const paths = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === "--output" && i + 1 < args.length) {
+      outputPath = args[++i];
+    } else if (!args[i].startsWith("--")) {
+      paths.push(args[i]);
+    }
+  }
+  if (paths.length === 0) {
+    console.error("Error: No eval file path provided");
+    process.exit(1);
+  }
+  const evalFiles = [];
+  for (const p of paths) {
+    evalFiles.push(...collectEvalFiles(p));
+  }
+  if (evalFiles.length === 0) {
+    console.error("Error: No eval files found");
+    process.exit(1);
+  }
+  const results = [];
+  for (const filePath of evalFiles) {
+    try {
+      const mod = await import(path.resolve(filePath));
+      const evalConfig = mod.default ?? mod.config ?? mod;
+      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+        console.error(
+          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
+        );
+        continue;
+      }
+      const executeWorkflow = mod.executeWorkflow ?? (async (input) => ({ output: input }));
+      const result = await runEval(evalConfig, executeWorkflow, mod.provider);
+      results.push(result);
+      console.log("\n" + formatTable(result) + "\n");
+    } catch (err) {
+      console.error(
+        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+  if (outputPath && results.length > 0) {
+    const output = results.length === 1 ? results[0] : results;
+    const outputDir = path.dirname(path.resolve(outputPath));
+    await mkdir(outputDir, { recursive: true });
+    await writeFileAsync(outputPath, JSON.stringify(output, null, 2), "utf-8");
+    console.log(`Results saved to ${outputPath}`);
+  }
+}
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\n\nasync function main() {\n  const args = process.argv.slice(2);\n\n  if (args.length === 0 || args[0] === '--help') {\n    console.log(`\nUsage:\n  npx axl eval <path>              Run eval file(s)\n  npx axl eval <path> --output <f> Run eval and save results to JSON\n  npx axl eval compare <a> <b>     Compare two eval result files\n  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === 'compare') {\n    await runCompare(args.slice(1));\n    return;\n  }\n\n  await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n  const failOnRegression = args.includes('--fail-on-regression');\n  const files = args.filter((a) => !a.startsWith('--'));\n\n  if (files.length !== 2) {\n    console.error('Usage: npx axl eval compare <baseline.json> <candidate.json>');\n    process.exit(1);\n  }\n\n  const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n  const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n  const comparison = evalCompare(baseline, candidate);\n\n  console.log(\n    `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n  );\n\n  const scorerNames = Object.keys(comparison.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n  console.log(`  ${'Scorer'.padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);\n  console.log(`  ${''.padEnd(maxNameLen, '-')}  --------  ---------  --------  ------`);\n\n  for (const name of scorerNames) {\n    const s = comparison.scorers[name];\n    const sign = s.delta > 0 ? '+' : '';\n    console.log(\n      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n    );\n  }\n\n  const stable = Math.max(\n    0,\n    baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n  );\n  console.log(\n    `\\n  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n  );\n\n  if (failOnRegression && comparison.regressions.length > 0) {\n    process.exit(1);\n  }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n  const resolved = path.resolve(p);\n  try {\n    const stat = statSync(resolved);\n    if (stat.isDirectory()) {\n      const entries = readdirSync(resolved);\n      return entries\n        .filter((e) => e.endsWith('.eval.ts') || e.endsWith('.eval.js'))\n        .map((e) => path.join(resolved, e));\n    }\n    return [resolved];\n  } catch {\n    return [resolved];\n  }\n}\n\nfunction formatTable(result: EvalResult): string {\n  const lines: string[] = [];\n  const scorerNames = Object.keys(result.summary.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n  const colWidth = 8;\n\n  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n  lines.push(\n    `  ${'Scorer'.padEnd(maxNameLen)}  ${'Mean'.padStart(colWidth)}  ${'Min'.padStart(colWidth)}  ${'Max'.padStart(colWidth)}  ${'p50'.padStart(colWidth)}  ${'p95'.padStart(colWidth)}`,\n  );\n\n  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n  lines.push('  ' + '\\u2500'.repeat(ruleLen));\n\n  for (const name of scorerNames) {\n    const s = result.summary.scorers[name];\n    lines.push(\n      `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`,\n    );\n  }\n\n  const durationSec = (result.duration / 1000).toFixed(1);\n  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n  lines.push('');\n  lines.push(\n    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n  );\n\n  return lines.join('\\n');\n}\n\nasync function runEvalCommand(args: string[]) {\n  let outputPath: string | undefined;\n  const paths: string[] = [];\n  for (let i = 0; i < args.length; i++) {\n    if (args[i] === '--output' && i + 1 < args.length) {\n      outputPath = args[++i];\n    } else if (!args[i].startsWith('--')) {\n      paths.push(args[i]);\n    }\n  }\n\n  if (paths.length === 0) {\n    console.error('Error: No eval file path provided');\n    process.exit(1);\n  }\n\n  const evalFiles: string[] = [];\n  for (const p of paths) {\n    evalFiles.push(...collectEvalFiles(p));\n  }\n\n  if (evalFiles.length === 0) {\n    console.error('Error: No eval files found');\n    process.exit(1);\n  }\n\n  const results: EvalResult[] = [];\n\n  for (const filePath of evalFiles) {\n    try {\n      const mod = await import(path.resolve(filePath));\n      const evalConfig: EvalConfig = mod.default ?? mod.config ?? mod;\n\n      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n        console.error(\n          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n        );\n        continue;\n      }\n\n      const executeWorkflow =\n        mod.executeWorkflow ?? (async (input: unknown) => ({ output: input }));\n\n      const result = await runEval(evalConfig, executeWorkflow, mod.provider);\n      results.push(result);\n\n      console.log('\\n' + formatTable(result) + '\\n');\n    } catch (err) {\n      console.error(\n        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n      );\n    }\n  }\n\n  if (outputPath && results.length > 0) {\n    const output = results.length === 1 ? results[0] : results;\n    // Ensure output directory exists\n    const outputDir = path.dirname(path.resolve(outputPath));\n    await mkdir(outputDir, { recursive: true });\n    await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n    console.log(`Results saved to ${outputPath}`);\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n"],"mappings":";;;;;;;AAEA,SAAS,aAAa,gBAAgB;AACtC,SAAS,YAAY,eAAe,aAAa,gBAAgB,aAAa;AAC9E,YAAY,UAAU;AAKtB,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,UAAU;AAC7C,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAMf;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,8DAA8D;AAC5E,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,MAAM,cAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,UAAU,YAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC,EAC9D,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,eAAe,eAAe,MAAgB;AAC5C,MAAI;AACJ,QAAM,QAAkB,CAAC;AACzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,QAAI,KAAK,CAAC,MAAM,cAAc,IAAI,IAAI,KAAK,QAAQ;AACjD,mBAAa,KAAK,EAAE,CAAC;AAAA,IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,GAAG;AACpC,YAAM,KAAK,KAAK,CAAC,CAAC;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAwB,CAAC;AAE/B,aAAW,YAAY,WAAW;AAChC,QAAI;AACF,YAAM,MAAM,MAAM,OAAY,aAAQ,QAAQ;AAC9C,YAAM,aAAyB,IAAI,WAAW,IAAI,UAAU;AAE5D,UAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,gBAAQ;AAAA,UACN,UAAU,QAAQ;AAAA,QACpB;AACA;AAAA,MACF;AAEA,YAAM,kBACJ,IAAI,oBAAoB,OAAO,WAAoB,EAAE,QAAQ,MAAM;AAErE,YAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,QAAQ;AACtE,cAAQ,KAAK,MAAM;AAEnB,cAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,IAC/C,SAAS,KAAK;AACZ,cAAQ;AAAA,QACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MACrF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,UAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AAEnD,UAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,UAAM,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,UAAM,eAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,YAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,EAC9C;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}