@axlsdk/eval 0.10.0 → 0.10.2

package/README.md

@@ -127,33 +127,121 @@ console.log(comparison.improvements);
 
 ### `defineEval(config)`
 
-Register an eval for CLI discovery:
+Define an eval file for discovery by the CLI or Studio middleware:
 
 ```typescript
-import { defineEval } from '@axlsdk/eval';
+// evals/math.eval.ts
+import { defineEval, dataset, scorer } from '@axlsdk/eval';
+import { z } from 'zod';
+
+const mathDataset = dataset({
+  name: 'math-basics',
+  schema: z.object({ question: z.string() }),
+  annotations: z.object({ answer: z.number() }),
+  items: [
+    { input: { question: '2+2' }, annotations: { answer: 4 } },
+    { input: { question: '3*5' }, annotations: { answer: 15 } },
+  ],
+});
+
+const correctAnswer = scorer({
+  name: 'correct-answer',
+  description: 'Checks if the output contains the expected numeric answer',
+  score: (output, _input, annotations) =>
+    String(output).includes(String(annotations?.answer)) ? 1 : 0,
+});
 
 export default defineEval({
   workflow: 'math-workflow',
-  dataset: ds,
-  scorers: [exactMatch],
+  dataset: mathDataset,
+  scorers: [correctAnswer],
 });
 ```
 
+By default, the eval runner calls `runtime.execute(workflow, input)` for each dataset item, which requires the workflow to be registered on the runtime.
+
+For self-contained evals, export an `executeWorkflow` function alongside the config. This is called instead of `runtime.execute()` for each dataset item.
+
+The function receives `(input, runtime?)` — the second argument is the `AxlRuntime` instance when running via Studio or `runtime.runRegisteredEval()`. Use it to call agents without needing a registered workflow:
+
+```typescript
+// evals/qa-quality.eval.ts — calls an agent directly via the runtime
+import { defineEval, dataset, scorer } from '@axlsdk/eval';
+import { z } from 'zod';
+import { qaAgent } from '../src/agents/qa.js'; // import your agent definition
+
+const qaDataset = dataset({
+  name: 'qa-pairs',
+  schema: z.object({ question: z.string() }),
+  items: [
+    { input: { question: 'What is TypeScript?' } },
+    { input: { question: 'How do promises work?' } },
+  ],
+});
+
+const notEmpty = scorer({
+  name: 'not-empty',
+  description: 'Output is non-empty',
+  score: (output) => (String(output).length > 0 ? 1 : 0),
+});
+
+export default defineEval({
+  workflow: 'qa-quality', // label for results (not a runtime lookup)
+  dataset: qaDataset,
+  scorers: [notEmpty],
+});
+
+export async function executeWorkflow(input: { question: string }, runtime?: any) {
+  // runtime is the AxlRuntime instance — use it to call agents
+  const ctx = runtime.createContext();
+  const output = await ctx.ask(qaAgent, input.question);
+  return { output };
+}
+```
+
+When no runtime is needed (e.g., testing a pure function), just ignore the second argument:
+
+```typescript
+// evals/parser.eval.ts — tests a pure function, no runtime needed
+export async function executeWorkflow(input: { raw: string }) {
+  const result = parseDocument(input.raw);
+  return { output: result };
+}
+```
+
 ## CLI
 
 Run evaluations from the command line:
 
 ```bash
 # Run an eval file
-npx axl eval ./evals/math.ts
+npx axl eval ./evals/math.eval.ts
+
+# Run all evals in a directory
+npx axl eval ./evals/
 
 # Save results to a file
-npx axl eval ./evals/math.ts --output ./results/baseline.json
+npx axl eval ./evals/math.eval.ts --output ./results/baseline.json
 
 # Compare two results
 npx axl eval compare ./results/baseline.json ./results/candidate.json
 ```
 
+## Studio Integration
+
+Eval files can also be lazy-loaded by the Studio middleware, enabling the Eval Runner panel without static imports that would create circular dependencies:
+
+```typescript
+import { createStudioMiddleware } from '@axlsdk/studio/middleware';
+
+const studio = createStudioMiddleware({
+  runtime,
+  evals: 'evals/**/*.eval.ts',
+});
+```
+
+See the [@axlsdk/studio README](../axl-studio/README.md#lazy-eval-loading) for details.
+
 ## Integration with AxlRuntime
 
 ```typescript

package/dist/{chunk-DRWANNYS.js → chunk-LMQJNZWT.js}

@@ -93,7 +93,7 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, provider) {
+async function runEval(config, executeWorkflow, provider, runtime) {
   const startTime = Date.now();
   const id = randomUUID();
   const items = await config.dataset.getItems();
@@ -127,7 +127,7 @@ async function runEval(config, executeWorkflow, provider) {
       scores: {}
     };
     try {
-      const result = await executeWorkflow(item.input);
+      const result = await executeWorkflow(item.input, runtime);
       evalItem.output = result.output;
       if (result.cost != null) {
         totalCost += result.cost;
@@ -195,4 +195,4 @@ export {
   evalCompare,
   runEval
 };
-//# sourceMappingURL=chunk-DRWANNYS.js.map
+//# sourceMappingURL=chunk-LMQJNZWT.js.map

package/dist/chunk-LMQJNZWT.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/compare.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime?: unknown,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  provider?: LlmScorerProvider,\n  runtime?: unknown,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AChFA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBAIA,UACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;","names":["round"]}

package/dist/cli.cjs

@@ -123,7 +123,7 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, provider) {
+async function runEval(config, executeWorkflow, provider, runtime) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
@@ -157,7 +157,7 @@ async function runEval(config, executeWorkflow, provider) {
       scores: {}
     };
     try {
-      const result = await executeWorkflow(item.input);
+      const result = await executeWorkflow(item.input, runtime);
       evalItem.output = result.output;
       if (result.cost != null) {
         totalCost += result.cost;

package/dist/cli.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\n\nasync function main() {\n  const args = process.argv.slice(2);\n\n  if (args.length === 0 || args[0] === '--help') {\n    console.log(`\nUsage:\n  npx axl eval <path>              Run eval file(s)\n  npx axl eval <path> --output <f> Run eval and save results to JSON\n  npx axl eval compare <a> <b>     Compare two eval result files\n  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === 'compare') {\n    await runCompare(args.slice(1));\n    return;\n  }\n\n  await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n  const failOnRegression = args.includes('--fail-on-regression');\n  const files = args.filter((a) => !a.startsWith('--'));\n\n  if (files.length !== 2) {\n    console.error('Usage: npx axl eval compare <baseline.json> <candidate.json>');\n    process.exit(1);\n  }\n\n  const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n  const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n  const comparison = evalCompare(baseline, candidate);\n\n  console.log(\n    `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n  );\n\n  const scorerNames = Object.keys(comparison.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n  console.log(`  ${'Scorer'.padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);\n  console.log(`  ${''.padEnd(maxNameLen, '-')}  --------  ---------  --------  ------`);\n\n  for (const name of scorerNames) {\n    const s = comparison.scorers[name];\n    const sign = s.delta > 0 ? '+' : '';\n    console.log(\n      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n    );\n  }\n\n  const stable = Math.max(\n    0,\n    baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n  );\n  console.log(\n    `\\n  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n  );\n\n  if (failOnRegression && comparison.regressions.length > 0) {\n    process.exit(1);\n  }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n  const resolved = path.resolve(p);\n  try {\n    const stat = statSync(resolved);\n    if (stat.isDirectory()) {\n      const entries = readdirSync(resolved);\n      return entries\n        .filter((e) => e.endsWith('.eval.ts') || e.endsWith('.eval.js'))\n        .map((e) => path.join(resolved, e));\n    }\n    return [resolved];\n  } catch {\n    return [resolved];\n  }\n}\n\nfunction formatTable(result: EvalResult): string {\n  const lines: string[] = [];\n  const scorerNames = Object.keys(result.summary.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n  const colWidth = 8;\n\n  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n  lines.push(\n    `  ${'Scorer'.padEnd(maxNameLen)}  ${'Mean'.padStart(colWidth)}  ${'Min'.padStart(colWidth)}  ${'Max'.padStart(colWidth)}  ${'p50'.padStart(colWidth)}  ${'p95'.padStart(colWidth)}`,\n  );\n\n  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n  lines.push('  ' + '\\u2500'.repeat(ruleLen));\n\n  for (const name of scorerNames) {\n    const s = result.summary.scorers[name];\n    lines.push(\n      `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`,\n    );\n  }\n\n  const durationSec = (result.duration / 1000).toFixed(1);\n  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n  lines.push('');\n  lines.push(\n    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n  );\n\n  return lines.join('\\n');\n}\n\nasync function runEvalCommand(args: string[]) {\n  let outputPath: string | undefined;\n  const paths: string[] = [];\n  for (let i = 0; i < args.length; i++) {\n    if (args[i] === '--output' && i + 1 < args.length) {\n      outputPath = args[++i];\n    } else if (!args[i].startsWith('--')) {\n      paths.push(args[i]);\n    }\n  }\n\n  if (paths.length === 0) {\n    console.error('Error: No eval file path provided');\n    process.exit(1);\n  }\n\n  const evalFiles: string[] = [];\n  for (const p of paths) {\n    evalFiles.push(...collectEvalFiles(p));\n  }\n\n  if (evalFiles.length === 0) {\n    console.error('Error: No eval files found');\n    process.exit(1);\n  }\n\n  const results: EvalResult[] = [];\n\n  for (const filePath of evalFiles) {\n    try {\n      const mod = await import(path.resolve(filePath));\n      const evalConfig: EvalConfig = mod.default ?? mod.config ?? mod;\n\n      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n        console.error(\n          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n        );\n        continue;\n      }\n\n      const executeWorkflow =\n        mod.executeWorkflow ?? (async (input: unknown) => ({ output: input }));\n\n      const result = await runEval(evalConfig, executeWorkflow, mod.provider);\n      results.push(result);\n\n      console.log('\\n' + formatTable(result) + '\\n');\n    } catch (err) {\n      console.error(\n        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n      );\n    }\n  }\n\n  if (outputPath && results.length > 0) {\n    const output = results.length === 1 ? results[0] : results;\n    // Ensure output directory exists\n    const outputDir = path.dirname(path.resolve(outputPath));\n    await mkdir(outputDir, { recursive: true });\n    await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n    console.log(`Results saved to ${outputPath}`);\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (input: unknown) => Promise<{ output: unknown; cost?: number }>,\n  provider?: LlmScorerProvider,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,qBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AChFA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBACA,UACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK;AAC/C,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;AF7IA,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,UAAU;AAC7C,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAMf;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,8DAA8D;AAC5E,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,yBAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,4BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC,EAC9D,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,eAAe,eAAe,MAAgB;AAC5C,MAAI;AACJ,QAAM,QAAkB,CAAC;AACzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,QAAI,KAAK,CAAC,MAAM,cAAc,IAAI,IAAI,KAAK,QAAQ;AACjD,mBAAa,KAAK,EAAE,CAAC;AAAA,IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,GAAG;AACpC,YAAM,KAAK,KAAK,CAAC,CAAC;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAwB,CAAC;AAE/B,aAAW,YAAY,WAAW;AAChC,QAAI;AACF,YAAM,MAAM,MAAM,OAAY,aAAQ,QAAQ;AAC9C,YAAM,aAAyB,IAAI,WAAW,IAAI,UAAU;AAE5D,UAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,gBAAQ;AAAA,UACN,UAAU,QAAQ;AAAA,QACpB;AACA;AAAA,MACF;AAEA,YAAM,kBACJ,IAAI,oBAAoB,OAAO,WAAoB,EAAE,QAAQ,MAAM;AAErE,YAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,QAAQ;AACtE,cAAQ,KAAK,MAAM;AAEnB,cAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,IAC/C,SAAS,KAAK;AACZ,cAAQ;AAAA,QACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MACrF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,UAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AAEnD,UAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,cAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,cAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,YAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,EAC9C;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["round","readFileAsync","writeFileAsync"]}
+{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\n\nasync function main() {\n  const args = process.argv.slice(2);\n\n  if (args.length === 0 || args[0] === '--help') {\n    console.log(`\nUsage:\n  npx axl eval <path>              Run eval file(s)\n  npx axl eval <path> --output <f> Run eval and save results to JSON\n  npx axl eval compare <a> <b>     Compare two eval result files\n  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === 'compare') {\n    await runCompare(args.slice(1));\n    return;\n  }\n\n  await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n  const failOnRegression = args.includes('--fail-on-regression');\n  const files = args.filter((a) => !a.startsWith('--'));\n\n  if (files.length !== 2) {\n    console.error('Usage: npx axl eval compare <baseline.json> <candidate.json>');\n    process.exit(1);\n  }\n\n  const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n  const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n  const comparison = evalCompare(baseline, candidate);\n\n  console.log(\n    `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n  );\n\n  const scorerNames = Object.keys(comparison.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n  console.log(`  ${'Scorer'.padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);\n  console.log(`  ${''.padEnd(maxNameLen, '-')}  --------  ---------  --------  ------`);\n\n  for (const name of scorerNames) {\n    const s = comparison.scorers[name];\n    const sign = s.delta > 0 ? '+' : '';\n    console.log(\n      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n    );\n  }\n\n  const stable = Math.max(\n    0,\n    baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n  );\n  console.log(\n    `\\n  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n  );\n\n  if (failOnRegression && comparison.regressions.length > 0) {\n    process.exit(1);\n  }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n  const resolved = path.resolve(p);\n  try {\n    const stat = statSync(resolved);\n    if (stat.isDirectory()) {\n      const entries = readdirSync(resolved);\n      return entries\n        .filter((e) => e.endsWith('.eval.ts') || e.endsWith('.eval.js'))\n        .map((e) => path.join(resolved, e));\n    }\n    return [resolved];\n  } catch {\n    return [resolved];\n  }\n}\n\nfunction formatTable(result: EvalResult): string {\n  const lines: string[] = [];\n  const scorerNames = Object.keys(result.summary.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n  const colWidth = 8;\n\n  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n  lines.push(\n    `  ${'Scorer'.padEnd(maxNameLen)}  ${'Mean'.padStart(colWidth)}  ${'Min'.padStart(colWidth)}  ${'Max'.padStart(colWidth)}  ${'p50'.padStart(colWidth)}  ${'p95'.padStart(colWidth)}`,\n  );\n\n  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n  lines.push('  ' + '\\u2500'.repeat(ruleLen));\n\n  for (const name of scorerNames) {\n    const s = result.summary.scorers[name];\n    lines.push(\n      `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`,\n    );\n  }\n\n  const durationSec = (result.duration / 1000).toFixed(1);\n  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n  lines.push('');\n  lines.push(\n    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n  );\n\n  return lines.join('\\n');\n}\n\nasync function runEvalCommand(args: string[]) {\n  let outputPath: string | undefined;\n  const paths: string[] = [];\n  for (let i = 0; i < args.length; i++) {\n    if (args[i] === '--output' && i + 1 < args.length) {\n      outputPath = args[++i];\n    } else if (!args[i].startsWith('--')) {\n      paths.push(args[i]);\n    }\n  }\n\n  if (paths.length === 0) {\n    console.error('Error: No eval file path provided');\n    process.exit(1);\n  }\n\n  const evalFiles: string[] = [];\n  for (const p of paths) {\n    evalFiles.push(...collectEvalFiles(p));\n  }\n\n  if (evalFiles.length === 0) {\n    console.error('Error: No eval files found');\n    process.exit(1);\n  }\n\n  const results: EvalResult[] = [];\n\n  for (const filePath of evalFiles) {\n    try {\n      const mod = await import(path.resolve(filePath));\n      const evalConfig: EvalConfig = mod.default ?? mod.config ?? mod;\n\n      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n        console.error(\n          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n        );\n        continue;\n      }\n\n      const executeWorkflow =\n        mod.executeWorkflow ?? (async (input: unknown) => ({ output: input }));\n\n      const result = await runEval(evalConfig, executeWorkflow, mod.provider);\n      results.push(result);\n\n      console.log('\\n' + formatTable(result) + '\\n');\n    } catch (err) {\n      console.error(\n        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n      );\n    }\n  }\n\n  if (outputPath && results.length > 0) {\n    const output = results.length === 1 ? results[0] : results;\n    // Ensure output directory exists\n    const outputDir = path.dirname(path.resolve(outputPath));\n    await mkdir(outputDir, { recursive: true });\n    await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n    console.log(`Results saved to ${outputPath}`);\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime?: unknown,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  provider?: LlmScorerProvider,\n  runtime?: unknown,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,qBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AChFA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBAIA,UACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;AFjJA,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,UAAU;AAC7C,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,CAMf;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,8DAA8D;AAC5E,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,yBAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,4BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC,EAC9D,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AACrC,UAAM;AAAA,MACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,IACvO;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAEA,eAAe,eAAe,MAAgB;AAC5C,MAAI;AACJ,QAAM,QAAkB,CAAC;AACzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,QAAI,KAAK,CAAC,MAAM,cAAc,IAAI,IAAI,KAAK,QAAQ;AACjD,mBAAa,KAAK,EAAE,CAAC;AAAA,IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,WAAW,IAAI,GAAG;AACpC,YAAM,KAAK,KAAK,CAAC,CAAC;AAAA,IACpB;AAAA,EACF;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAwB,CAAC;AAE/B,aAAW,YAAY,WAAW;AAChC,QAAI;AACF,YAAM,MAAM,MAAM,OAAY,aAAQ,QAAQ;AAC9C,YAAM,aAAyB,IAAI,WAAW,IAAI,UAAU;AAE5D,UAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,gBAAQ;AAAA,UACN,UAAU,QAAQ;AAAA,QACpB;AACA;AAAA,MACF;AAEA,YAAM,kBACJ,IAAI,oBAAoB,OAAO,WAAoB,EAAE,QAAQ,MAAM;AAErE,YAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,IAAI,QAAQ;AACtE,cAAQ,KAAK,MAAM;AAEnB,cAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,IAC/C,SAAS,KAAK;AACZ,cAAQ;AAAA,QACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,MACrF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,UAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AAEnD,UAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,cAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,cAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,YAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,EAC9C;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["round","readFileAsync","writeFileAsync"]}

package/dist/cli.js

@@ -2,7 +2,7 @@
 import {
   evalCompare,
   runEval
-} from "./chunk-DRWANNYS.js";
+} from "./chunk-LMQJNZWT.js";
 
 // src/cli.ts
 import { readdirSync, statSync } from "fs";

package/dist/index.cjs

@@ -150,7 +150,7 @@ function parseCost(cost) {
   if (!match) throw new Error(`Invalid cost format: "${cost}"`);
   return parseFloat(match[1]);
 }
-async function runEval(config, executeWorkflow, provider) {
+async function runEval(config, executeWorkflow, provider, runtime) {
   const startTime = Date.now();
   const id = (0, import_node_crypto.randomUUID)();
   const items = await config.dataset.getItems();
@@ -184,7 +184,7 @@ async function runEval(config, executeWorkflow, provider) {
       scores: {}
     };
     try {
-      const result = await executeWorkflow(item.input);
+      const result = await executeWorkflow(item.input, runtime);
       evalItem.output = result.output;
       if (result.cost != null) {
         totalCost += result.cost;

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/runner.ts","../src/compare.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerFn } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","export type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(output: TOutput, input: TInput, annotations?: TAnnotations): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer } from './scorer.js';\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _provider?: LlmScorerProvider } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(output: unknown, input: unknown, annotations?: unknown): Promise<number> {\n      const injectedProvider = scorerInstance._provider;\n      if (!injectedProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" requires a provider. Run via axl.eval() or the CLI.`,\n        );\n      }\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const colonIdx = config.model.indexOf(':');\n      const model = colonIdx > -1 ? config.model.slice(colonIdx + 1) : config.model;\n\n      const response = await injectedProvider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2 },\n      );\n\n      const parsed = JSON.parse(response.content);\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (input: unknown) => Promise<{ output: unknown; cost?: number }>,\n  provider?: LlmScorerProvider,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC1CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACRO,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAA6D;AAAA,IACjE,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MAAM,QAAiB,OAAgB,aAAwC;AACnF,YAAM,mBAAmB,eAAe;AACxC,UAAI,CAAC,kBAAkB;AACrB,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAC5B;AAAA,MACF;AAEA,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,OAAO,MAAM,QAAQ,GAAG;AACzC,YAAM,QAAQ,WAAW,KAAK,OAAO,MAAM,MAAM,WAAW,CAAC,IAAI,OAAO;AAExE,YAAM,WAAW,MAAM,iBAAiB;AAAA,QACtC;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,IAAI;AAAA,MAClD;AAEA,YAAM,SAAS,KAAK,MAAM,SAAS,OAAO;AAC1C,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AChEO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACHA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBACA,UACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAIA,QAAO,OAAO;AAChB,QAACA,QAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK;AAC/C,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAMA,QAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAWA,QAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;ACpJA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQA,OAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;","names":["scorer","round"]}
+{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/runner.ts","../src/compare.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerFn } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","export type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(output: TOutput, input: TInput, annotations?: TAnnotations): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer } from './scorer.js';\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _provider?: LlmScorerProvider } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(output: unknown, input: unknown, annotations?: unknown): Promise<number> {\n      const injectedProvider = scorerInstance._provider;\n      if (!injectedProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" requires a provider. Run via axl.eval() or the CLI.`,\n        );\n      }\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const colonIdx = config.model.indexOf(':');\n      const model = colonIdx > -1 ? config.model.slice(colonIdx + 1) : config.model;\n\n      const response = await injectedProvider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2 },\n      );\n\n      const parsed = JSON.parse(response.content);\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime?: unknown,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  provider?: LlmScorerProvider,\n  runtime?: unknown,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC1CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACRO,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAA6D;AAAA,IACjE,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MAAM,QAAiB,OAAgB,aAAwC;AACnF,YAAM,mBAAmB,eAAe;AACxC,UAAI,CAAC,kBAAkB;AACrB,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAC5B;AAAA,MACF;AAEA,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,OAAO,MAAM,QAAQ,GAAG;AACzC,YAAM,QAAQ,WAAW,KAAK,OAAO,MAAM,MAAM,WAAW,CAAC,IAAI,OAAO;AAExE,YAAM,WAAW,MAAM,iBAAiB;AAAA,QACtC;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,IAAI;AAAA,MAClD;AAEA,YAAM,SAAS,KAAK,MAAM,SAAS,OAAO;AAC1C,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AChEO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACHA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBAIA,UACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAIA,QAAO,OAAO;AAChB,QAACA,QAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAMA,QAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAWA,QAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;ACxJA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQA,OAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;","names":["scorer","round"]}

package/dist/index.d.cts

@@ -128,10 +128,10 @@ type LlmScorerProvider = {
         content: string;
     }>;
 };
-declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown) => Promise<{
+declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, runtime?: unknown) => Promise<{
     output: unknown;
     cost?: number;
-}>, provider?: LlmScorerProvider): Promise<EvalResult>;
+}>, provider?: LlmScorerProvider, runtime?: unknown): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
 

package/dist/index.d.ts

@@ -128,10 +128,10 @@ type LlmScorerProvider = {
         content: string;
     }>;
 };
-declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown) => Promise<{
+declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, runtime?: unknown) => Promise<{
     output: unknown;
     cost?: number;
-}>, provider?: LlmScorerProvider): Promise<EvalResult>;
+}>, provider?: LlmScorerProvider, runtime?: unknown): Promise<EvalResult>;
 
 declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
 

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   evalCompare,
   runEval
-} from "./chunk-DRWANNYS.js";
+} from "./chunk-LMQJNZWT.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.10.0"
+    "@axlsdk/axl": "0.10.2"
   },
   "devDependencies": {
     "@types/node": "^25.2.3",

package/dist/chunk-DRWANNYS.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/compare.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (input: unknown) => Promise<{ output: unknown; cost?: number }>,\n  provider?: LlmScorerProvider,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AChFA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBACA,UACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK;AAC/C,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;","names":["round"]}