@axlsdk/eval 0.13.1 → 0.13.3

package/README.md

@@ -10,63 +10,120 @@ Evaluation framework for [Axl](https://github.com/axl-sdk/axl) agentic workflows
 npm install @axlsdk/eval
 ```
 
-## API
+TypeScript eval files require [tsx](https://github.com/privatenumber/tsx) as a dev dependency (`npm install -D tsx`).
 
-### `dataset(config)`
+## Quick Start
 
-Define evaluation datasets:
+An eval file defines what to test (dataset), how to run it (execution function), and how to score it (scorers):
+
+```typescript
+// evals/qa.eval.ts
+import { defineEval, dataset, scorer, llmScorer } from '@axlsdk/eval';
+import type { AxlRuntime } from '@axlsdk/axl';
+import { z } from 'zod';
+import { qaAgent } from '../src/agents/qa.js';
+
+export default defineEval({
+  workflow: 'qa-eval',  // label for results (used in output table and comparisons)
+  dataset: dataset({
+    name: 'qa-basics',
+    schema: z.object({ question: z.string() }),
+    items: [
+      { input: { question: 'What is TypeScript?' } },
+      { input: { question: 'Explain closures.' } },
+    ],
+  }),
+  scorers: [
+    // Deterministic — runs in-process, no LLM call
+    scorer({
+      name: 'not-empty',
+      description: 'Output is non-empty',
+      score: (output) => (String(output).length > 10 ? 1 : 0),
+    }),
+    // LLM-as-judge — calls an LLM to evaluate quality
+    llmScorer({
+      name: 'relevance',
+      description: 'Is the answer relevant to the question?',
+      model: 'openai:gpt-4o',
+      system: 'Rate whether the answer is relevant to the question asked.',
+      schema: z.object({ score: z.number(), reasoning: z.string() }),
+    }),
+  ],
+});
+
+// How to produce output for each dataset item
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
+  const ctx = runtime.createContext();
+  return { output: await ctx.ask(qaAgent, input.question) };
+}
+```
+
+Run it:
+
+```bash
+OPENAI_API_KEY=sk-... npx axl-eval ./evals/qa.eval.ts
+```
+
+```
+Eval: qa-eval x qa-basics (2 items)
+  Scorer     Mean      Min      Max      p50      p95
+  ─────────────────────────────────────────────────────
+  not-empty  1.00     1.00     1.00     1.00     1.00
+  relevance  0.90     0.80     1.00     0.90     1.00
+
+  Failures: 0/2 | Cost: $0.01 | Duration: 3.2s
+```
+
+If an LLM scorer fails (wrong API key, provider down, invalid response), you'll see:
+
+```
+  relevance    --       --       --       --       --
+
+  Scorer errors (2/2 items affected):
+    - Scorer "relevance" threw: OpenAI API key is required. Set OPENAI_API_KEY or pass apiKey in options.
+```
+
+## Datasets
 
 ```typescript
 import { dataset } from '@axlsdk/eval';
 import { z } from 'zod';
 
-// Inline items with schema and annotations
 const ds = dataset({
   name: 'math-basics',
   schema: z.object({ question: z.string() }),
-  annotations: z.object({ answer: z.number() }),
+  annotations: z.object({ answer: z.number() }),  // optional ground truth
   items: [
     { input: { question: '2+2' }, annotations: { answer: 4 } },
     { input: { question: '3*5' }, annotations: { answer: 15 } },
-    { input: { question: '10/2' }, annotations: { answer: 5 } },
   ],
 });
-
-// Load from file
-const ds = dataset({
-  name: 'math-advanced',
-  schema: z.object({ question: z.string() }),
-  file: './datasets/math.json',
-});
 ```
 
-### `scorer(config)`
+You can also load from a file: `dataset({ name: 'large', schema, file: './data.json' })`.
+
+## Scorers
 
-Define deterministic scoring functions. The `score` callback receives `(output, input, annotations?)`:
+Scorers rate each output on a 0-1 scale. You can mix deterministic and LLM scorers in the same eval.
+
+### Deterministic scorers
+
+Pure functions — fast, free, deterministic. The `score` callback receives `(output, input, annotations?)`:
 
 ```typescript
 import { scorer } from '@axlsdk/eval';
 
-const exactMatch = scorer({
-  name: 'exact-match',
-  description: 'Checks if the output answer exactly matches the expected answer',
-  score: (output, input, annotations) => output.answer === annotations?.answer ? 1 : 0,
-});
-
-const partialMatch = scorer({
-  name: 'partial',
-  description: 'Checks if the output contains the expected answer',
-  score: (output, input, annotations) => {
-    const outputStr = String(output).toLowerCase();
-    const expected = String(annotations?.answer).toLowerCase();
-    return outputStr.includes(expected) ? 1 : 0;
-  },
+const containsAnswer = scorer({
+  name: 'contains-answer',
+  description: 'Output contains the expected numeric answer',
+  score: (output, _input, annotations) =>
+    String(output).includes(String(annotations?.answer)) ? 1 : 0,
 });
 ```
 
-### `llmScorer(config)`
+### LLM scorers
 
-Use an LLM as a judge. The scorer automatically constructs a prompt from the input, output, and annotations, then validates the response against your Zod schema:
+Use an LLM as a judge. The scorer constructs a prompt from the input, output, and annotations, calls the LLM, and validates the response against your Zod schema:
 
 ```typescript
 import { llmScorer } from '@axlsdk/eval';
@@ -74,285 +131,216 @@ import { z } from 'zod';
 
 const qualityJudge = llmScorer({
   name: 'quality',
-  description: 'Rates output quality on a 0-1 scale',
-  model: 'openai-responses:gpt-5.4',
-  system: 'You are an expert evaluator. Rate the quality of AI outputs.',
+  description: 'Rates overall output quality',
+  model: 'openai:gpt-4o',           // provider:model URI
+  system: 'Rate the quality of the AI output.',
   schema: z.object({
     score: z.number().min(0).max(1),
     reasoning: z.string(),
   }),
-  temperature: 0.2,
+  temperature: 0.2,                  // default: 0.2 (low for consistency)
 });
 ```
 
-### `runEval(config, executeFn, runtime)`
+The `model` field uses a `provider:model` URI. The provider is resolved automatically at eval time — just set the right API key:
 
-Run an evaluation. LLM scorer providers are automatically resolved from the runtime's provider registry using each scorer's `model` URI (e.g., `google:gemini-3.1-flash`). Ensure the relevant API key environment variable is set (e.g., `OPENAI_API_KEY`, `GOOGLE_AI_API_KEY`) or register providers via `runtime.registerProvider()`:
+| Provider | URI prefix | Env var |
+|----------|-----------|---------|
+| OpenAI (Chat Completions) | `openai:` | `OPENAI_API_KEY` |
+| OpenAI (Responses API) | `openai-responses:` | `OPENAI_API_KEY` |
+| Anthropic | `anthropic:` | `ANTHROPIC_API_KEY` |
+| Google Gemini | `google:` | `GOOGLE_API_KEY` or `GEMINI_API_KEY` |
+| Custom | `your-name:` | Via `runtime.registerProvider('your-name', provider)` |
 
-```typescript
-import { runEval } from '@axlsdk/eval';
+Different LLM scorers can use different providers — each resolves independently:
 
-const results = await runEval(
-  {
-    workflow: 'my-workflow',  // label stored in results for comparison
-    dataset: ds,
-    scorers: [exactMatch, qualityJudge],
-    concurrency: 5,
-  },
-  async (input) => {
-    const output = await runtime.execute('my-workflow', input);
-    return { output };
-  },
-  runtime,
-);
-
-console.log(results.summary);
-// { count: 3, failures: 0, scorers: { 'exact-match': { mean: 0.85, min: 0, max: 1, p50: 1, p95: 1 }, ... } }
+```typescript
+const qualityJudge = llmScorer({ name: 'quality', model: 'openai:gpt-4o', ... });
+const safetyJudge = llmScorer({ name: 'safety', model: 'anthropic:claude-sonnet-4-5-20250514', ... });
 ```
 
-### `evalCompare(baseline, candidate)`
-
-Compare two evaluation runs:
+## Running Evals
 
-```typescript
-import { evalCompare } from '@axlsdk/eval';
+### CLI
 
-const comparison = evalCompare(baselineResults, candidateResults);
+The most common way to run evals:
 
-console.log(comparison.regressions);
-// [{ input: ..., scorer: 'exact-match', baselineScore: 1, candidateScore: 0, delta: -1 }]
-
-console.log(comparison.improvements);
-// [{ input: ..., scorer: 'quality', baselineScore: 0.5, candidateScore: 0.9, delta: 0.4 }]
+```bash
+npx axl-eval ./evals/qa.eval.ts                    # run a single file
+npx axl-eval ./evals/                               # run all *.eval.* files in a directory
+npx axl-eval ./evals/ --output ./results/v1.json    # save results to JSON
+npx axl-eval ./evals/ --config ./axl.config.ts      # use a specific runtime config
 ```
 
-### `defineEval(config)`
+The CLI resolves a runtime automatically: `--config <path>` > auto-detect `axl.config.*` > bare `new AxlRuntime()` (providers from env vars).
 
-Define an eval file for discovery by the CLI or Studio middleware:
+### Programmatic
+
+**`runtime.eval()`** — when you have a workflow registered on the runtime. The `workflow` field must match the registered name:
 
 ```typescript
-// evals/math.eval.ts
-import { defineEval, dataset, scorer } from '@axlsdk/eval';
+import { AxlRuntime, workflow, agent } from '@axlsdk/axl';
+import { dataset, scorer } from '@axlsdk/eval';
 import { z } from 'zod';
 
-const mathDataset = dataset({
-  name: 'math-basics',
-  schema: z.object({ question: z.string() }),
-  annotations: z.object({ answer: z.number() }),
-  items: [
-    { input: { question: '2+2' }, annotations: { answer: 4 } },
-    { input: { question: '3*5' }, annotations: { answer: 15 } },
-  ],
+const qaAgent = agent({ name: 'qa', model: 'openai:gpt-4o', system: 'Answer questions.' });
+const qaWorkflow = workflow({
+  name: 'qa-workflow',
+  input: z.object({ question: z.string() }),
+  handler: async (ctx) => ctx.ask(qaAgent, ctx.input.question),
 });
 
-const correctAnswer = scorer({
-  name: 'correct-answer',
-  description: 'Checks if the output contains the expected numeric answer',
-  score: (output, _input, annotations) =>
-    String(output).includes(String(annotations?.answer)) ? 1 : 0,
-});
+const runtime = new AxlRuntime();
+runtime.register(qaWorkflow);
 
-export default defineEval({
-  workflow: 'math-workflow',
-  dataset: mathDataset,
-  scorers: [correctAnswer],
+const results = await runtime.eval({
+  workflow: 'qa-workflow',  // must match the registered workflow name
+  dataset: ds,
+  scorers: [containsAnswer, qualityJudge],
 });
 ```
 
-When running via `runtime.eval()` or `runtime.runRegisteredEval()`, the runner calls `runtime.execute(workflow, input)` by default, which requires the workflow to be registered on the runtime.
+**`runEval()`** — when you want full control. The `workflow` field is just a label; the second argument is the function that produces output:
 
-When running via the CLI, the eval runner resolves the execution function in order: (1) the exported `executeWorkflow` function, (2) `runtime.execute(workflow, input)` if the workflow is registered on the runtime, or (3) a passthrough that returns the input as the output (with a warning).
+```typescript
+import { runEval } from '@axlsdk/eval';
 
-The function receives `(input, runtime)` — the second argument is the `AxlRuntime` instance, always provided by the CLI, Studio, and `runtime.runRegisteredEval()`. Use it to call agents without needing a registered workflow:
+const results = await runEval(
+  { workflow: 'my-eval', dataset: ds, scorers: [containsAnswer, qualityJudge] },
+  async (input, runtime) => {
+    const ctx = runtime.createContext();
+    return { output: await ctx.ask(qaAgent, input.question) };
+  },
+  runtime,
+);
+```
 
-```typescript
-// evals/qa-quality.eval.ts — calls an agent directly via the runtime
-import { defineEval, dataset, scorer } from '@axlsdk/eval';
-import type { AxlRuntime } from '@axlsdk/axl';
-import { z } from 'zod';
-import { qaAgent } from '../src/agents/qa.js'; // import your agent definition
+### Studio
 
-const qaDataset = dataset({
-  name: 'qa-pairs',
-  schema: z.object({ question: z.string() }),
-  items: [
-    { input: { question: 'What is TypeScript?' } },
-    { input: { question: 'How do promises work?' } },
-  ],
-});
+Eval files can be lazy-loaded by the Studio middleware for the Eval Runner panel:
 
-const notEmpty = scorer({
-  name: 'not-empty',
-  description: 'Output is non-empty',
-  score: (output) => (String(output).length > 0 ? 1 : 0),
-});
+```typescript
+import { createStudioMiddleware } from '@axlsdk/studio/middleware';
 
-export default defineEval({
-  workflow: 'qa-quality', // label for results (not a runtime lookup)
-  dataset: qaDataset,
-  scorers: [notEmpty],
+const studio = createStudioMiddleware({
+  runtime,
+  evals: 'evals/**/*.eval.ts',
 });
-
-export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
-  const ctx = runtime.createContext();
-  const output = await ctx.ask(qaAgent, input.question);
-  return { output }; // cost tracked automatically via runtime.trackCost()
-}
 ```
 
-When no runtime is needed (e.g., testing a pure function), omit the runtime parameter — these evals work from both the CLI and Studio:
+See the [@axlsdk/studio README](../axl-studio/README.md#lazy-eval-loading) for details.
 
-```typescript
-// evals/parser.eval.ts — tests a pure function, no runtime needed
-export async function executeWorkflow(input: { raw: string }) {
-  const result = parseDocument(input.raw);
-  return { output: result };
-}
-```
+## Understanding Results
 
-> **Note:** The CLI always provides a runtime to `executeWorkflow`. By default it auto-detects `axl.config.*` in the working directory or falls back to a bare `AxlRuntime` (providers resolved from environment variables). Use `--config` for explicit config files.
+Each eval run returns an `EvalResult` with per-item scores and aggregate statistics.
 
-### Cost Tracking
+**Scores** are `number | null` per scorer per item:
+- `0` to `1` — valid score from the scorer
+- `null` — the scorer failed (see `item.scorerErrors` for details)
 
-Cost is tracked automatically. You don't need to return `cost` from your `executeWorkflow` — the eval runner wraps each item with `runtime.trackCost()`, which captures cost from all `createContext()` and `execute()` calls.
+Summary statistics (mean, p50, p95, etc.) exclude `null` scores. If all scores for a scorer are `null`, the CLI shows `--` instead of misleading `0.00`.
 
 ```typescript
-export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
-  const ctx = runtime.createContext();
-  const output = await ctx.ask(myAgent, input.question);
-  return { output }; // cost captured automatically
+const results = await runtime.eval({ ... });
+
+// Aggregate stats
+console.log(results.summary.scorers['quality'].mean);  // 0.85
+console.log(results.summary.count);                     // 50 items
+console.log(results.summary.failures);                  // 2 workflow errors
+
+// Per-item inspection
+for (const item of results.items) {
+  if (item.error) continue;                              // workflow threw
+  if (item.scores['quality'] === null) {
+    console.log('Scorer failed:', item.scorerErrors);    // e.g., ["Scorer "quality" threw: ..."]
+  }
 }
-```
-
-To override the automatic cost (e.g., to exclude setup calls), return it explicitly:
 
-```typescript
-return { output, cost: ctx.totalCost };
+// Cost includes both workflow and LLM scorer calls
+console.log(results.totalCost);  // 0.42
 ```
 
-You can also read `ctx.totalCost` at any point to inspect accumulated cost.
+## Comparing Results
 
-### Common Patterns
+Compare two runs to detect regressions and improvements:
 
-**Budget per item** — cap cost to prevent runaway evals:
+```bash
+npx axl-eval compare ./results/v1.json ./results/v2.json
+npx axl-eval compare v1.json v2.json --fail-on-regression  # exit 1 if worse
+```
 
 ```typescript
-export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
-  const ctx = runtime.createContext({ budget: '$0.50' });
-  const output = await ctx.ask(myAgent, input.question);
-  return { output };
-}
+import { evalCompare } from '@axlsdk/eval';
+
+const comparison = evalCompare(v1Results, v2Results);
+console.log(comparison.regressions);   // items that got worse
+console.log(comparison.improvements);  // items that got better
 ```
 
-**Timeout per item** — cancel items that take too long:
+## Eval Files in Detail
 
-```typescript
-export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
-  const controller = new AbortController();
-  setTimeout(() => controller.abort(), 30_000);
-  const ctx = runtime.createContext({ signal: controller.signal });
-  const output = await ctx.ask(myAgent, input.question);
-  return { output };
-}
-```
+### Execution function
 
-**Auto-approve tools** — when testing agents that have tools with `requireApproval`:
+By default, the runner calls `runtime.execute(workflow, input)` for each item. Export `executeWorkflow` to override:
 
 ```typescript
+// evals/qa.eval.ts
+import { defineEval, dataset, scorer } from '@axlsdk/eval';
+import type { AxlRuntime } from '@axlsdk/axl';
+import { qaAgent } from '../src/agents/qa.js';
+
+export default defineEval({
+  workflow: 'qa-eval',
+  dataset: dataset({ ... }),
+  scorers: [scorer({ ... })],
+});
+
 export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
-  const ctx = runtime.createContext({
-    awaitHumanHandler: async () => ({ approved: true }),
-  });
-  const output = await ctx.ask(myAgent, input.question);
-  return { output };
+  const ctx = runtime.createContext();
+  return { output: await ctx.ask(qaAgent, input.question) };
 }
 ```
 
-**Multi-turn evaluation** — test follow-up responses with conversation history:
+When no runtime is needed (e.g., testing a pure function):
 
 ```typescript
-export async function executeWorkflow(
-  input: { setupPrompt: string; setupResponse: string; followUp: string },
-  runtime: AxlRuntime,
-) {
-  const ctx = runtime.createContext({
-    sessionHistory: [
-      { role: 'user', content: input.setupPrompt },
-      { role: 'assistant', content: input.setupResponse },
-    ],
-  });
-  const output = await ctx.ask(myAgent, input.followUp);
-  return { output };
+export async function executeWorkflow(input: { raw: string }) {
+  return { output: parseDocument(input.raw) };
 }
 ```
 
-## CLI
-
-Run evaluations from the command line:
-
-```bash
-# Run an eval file
-npx axl-eval ./evals/math.eval.ts
+### Cost tracking
 
-# Run all evals in a directory
-npx axl-eval ./evals/
+Cost is tracked automatically — the runner wraps each item with `runtime.trackCost()`. LLM scorer costs are also included in `totalCost` and count toward the `budget` limit.
 
-# Save results to a file
-npx axl-eval ./evals/math.eval.ts --output ./results/baseline.json
+To override (e.g., exclude setup calls), return cost explicitly:
 
-# Use a config file for runtime access
-npx axl-eval ./evals/ --config ./axl.config.ts
-
-# Use custom import conditions (monorepo source exports)
-npx axl-eval ./evals/ --config ./axl.config.ts --conditions development
-
-# Compare two results
-npx axl-eval compare ./results/baseline.json ./results/candidate.json
-
-# Fail CI if regressions detected
-npx axl-eval compare baseline.json candidate.json --fail-on-regression
+```typescript
+return { output, cost: ctx.totalCost };
 ```
 
-TypeScript config and eval files require [tsx](https://github.com/privatenumber/tsx) as a dev dependency (`npm install -D tsx`).
-
-### Runtime Resolution
-
-The CLI resolves an `AxlRuntime` to pass to `executeWorkflow`:
-
-1. **`--config <path>`** — use an explicit config file
-2. **Auto-detect** — search for `axl.config.mts` → `.ts` → `.mjs` → `.js` in cwd
-3. **Fallback** — create a bare `new AxlRuntime()` (providers resolve from environment variables)
-
-When a runtime is available, each `executeWorkflow` call is wrapped with `runtime.trackCost()` for automatic cost attribution.
+### Common patterns
 
-## Studio Integration
+**Budget** — cap cost per item with `runtime.createContext({ budget: '$0.50' })`.
 
-Eval files can also be lazy-loaded by the Studio middleware, enabling the Eval Runner panel without static imports that would create circular dependencies:
+**Timeout** — abort slow items:
 
 ```typescript
-import { createStudioMiddleware } from '@axlsdk/studio/middleware';
-
-const studio = createStudioMiddleware({
-  runtime,
-  evals: 'evals/**/*.eval.ts',
-});
+const controller = new AbortController();
+setTimeout(() => controller.abort(), 30_000);
+const ctx = runtime.createContext({ signal: controller.signal });
 ```
 
-See the [@axlsdk/studio README](../axl-studio/README.md#lazy-eval-loading) for details.
+**Auto-approve tools** — skip human approval in evals: `runtime.createContext({ awaitHumanHandler: async () => ({ approved: true }) })`.
 
-## Integration with AxlRuntime
+**Multi-turn** — provide conversation history:
 
 ```typescript
-import { AxlRuntime } from '@axlsdk/axl';
-
-const runtime = new AxlRuntime();
-runtime.register(myWorkflow);
-
-const results = await runtime.eval({
-  workflow: 'my-workflow',
-  dataset: ds,
-  scorers: [exactMatch],
-  concurrency: 3,
+const ctx = runtime.createContext({
+  sessionHistory: [
+    { role: 'user', content: input.setupPrompt },
+    { role: 'assistant', content: input.setupResponse },
+  ],
 });
 ```
 

package/dist/{chunk-K73SNPVV.js → chunk-OHZX3VFE.js}

@@ -157,7 +157,7 @@ async function runEval(config, executeWorkflow, runtime) {
             totalCost += lastCost;
           }
         }
-        if (score < 0 || score > 1) {
+        if (!Number.isFinite(score) || score < 0 || score > 1) {
           if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
           evalItem.scorerErrors.push(
             `Scorer "${scorer.name}" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`
@@ -215,4 +215,4 @@ export {
   evalCompare,
   runEval
 };
-//# sourceMappingURL=chunk-K73SNPVV.js.map
+//# sourceMappingURL=chunk-OHZX3VFE.js.map

package/dist/chunk-OHZX3VFE.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/compare.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!Number.isFinite(score) || score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9EA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AACrD,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;","names":["round"]}

package/dist/cli.cjs

@@ -187,7 +187,7 @@ async function runEval(config, executeWorkflow, runtime) {
             totalCost += lastCost;
           }
         }
-        if (score < 0 || score > 1) {
+        if (!Number.isFinite(score) || score < 0 || score > 1) {
           if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
           evalItem.scorerErrors.push(
             `Scorer "${scorer.name}" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`

package/dist/cli.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n  findConfig,\n  resolveRuntime,\n  importModule,\n  registerConditions,\n  CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set(['--output', '--config', '--conditions', '--fail-on-regression']);\n\nasync function main() {\n  const args = process.argv.slice(2);\n\n  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n    console.log(`\nUsage:\n  axl-eval <path>                         Run eval file(s)\n  axl-eval <path> --output <file>         Save results to JSON\n  axl-eval <path> --config <file>         Use config file for runtime\n  axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)\n  axl-eval compare <a> <b>                Compare two eval result files\n  axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n  ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === 'compare') {\n    await runCompare(args.slice(1));\n    return;\n  }\n\n  await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n  const failOnRegression = args.includes('--fail-on-regression');\n  const files = args.filter((a) => !a.startsWith('--'));\n\n  if (files.length !== 2) {\n    console.error('Usage: axl-eval compare <baseline.json> <candidate.json>');\n    process.exit(1);\n  }\n\n  const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n  const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n  const comparison = evalCompare(baseline, candidate);\n\n  console.log(\n    `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n  );\n\n  const scorerNames = Object.keys(comparison.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n  console.log(`  ${'Scorer'.padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);\n  console.log(`  ${''.padEnd(maxNameLen, '-')}  --------  ---------  --------  ------`);\n\n  for (const name of scorerNames) {\n    const s = comparison.scorers[name];\n    const sign = s.delta > 0 ? '+' : '';\n    console.log(\n      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n    );\n  }\n\n  const stable = Math.max(\n    0,\n    baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n  );\n  console.log(\n    `\\n  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n  );\n\n  if (failOnRegression && comparison.regressions.length > 0) {\n    process.exit(1);\n  }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n  const resolved = path.resolve(p);\n  try {\n    const stat = statSync(resolved);\n    if (stat.isDirectory()) {\n      const entries = readdirSync(resolved);\n      return entries\n        .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n        .map((e) => path.join(resolved, e));\n    }\n    return [resolved];\n  } catch {\n    return [resolved];\n  }\n}\n\nfunction formatTable(result: EvalResult): string {\n  const lines: string[] = [];\n  const scorerNames = Object.keys(result.summary.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n  const colWidth = 8;\n\n  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n  lines.push(\n    `  ${'Scorer'.padEnd(maxNameLen)}  ${'Mean'.padStart(colWidth)}  ${'Min'.padStart(colWidth)}  ${'Max'.padStart(colWidth)}  ${'p50'.padStart(colWidth)}  ${'p95'.padStart(colWidth)}`,\n  );\n\n  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n  lines.push('  ' + '\\u2500'.repeat(ruleLen));\n\n  for (const name of scorerNames) {\n    const s = result.summary.scorers[name];\n    // Detect scorers with no valid scores (all items errored → computeStats([]) → all zeros)\n    const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;\n    if (validScoreCount === 0) {\n      lines.push(\n        `  ${name.padEnd(maxNameLen)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}`,\n      );\n    } else {\n      lines.push(\n        `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`,\n      );\n    }\n  }\n\n  const durationSec = (result.duration / 1000).toFixed(1);\n  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n  lines.push('');\n  lines.push(\n    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n  );\n\n  const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);\n  if (itemsWithErrors.length > 0) {\n    const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors!))];\n    lines.push('');\n    lines.push(\n      `  Scorer errors (${itemsWithErrors.length}/${result.summary.count} items affected):`,\n    );\n    for (const err of uniqueErrors.slice(0, 5)) {\n      lines.push(`    - ${err}`);\n    }\n    if (uniqueErrors.length > 5) {\n      lines.push(`    ... and ${uniqueErrors.length - 5} more`);\n    }\n  }\n\n  return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n  try {\n    const mod = await importModule(configPath, import.meta.url);\n    const runtime = resolveRuntime(mod) as AxlRuntime;\n\n    if (!runtime || typeof runtime.execute !== 'function') {\n      console.error(`Config must export a default AxlRuntime instance.`);\n      if (runtime) {\n        const keys = Object.keys(runtime as object)\n          .slice(0, 5)\n          .join(', ');\n        console.error(`  Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n      }\n      console.error(\n        `Example:\\n  import { AxlRuntime } from '@axlsdk/axl';\\n  export default new AxlRuntime({ ... });`,\n      );\n      process.exit(1);\n    }\n\n    return runtime;\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err);\n    if (\n      /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n        msg,\n      )\n    ) {\n      const ext = path.extname(configPath);\n      console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n      if (ext === '.ts' || ext === '.tsx') {\n        console.error(\n          `  Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n        );\n      } else {\n        console.error(`  Tip: add \"type\": \"module\" to your package.json.`);\n      }\n      console.error();\n    }\n    console.error(`Failed to load config:`, err);\n    process.exit(1);\n  }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n  // Register import conditions before any config loading\n  if (conditions && conditions.length > 0) {\n    await registerConditions(conditions);\n  }\n\n  // 1. Explicit --config\n  if (configArg) {\n    const configPath = path.resolve(process.cwd(), configArg);\n    const stat = statSync(configPath, { throwIfNoEntry: false });\n    if (!stat?.isFile()) {\n      console.error(`Config file not found: ${configPath}`);\n      process.exit(1);\n    }\n    console.error(`[axl-eval] Loading config from ${configPath}`);\n    return resolveRuntimeFromConfig(configPath);\n  }\n\n  // 2. Auto-detect\n  const found = findConfig(process.cwd());\n  if (found) {\n    console.error(`[axl-eval] Auto-detected config: ${found}`);\n    return resolveRuntimeFromConfig(found);\n  }\n\n  // 3. Bare runtime (providers from env vars)\n  const { AxlRuntime } = await import('@axlsdk/axl');\n  return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n  outputPath?: string;\n  configArg?: string;\n  conditions: string[];\n  paths: string[];\n} {\n  let outputPath: string | undefined;\n  let configArg: string | undefined;\n  let conditions: string[] = [];\n  const paths: string[] = [];\n\n  for (let i = 0; i < args.length; i++) {\n    const arg = args[i];\n    if (arg === '--output' || arg === '--config' || arg === '--conditions') {\n      if (i + 1 >= args.length) {\n        console.error(`Error: ${arg} requires a value`);\n        process.exit(1);\n      }\n      const value = args[++i];\n      if (arg === '--output') outputPath = value;\n      else if (arg === '--config') configArg = value;\n      else\n        conditions = value\n          .split(',')\n          .map((c) => c.trim())\n          .filter(Boolean);\n    } else if (arg.startsWith('--')) {\n      if (!KNOWN_FLAGS.has(arg)) {\n        console.error(`Unknown flag: ${arg}`);\n        process.exit(1);\n      }\n    } else {\n      paths.push(arg);\n    }\n  }\n\n  return { outputPath, configArg, conditions, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n  if (paths.length === 0) {\n    console.error('Error: No eval file path provided');\n    process.exit(1);\n  }\n\n  const evalFiles: string[] = [];\n  for (const p of paths) {\n    evalFiles.push(...collectEvalFiles(p));\n  }\n\n  if (evalFiles.length === 0) {\n    console.error('Error: No eval files found');\n    process.exit(1);\n  }\n\n  const runtime = await getRuntime(configArg, conditions);\n  const results: EvalResult[] = [];\n\n  try {\n    for (const filePath of evalFiles) {\n      try {\n        const mod = await importModule(path.resolve(filePath), import.meta.url);\n        const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n        if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n          console.error(\n            `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n          );\n          continue;\n        }\n\n        // Resolve executeWorkflow: custom export > registered workflow > passthrough\n        let executeWorkflow: (\n          input: unknown,\n          rt?: unknown,\n        ) => Promise<{ output: unknown; cost?: number }>;\n\n        if (mod.executeWorkflow) {\n          // Wrap custom executeWorkflow with trackCost for automatic cost attribution\n          executeWorkflow = async (input, rt) => {\n            const { result, cost: trackedCost } = await runtime.trackCost(async () => {\n              return mod.executeWorkflow(input, rt);\n            });\n            return { output: result.output, cost: result.cost ?? trackedCost };\n          };\n        } else if (runtime.getWorkflow(evalConfig.workflow)) {\n          // No executeWorkflow exported but workflow is registered — use runtime.execute()\n          executeWorkflow = async (input) => {\n            const { result, cost } = await runtime.trackCost(async () => {\n              return runtime.execute(evalConfig.workflow, input);\n            });\n            return { output: result, cost };\n          };\n        } else {\n          console.warn(\n            `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n          );\n          executeWorkflow = async (input) => ({ output: input });\n        }\n\n        const result = await runEval(evalConfig, executeWorkflow, runtime);\n        results.push(result);\n\n        console.log('\\n' + formatTable(result) + '\\n');\n      } catch (err) {\n        console.error(\n          `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n        );\n      }\n    }\n\n    if (outputPath && results.length > 0) {\n      const output = results.length === 1 ? results[0] : results;\n      const outputDir = path.dirname(path.resolve(outputPath));\n      await mkdir(outputDir, { recursive: true });\n      await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n      console.log(`Results saved to ${outputPath}`);\n    }\n  } finally {\n    await runtime.shutdown().catch(() => {});\n  }\n\n  // Exit with non-zero code if no evals succeeded\n  if (results.length === 0) {\n    process.exit(1);\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n  'axl.config.mts',\n  'axl.config.ts',\n  'axl.config.mjs',\n  'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n  for (const name of CONFIG_CANDIDATES) {\n    const p = resolve(cwd, name);\n    if (existsSync(p)) return p;\n  }\n  return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n  return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n  const def = mod.default as Record<string, any> | undefined;\n  return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n  | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n  | null\n  | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n  filePath: string,\n  parentURL: string,\n): Promise<Record<string, any>> {\n  if (needsTsxLoader(filePath)) {\n    if (tsImportFn === undefined) {\n      try {\n        // @ts-expect-error — tsx is an optional runtime dependency\n        const mod = await import('tsx/esm/api');\n        tsImportFn = mod.tsImport ?? null;\n      } catch {\n        tsImportFn = null;\n        console.warn(\n          '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n            '  Install it with: npm install -D tsx',\n        );\n      }\n    }\n    if (tsImportFn) {\n      return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n    }\n  }\n  return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n  try {\n    const nodeModule = await import('node:module');\n    const hookCode = [\n      `const extra = ${JSON.stringify(conditions)};`,\n      `export async function resolve(specifier, context, nextResolve) {`,\n      `  return nextResolve(specifier, {`,\n      `    ...context,`,\n      `    conditions: [...new Set([...context.conditions, ...extra])],`,\n      `  });`,\n      `}`,\n    ].join('\\n');\n    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n  } catch {\n    console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,kBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9EA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;ACpKA,uBAAwB;AACxB,qBAA2B;AAC3B,sBAA8B;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,QAAI,0BAAQ,KAAK,IAAI;AAC3B,YAAI,2BAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,eAAW,+BAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,WAAO,+BAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AH/GA;AAiBA,IAAM,cAAc,oBAAI,IAAI,CAAC,YAAY,YAAY,gBAAgB,sBAAsB,CAAC;AAE5F,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,0DAA0D;AACxE,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,0BAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,6BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AAErC,UAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAAE;AACvF,QAAI,oBAAoB,GAAG;AACzB,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,MAC1K;AAAA,IACF,OAAO;AACL,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,MACvO;AAAA,IACF;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,QAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,MAAM;AACzE,MAAI,gBAAgB,SAAS,GAAG;AAC9B,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,gBAAgB,QAAQ,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC;AACjF,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,oBAAoB,gBAAgB,MAAM,IAAI,OAAO,QAAQ,KAAK;AAAA,IACpE;AACA,eAAW,OAAO,aAAa,MAAM,GAAG,CAAC,GAAG;AAC1C,YAAM,KAAK,SAAS,GAAG,EAAE;AAAA,IAC3B;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,KAAK,eAAe,aAAa,SAAS,CAAC,OAAO;AAAA,IAC1D;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,WAAO,0BAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAKrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,gBAAgB;AACtE,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA;AAEvC,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM;AACpD;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM,EAAE,QAAAC,SAAQ,MAAM,YAAY,IAAI,MAAM,QAAQ,UAAU,YAAY;AACxE,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO,EAAE,QAAQA,QAAO,QAAQ,MAAMA,QAAO,QAAQ,YAAY;AAAA,UACnE;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAAA,SAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU,YAAY;AAC3D,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQA,SAAQ,KAAK;AAAA,UAChC;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,cAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,OAAO;AACjE,gBAAQ,KAAK,MAAM;AAEnB,gBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,MAC/C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["import_node_fs","round","readFileAsync","result","writeFileAsync"]}
+{"version":3,"sources":["../src/cli.ts","../src/compare.ts","../src/runner.ts","../src/cli-utils.ts"],"sourcesContent":["#!/usr/bin/env node\n\nimport { readdirSync, statSync } from 'node:fs';\nimport { readFile as readFileAsync, writeFile as writeFileAsync, mkdir } from 'node:fs/promises';\nimport * as path from 'node:path';\nimport type { AxlRuntime } from '@axlsdk/axl';\nimport { evalCompare } from './compare.js';\nimport { runEval } from './runner.js';\nimport type { EvalConfig, EvalResult } from './types.js';\nimport {\n  findConfig,\n  resolveRuntime,\n  importModule,\n  registerConditions,\n  CONFIG_CANDIDATES,\n} from './cli-utils.js';\n\nconst KNOWN_FLAGS = new Set(['--output', '--config', '--conditions', '--fail-on-regression']);\n\nasync function main() {\n  const args = process.argv.slice(2);\n\n  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {\n    console.log(`\nUsage:\n  axl-eval <path>                         Run eval file(s)\n  axl-eval <path> --output <file>         Save results to JSON\n  axl-eval <path> --config <file>         Use config file for runtime\n  axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)\n  axl-eval compare <a> <b>                Compare two eval result files\n  axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions\n\nConfig auto-detection (when --config is not specified):\n  ${CONFIG_CANDIDATES.join(' -> ')}\n\nWhen a config is found, the exported AxlRuntime is passed to executeWorkflow\nand cost is tracked automatically via runtime.trackCost().\nWhen no config is found, a bare AxlRuntime is created (providers from env vars).\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === 'compare') {\n    await runCompare(args.slice(1));\n    return;\n  }\n\n  await runEvalCommand(args);\n}\n\nasync function runCompare(args: string[]) {\n  const failOnRegression = args.includes('--fail-on-regression');\n  const files = args.filter((a) => !a.startsWith('--'));\n\n  if (files.length !== 2) {\n    console.error('Usage: axl-eval compare <baseline.json> <candidate.json>');\n    process.exit(1);\n  }\n\n  const baseline: EvalResult = JSON.parse(await readFileAsync(files[0], 'utf-8'));\n  const candidate: EvalResult = JSON.parse(await readFileAsync(files[1], 'utf-8'));\n  const comparison = evalCompare(baseline, candidate);\n\n  console.log(\n    `\\nCompare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${comparison.candidate.id.slice(0, 8)})\\n`,\n  );\n\n  const scorerNames = Object.keys(comparison.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 6);\n\n  console.log(`  ${'Scorer'.padEnd(maxNameLen)}  Baseline  Candidate  Delta     Change`);\n  console.log(`  ${''.padEnd(maxNameLen, '-')}  --------  ---------  --------  ------`);\n\n  for (const name of scorerNames) {\n    const s = comparison.scorers[name];\n    const sign = s.delta > 0 ? '+' : '';\n    console.log(\n      `  ${name.padEnd(maxNameLen)}  ${s.baselineMean.toFixed(3).padStart(8)}  ${s.candidateMean.toFixed(3).padStart(9)}  ${(sign + s.delta.toFixed(3)).padStart(8)}  ${(sign + s.deltaPercent.toFixed(1) + '%').padStart(6)}`,\n    );\n  }\n\n  const stable = Math.max(\n    0,\n    baseline.items.length - comparison.regressions.length - comparison.improvements.length,\n  );\n  console.log(\n    `\\n  Regressions: ${comparison.regressions.length} | Improvements: ${comparison.improvements.length} | Stable: ${stable}\\n`,\n  );\n\n  if (failOnRegression && comparison.regressions.length > 0) {\n    process.exit(1);\n  }\n}\n\nfunction collectEvalFiles(p: string): string[] {\n  const resolved = path.resolve(p);\n  try {\n    const stat = statSync(resolved);\n    if (stat.isDirectory()) {\n      const entries = readdirSync(resolved);\n      return entries\n        .filter((e) => /\\.eval\\.[mc]?[jt]sx?$/.test(e))\n        .map((e) => path.join(resolved, e));\n    }\n    return [resolved];\n  } catch {\n    return [resolved];\n  }\n}\n\nfunction formatTable(result: EvalResult): string {\n  const lines: string[] = [];\n  const scorerNames = Object.keys(result.summary.scorers);\n  const maxNameLen = Math.max(...scorerNames.map((n) => n.length), 'Scorer'.length);\n  const colWidth = 8;\n\n  lines.push(`Eval: ${result.workflow} x ${result.dataset} (${result.summary.count} items)`);\n  lines.push(\n    `  ${'Scorer'.padEnd(maxNameLen)}  ${'Mean'.padStart(colWidth)}  ${'Min'.padStart(colWidth)}  ${'Max'.padStart(colWidth)}  ${'p50'.padStart(colWidth)}  ${'p95'.padStart(colWidth)}`,\n  );\n\n  const ruleLen = maxNameLen + 2 + (colWidth + 2) * 5;\n  lines.push('  ' + '\\u2500'.repeat(ruleLen));\n\n  for (const name of scorerNames) {\n    const s = result.summary.scorers[name];\n    // Detect scorers with no valid scores (all items errored → computeStats([]) → all zeros)\n    const validScoreCount = result.items.filter((i) => !i.error && i.scores[name] != null).length;\n    if (validScoreCount === 0) {\n      lines.push(\n        `  ${name.padEnd(maxNameLen)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}  ${'--'.padStart(colWidth)}`,\n      );\n    } else {\n      lines.push(\n        `  ${name.padEnd(maxNameLen)}  ${s.mean.toFixed(2).padStart(colWidth)}  ${s.min.toFixed(2).padStart(colWidth)}  ${s.max.toFixed(2).padStart(colWidth)}  ${s.p50.toFixed(2).padStart(colWidth)}  ${s.p95.toFixed(2).padStart(colWidth)}`,\n      );\n    }\n  }\n\n  const durationSec = (result.duration / 1000).toFixed(1);\n  const costStr = result.totalCost > 0 ? `$${result.totalCost.toFixed(2)}` : '$0.00';\n  lines.push('');\n  lines.push(\n    `  Failures: ${result.summary.failures}/${result.summary.count} | Cost: ${costStr} | Duration: ${durationSec}s`,\n  );\n\n  const itemsWithErrors = result.items.filter((i) => i.scorerErrors?.length);\n  if (itemsWithErrors.length > 0) {\n    const uniqueErrors = [...new Set(itemsWithErrors.flatMap((i) => i.scorerErrors!))];\n    lines.push('');\n    lines.push(\n      `  Scorer errors (${itemsWithErrors.length}/${result.summary.count} items affected):`,\n    );\n    for (const err of uniqueErrors.slice(0, 5)) {\n      lines.push(`    - ${err}`);\n    }\n    if (uniqueErrors.length > 5) {\n      lines.push(`    ... and ${uniqueErrors.length - 5} more`);\n    }\n  }\n\n  return lines.join('\\n');\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\nasync function resolveRuntimeFromConfig(configPath: string): Promise<AxlRuntime> {\n  try {\n    const mod = await importModule(configPath, import.meta.url);\n    const runtime = resolveRuntime(mod) as AxlRuntime;\n\n    if (!runtime || typeof runtime.execute !== 'function') {\n      console.error(`Config must export a default AxlRuntime instance.`);\n      if (runtime) {\n        const keys = Object.keys(runtime as object)\n          .slice(0, 5)\n          .join(', ');\n        console.error(`  Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ''}`);\n      }\n      console.error(\n        `Example:\\n  import { AxlRuntime } from '@axlsdk/axl';\\n  export default new AxlRuntime({ ... });`,\n      );\n      process.exit(1);\n    }\n\n    return runtime;\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err);\n    if (\n      /Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(\n        msg,\n      )\n    ) {\n      const ext = path.extname(configPath);\n      console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);\n      if (ext === '.ts' || ext === '.tsx') {\n        console.error(\n          `  Tip: try renaming to .mts to force ESM format, or ensure tsx is installed and up to date.`,\n        );\n      } else {\n        console.error(`  Tip: add \"type\": \"module\" to your package.json.`);\n      }\n      console.error();\n    }\n    console.error(`Failed to load config:`, err);\n    process.exit(1);\n  }\n}\n\nasync function getRuntime(configArg?: string, conditions?: string[]): Promise<AxlRuntime> {\n  // Register import conditions before any config loading\n  if (conditions && conditions.length > 0) {\n    await registerConditions(conditions);\n  }\n\n  // 1. Explicit --config\n  if (configArg) {\n    const configPath = path.resolve(process.cwd(), configArg);\n    const stat = statSync(configPath, { throwIfNoEntry: false });\n    if (!stat?.isFile()) {\n      console.error(`Config file not found: ${configPath}`);\n      process.exit(1);\n    }\n    console.error(`[axl-eval] Loading config from ${configPath}`);\n    return resolveRuntimeFromConfig(configPath);\n  }\n\n  // 2. Auto-detect\n  const found = findConfig(process.cwd());\n  if (found) {\n    console.error(`[axl-eval] Auto-detected config: ${found}`);\n    return resolveRuntimeFromConfig(found);\n  }\n\n  // 3. Bare runtime (providers from env vars)\n  const { AxlRuntime } = await import('@axlsdk/axl');\n  return new AxlRuntime();\n}\n\n// ── Arg parsing ────────────────────────────────────────────────────\n\nfunction parseEvalArgs(args: string[]): {\n  outputPath?: string;\n  configArg?: string;\n  conditions: string[];\n  paths: string[];\n} {\n  let outputPath: string | undefined;\n  let configArg: string | undefined;\n  let conditions: string[] = [];\n  const paths: string[] = [];\n\n  for (let i = 0; i < args.length; i++) {\n    const arg = args[i];\n    if (arg === '--output' || arg === '--config' || arg === '--conditions') {\n      if (i + 1 >= args.length) {\n        console.error(`Error: ${arg} requires a value`);\n        process.exit(1);\n      }\n      const value = args[++i];\n      if (arg === '--output') outputPath = value;\n      else if (arg === '--config') configArg = value;\n      else\n        conditions = value\n          .split(',')\n          .map((c) => c.trim())\n          .filter(Boolean);\n    } else if (arg.startsWith('--')) {\n      if (!KNOWN_FLAGS.has(arg)) {\n        console.error(`Unknown flag: ${arg}`);\n        process.exit(1);\n      }\n    } else {\n      paths.push(arg);\n    }\n  }\n\n  return { outputPath, configArg, conditions, paths };\n}\n\n// ── Main eval command ──────────────────────────────────────────────\n\nasync function runEvalCommand(args: string[]) {\n  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);\n\n  if (paths.length === 0) {\n    console.error('Error: No eval file path provided');\n    process.exit(1);\n  }\n\n  const evalFiles: string[] = [];\n  for (const p of paths) {\n    evalFiles.push(...collectEvalFiles(p));\n  }\n\n  if (evalFiles.length === 0) {\n    console.error('Error: No eval files found');\n    process.exit(1);\n  }\n\n  const runtime = await getRuntime(configArg, conditions);\n  const results: EvalResult[] = [];\n\n  try {\n    for (const filePath of evalFiles) {\n      try {\n        const mod = await importModule(path.resolve(filePath), import.meta.url);\n        const evalConfig: EvalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;\n\n        if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {\n          console.error(\n            `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`,\n          );\n          continue;\n        }\n\n        // Resolve executeWorkflow: custom export > registered workflow > passthrough\n        let executeWorkflow: (\n          input: unknown,\n          rt?: unknown,\n        ) => Promise<{ output: unknown; cost?: number }>;\n\n        if (mod.executeWorkflow) {\n          // Wrap custom executeWorkflow with trackCost for automatic cost attribution\n          executeWorkflow = async (input, rt) => {\n            const { result, cost: trackedCost } = await runtime.trackCost(async () => {\n              return mod.executeWorkflow(input, rt);\n            });\n            return { output: result.output, cost: result.cost ?? trackedCost };\n          };\n        } else if (runtime.getWorkflow(evalConfig.workflow)) {\n          // No executeWorkflow exported but workflow is registered — use runtime.execute()\n          executeWorkflow = async (input) => {\n            const { result, cost } = await runtime.trackCost(async () => {\n              return runtime.execute(evalConfig.workflow, input);\n            });\n            return { output: result, cost };\n          };\n        } else {\n          console.warn(\n            `[axl-eval] Warning: ${filePath} does not export executeWorkflow — using input passthrough`,\n          );\n          executeWorkflow = async (input) => ({ output: input });\n        }\n\n        const result = await runEval(evalConfig, executeWorkflow, runtime);\n        results.push(result);\n\n        console.log('\\n' + formatTable(result) + '\\n');\n      } catch (err) {\n        console.error(\n          `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`,\n        );\n      }\n    }\n\n    if (outputPath && results.length > 0) {\n      const output = results.length === 1 ? results[0] : results;\n      const outputDir = path.dirname(path.resolve(outputPath));\n      await mkdir(outputDir, { recursive: true });\n      await writeFileAsync(outputPath, JSON.stringify(output, null, 2), 'utf-8');\n      console.log(`Results saved to ${outputPath}`);\n    }\n  } finally {\n    await runtime.shutdown().catch(() => {});\n  }\n\n  // Exit with non-zero code if no evals succeeded\n  if (results.length === 0) {\n    process.exit(1);\n  }\n}\n\nmain().catch((err) => {\n  console.error(err);\n  process.exit(1);\n});\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!Number.isFinite(score) || score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","/**\n * CLI utilities for config detection, runtime resolution, and loader registration.\n *\n * These are duplicated from @axlsdk/studio (cli-utils.ts, resolve-runtime.ts,\n * eval-loader.ts, cli.ts) because @axlsdk/eval cannot depend on @axlsdk/studio.\n * Keep in sync with the studio versions if either changes.\n */\n\nimport { resolve } from 'node:path';\nimport { existsSync } from 'node:fs';\nimport { pathToFileURL } from 'node:url';\n\n// ── Config auto-detection ──────────────────────────────────────────\n\nexport const CONFIG_CANDIDATES = [\n  'axl.config.mts',\n  'axl.config.ts',\n  'axl.config.mjs',\n  'axl.config.js',\n];\n\nexport function findConfig(cwd: string): string | undefined {\n  for (const name of CONFIG_CANDIDATES) {\n    const p = resolve(cwd, name);\n    if (existsSync(p)) return p;\n  }\n  return undefined;\n}\n\n// ── Extension helpers ──────────────────────────────────────────────\n\n/** Returns true if the file is TypeScript and needs tsx to load. */\nexport function needsTsxLoader(filePath: string): boolean {\n  return /\\.[mc]?tsx?$/.test(filePath);\n}\n\n// ── Runtime resolution ─────────────────────────────────────────────\n\n/**\n * Resolve the AxlRuntime from a dynamically imported config module.\n *\n * Module shapes handled:\n * - ESM `export default runtime` → mod.default is the runtime\n * - CJS compiled from `export default runtime` → mod.default.default is the runtime\n * - CJS `module.exports = runtime` → mod.default is the runtime\n * - Named `export { runtime }` → mod.runtime is the runtime\n */\nexport function resolveRuntime(mod: Record<string, any>): unknown {\n  const def = mod.default as Record<string, any> | undefined;\n  return def?.default ?? def ?? mod.runtime;\n}\n\n// ── Module loading ────────────────────────────────────────────────\n\n// Lazily resolved tsImport function from tsx. `undefined` = not yet checked,\n// `null` = tsx not available.\nlet tsImportFn:\n  | ((specifier: string, parentURL: string) => Promise<Record<string, any>>)\n  | null\n  | undefined;\n\n/**\n * Import a module, using tsx's `tsImport()` for TypeScript files.\n *\n * `tsImport()` handles ESM/CJS format correctly without process-wide side effects —\n * no need for `register()` hooks or ESM-forcing workarounds. Falls back to regular\n * `import()` for non-TypeScript files or when tsx is not installed.\n */\nexport async function importModule(\n  filePath: string,\n  parentURL: string,\n): Promise<Record<string, any>> {\n  if (needsTsxLoader(filePath)) {\n    if (tsImportFn === undefined) {\n      try {\n        // @ts-expect-error — tsx is an optional runtime dependency\n        const mod = await import('tsx/esm/api');\n        tsImportFn = mod.tsImport ?? null;\n      } catch {\n        tsImportFn = null;\n        console.warn(\n          '[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\\n' +\n            '  Install it with: npm install -D tsx',\n        );\n      }\n    }\n    if (tsImportFn) {\n      return (await tsImportFn(pathToFileURL(filePath).href, parentURL)) as Record<string, any>;\n    }\n  }\n  return await import(pathToFileURL(filePath).href);\n}\n\n// ── Conditions ────────────────────────────────────────────────────\n\nexport async function registerConditions(conditions: string[]): Promise<void> {\n  try {\n    const nodeModule = await import('node:module');\n    const hookCode = [\n      `const extra = ${JSON.stringify(conditions)};`,\n      `export async function resolve(specifier, context, nextResolve) {`,\n      `  return nextResolve(specifier, {`,\n      `    ...context,`,\n      `    conditions: [...new Set([...context.conditions, ...extra])],`,\n      `  });`,\n      `}`,\n    ].join('\\n');\n    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);\n  } catch {\n    console.warn('[axl-eval] Warning: --conditions requires Node.js 20.6+');\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,IAAAA,kBAAsC;AACtC,sBAA8E;AAC9E,WAAsB;;;ACFtB,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9EA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMC,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AACrD,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;ACpKA,uBAAwB;AACxB,qBAA2B;AAC3B,sBAA8B;AAIvB,IAAM,oBAAoB;AAAA,EAC/B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,WAAW,KAAiC;AAC1D,aAAW,QAAQ,mBAAmB;AACpC,UAAM,QAAI,0BAAQ,KAAK,IAAI;AAC3B,YAAI,2BAAW,CAAC,EAAG,QAAO;AAAA,EAC5B;AACA,SAAO;AACT;AAKO,SAAS,eAAe,UAA2B;AACxD,SAAO,eAAe,KAAK,QAAQ;AACrC;AAaO,SAAS,eAAe,KAAmC;AAChE,QAAM,MAAM,IAAI;AAChB,SAAO,KAAK,WAAW,OAAO,IAAI;AACpC;AAMA,IAAI;AAYJ,eAAsB,aACpB,UACA,WAC8B;AAC9B,MAAI,eAAe,QAAQ,GAAG;AAC5B,QAAI,eAAe,QAAW;AAC5B,UAAI;AAEF,cAAM,MAAM,MAAM,OAAO,aAAa;AACtC,qBAAa,IAAI,YAAY;AAAA,MAC/B,QAAQ;AACN,qBAAa;AACb,gBAAQ;AAAA,UACN;AAAA,QAEF;AAAA,MACF;AAAA,IACF;AACA,QAAI,YAAY;AACd,aAAQ,MAAM,eAAW,+BAAc,QAAQ,EAAE,MAAM,SAAS;AAAA,IAClE;AAAA,EACF;AACA,SAAO,MAAM,WAAO,+BAAc,QAAQ,EAAE;AAC9C;AAIA,eAAsB,mBAAmB,YAAqC;AAC5E,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,QAAa;AAC7C,UAAM,WAAW;AAAA,MACf,iBAAiB,KAAK,UAAU,UAAU,CAAC;AAAA,MAC3C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,EAAE,KAAK,IAAI;AACX,eAAW,SAAS,wBAAwB,mBAAmB,QAAQ,CAAC,EAAE;AAAA,EAC5E,QAAQ;AACN,YAAQ,KAAK,yDAAyD;AAAA,EACxE;AACF;;;AH/GA;AAiBA,IAAM,cAAc,oBAAI,IAAI,CAAC,YAAY,YAAY,gBAAgB,sBAAsB,CAAC;AAE5F,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,KAAK,MAAM,CAAC;AAEjC,MAAI,KAAK,WAAW,KAAK,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,MAAM,MAAM;AACjE,YAAQ,IAAI;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAUZ,kBAAkB,KAAK,MAAM,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,CAKjC;AACG,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW;AACzB,UAAM,WAAW,KAAK,MAAM,CAAC,CAAC;AAC9B;AAAA,EACF;AAEA,QAAM,eAAe,IAAI;AAC3B;AAEA,eAAe,WAAW,MAAgB;AACxC,QAAM,mBAAmB,KAAK,SAAS,sBAAsB;AAC7D,QAAM,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,IAAI,CAAC;AAEpD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,0DAA0D;AACxE,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,WAAuB,KAAK,MAAM,UAAM,gBAAAC,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC9E,QAAM,YAAwB,KAAK,MAAM,UAAM,gBAAAA,UAAc,MAAM,CAAC,GAAG,OAAO,CAAC;AAC/E,QAAM,aAAa,YAAY,UAAU,SAAS;AAElD,UAAQ;AAAA,IACN;AAAA,qBAAwB,WAAW,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,mBAAmB,WAAW,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;AAAA;AAAA,EAClH;AAEA,QAAM,cAAc,OAAO,KAAK,WAAW,OAAO;AAClD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC;AAElE,UAAQ,IAAI,KAAK,SAAS,OAAO,UAAU,CAAC,yCAAyC;AACrF,UAAQ,IAAI,KAAK,GAAG,OAAO,YAAY,GAAG,CAAC,yCAAyC;AAEpF,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAQ;AAAA,MACN,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,aAAa,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,KAAK,EAAE,cAAc,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,MAAM,QAAQ,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,OAAO,EAAE,aAAa,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AAAA,IACxN;AAAA,EACF;AAEA,QAAM,SAAS,KAAK;AAAA,IAClB;AAAA,IACA,SAAS,MAAM,SAAS,WAAW,YAAY,SAAS,WAAW,aAAa;AAAA,EAClF;AACA,UAAQ;AAAA,IACN;AAAA,iBAAoB,WAAW,YAAY,MAAM,oBAAoB,WAAW,aAAa,MAAM,cAAc,MAAM;AAAA;AAAA,EACzH;AAEA,MAAI,oBAAoB,WAAW,YAAY,SAAS,GAAG;AACzD,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,SAAS,iBAAiB,GAAqB;AAC7C,QAAM,WAAgB,aAAQ,CAAC;AAC/B,MAAI;AACF,UAAM,WAAO,0BAAS,QAAQ;AAC9B,QAAI,KAAK,YAAY,GAAG;AACtB,YAAM,cAAU,6BAAY,QAAQ;AACpC,aAAO,QACJ,OAAO,CAAC,MAAM,wBAAwB,KAAK,CAAC,CAAC,EAC7C,IAAI,CAAC,MAAW,UAAK,UAAU,CAAC,CAAC;AAAA,IACtC;AACA,WAAO,CAAC,QAAQ;AAAA,EAClB,QAAQ;AACN,WAAO,CAAC,QAAQ;AAAA,EAClB;AACF;AAEA,SAAS,YAAY,QAA4B;AAC/C,QAAM,QAAkB,CAAC;AACzB,QAAM,cAAc,OAAO,KAAK,OAAO,QAAQ,OAAO;AACtD,QAAM,aAAa,KAAK,IAAI,GAAG,YAAY,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM;AAChF,QAAM,WAAW;AAEjB,QAAM,KAAK,SAAS,OAAO,QAAQ,MAAM,OAAO,OAAO,KAAK,OAAO,QAAQ,KAAK,SAAS;AACzF,QAAM;AAAA,IACJ,KAAK,SAAS,OAAO,UAAU,CAAC,KAAK,OAAO,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC,KAAK,MAAM,SAAS,QAAQ,CAAC;AAAA,EACpL;AAEA,QAAM,UAAU,aAAa,KAAK,WAAW,KAAK;AAClD,QAAM,KAAK,OAAO,SAAS,OAAO,OAAO,CAAC;AAE1C,aAAW,QAAQ,aAAa;AAC9B,UAAM,IAAI,OAAO,QAAQ,QAAQ,IAAI;AAErC,UAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAAE;AACvF,QAAI,oBAAoB,GAAG;AACzB,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC,KAAK,KAAK,SAAS,QAAQ,CAAC;AAAA,MAC1K;AAAA,IACF,OAAO;AACL,YAAM;AAAA,QACJ,KAAK,KAAK,OAAO,UAAU,CAAC,KAAK,EAAE,KAAK,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC,KAAK,EAAE,IAAI,QAAQ,CAAC,EAAE,SAAS,QAAQ,CAAC;AAAA,MACvO;AAAA,IACF;AAAA,EACF;AAEA,QAAM,eAAe,OAAO,WAAW,KAAM,QAAQ,CAAC;AACtD,QAAM,UAAU,OAAO,YAAY,IAAI,IAAI,OAAO,UAAU,QAAQ,CAAC,CAAC,KAAK;AAC3E,QAAM,KAAK,EAAE;AACb,QAAM;AAAA,IACJ,eAAe,OAAO,QAAQ,QAAQ,IAAI,OAAO,QAAQ,KAAK,YAAY,OAAO,gBAAgB,WAAW;AAAA,EAC9G;AAEA,QAAM,kBAAkB,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,cAAc,MAAM;AACzE,MAAI,gBAAgB,SAAS,GAAG;AAC9B,UAAM,eAAe,CAAC,GAAG,IAAI,IAAI,gBAAgB,QAAQ,CAAC,MAAM,EAAE,YAAa,CAAC,CAAC;AACjF,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,oBAAoB,gBAAgB,MAAM,IAAI,OAAO,QAAQ,KAAK;AAAA,IACpE;AACA,eAAW,OAAO,aAAa,MAAM,GAAG,CAAC,GAAG;AAC1C,YAAM,KAAK,SAAS,GAAG,EAAE;AAAA,IAC3B;AACA,QAAI,aAAa,SAAS,GAAG;AAC3B,YAAM,KAAK,eAAe,aAAa,SAAS,CAAC,OAAO;AAAA,IAC1D;AAAA,EACF;AAEA,SAAO,MAAM,KAAK,IAAI;AACxB;AAIA,eAAe,yBAAyB,YAAyC;AAC/E,MAAI;AACF,UAAM,MAAM,MAAM,aAAa,YAAY,YAAY,GAAG;AAC1D,UAAM,UAAU,eAAe,GAAG;AAElC,QAAI,CAAC,WAAW,OAAO,QAAQ,YAAY,YAAY;AACrD,cAAQ,MAAM,mDAAmD;AACjE,UAAI,SAAS;AACX,cAAM,OAAO,OAAO,KAAK,OAAiB,EACvC,MAAM,GAAG,CAAC,EACV,KAAK,IAAI;AACZ,gBAAQ,MAAM,UAAU,OAAO,OAAO,GAAG,OAAO,iBAAiB,IAAI,OAAO,EAAE,EAAE;AAAA,MAClF;AACA,cAAQ;AAAA,QACN;AAAA;AAAA;AAAA,MACF;AACA,cAAQ,KAAK,CAAC;AAAA,IAChB;AAEA,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,QACE,8FAA8F;AAAA,MAC5F;AAAA,IACF,GACA;AACA,YAAM,MAAW,aAAQ,UAAU;AACnC,cAAQ,MAAM,wEAAwE;AACtF,UAAI,QAAQ,SAAS,QAAQ,QAAQ;AACnC,gBAAQ;AAAA,UACN;AAAA,QACF;AAAA,MACF,OAAO;AACL,gBAAQ,MAAM,mDAAmD;AAAA,MACnE;AACA,cAAQ,MAAM;AAAA,IAChB;AACA,YAAQ,MAAM,0BAA0B,GAAG;AAC3C,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,eAAe,WAAW,WAAoB,YAA4C;AAExF,MAAI,cAAc,WAAW,SAAS,GAAG;AACvC,UAAM,mBAAmB,UAAU;AAAA,EACrC;AAGA,MAAI,WAAW;AACb,UAAM,aAAkB,aAAQ,QAAQ,IAAI,GAAG,SAAS;AACxD,UAAM,WAAO,0BAAS,YAAY,EAAE,gBAAgB,MAAM,CAAC;AAC3D,QAAI,CAAC,MAAM,OAAO,GAAG;AACnB,cAAQ,MAAM,0BAA0B,UAAU,EAAE;AACpD,cAAQ,KAAK,CAAC;AAAA,IAChB;AACA,YAAQ,MAAM,kCAAkC,UAAU,EAAE;AAC5D,WAAO,yBAAyB,UAAU;AAAA,EAC5C;AAGA,QAAM,QAAQ,WAAW,QAAQ,IAAI,CAAC;AACtC,MAAI,OAAO;AACT,YAAQ,MAAM,oCAAoC,KAAK,EAAE;AACzD,WAAO,yBAAyB,KAAK;AAAA,EACvC;AAGA,QAAM,EAAE,WAAW,IAAI,MAAM,OAAO,aAAa;AACjD,SAAO,IAAI,WAAW;AACxB;AAIA,SAAS,cAAc,MAKrB;AACA,MAAI;AACJ,MAAI;AACJ,MAAI,aAAuB,CAAC;AAC5B,QAAM,QAAkB,CAAC;AAEzB,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,QAAQ,cAAc,QAAQ,cAAc,QAAQ,gBAAgB;AACtE,UAAI,IAAI,KAAK,KAAK,QAAQ;AACxB,gBAAQ,MAAM,UAAU,GAAG,mBAAmB;AAC9C,gBAAQ,KAAK,CAAC;AAAA,MAChB;AACA,YAAM,QAAQ,KAAK,EAAE,CAAC;AACtB,UAAI,QAAQ,WAAY,cAAa;AAAA,eAC5B,QAAQ,WAAY,aAAY;AAAA;AAEvC,qBAAa,MACV,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,OAAO;AAAA,IACrB,WAAW,IAAI,WAAW,IAAI,GAAG;AAC/B,UAAI,CAAC,YAAY,IAAI,GAAG,GAAG;AACzB,gBAAQ,MAAM,iBAAiB,GAAG,EAAE;AACpC,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF,OAAO;AACL,YAAM,KAAK,GAAG;AAAA,IAChB;AAAA,EACF;AAEA,SAAO,EAAE,YAAY,WAAW,YAAY,MAAM;AACpD;AAIA,eAAe,eAAe,MAAgB;AAC5C,QAAM,EAAE,YAAY,WAAW,YAAY,MAAM,IAAI,cAAc,IAAI;AAEvE,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,MAAM,mCAAmC;AACjD,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,YAAsB,CAAC;AAC7B,aAAW,KAAK,OAAO;AACrB,cAAU,KAAK,GAAG,iBAAiB,CAAC,CAAC;AAAA,EACvC;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,MAAM,4BAA4B;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,QAAM,UAAU,MAAM,WAAW,WAAW,UAAU;AACtD,QAAM,UAAwB,CAAC;AAE/B,MAAI;AACF,eAAW,YAAY,WAAW;AAChC,UAAI;AACF,cAAM,MAAM,MAAM,aAAkB,aAAQ,QAAQ,GAAG,YAAY,GAAG;AACtE,cAAM,aAAyB,IAAI,SAAS,WAAW,IAAI,WAAW,IAAI,UAAU;AAEpF,YAAI,CAAC,WAAW,YAAY,CAAC,WAAW,WAAW,CAAC,WAAW,SAAS;AACtE,kBAAQ;AAAA,YACN,UAAU,QAAQ;AAAA,UACpB;AACA;AAAA,QACF;AAGA,YAAI;AAKJ,YAAI,IAAI,iBAAiB;AAEvB,4BAAkB,OAAO,OAAO,OAAO;AACrC,kBAAM,EAAE,QAAAC,SAAQ,MAAM,YAAY,IAAI,MAAM,QAAQ,UAAU,YAAY;AACxE,qBAAO,IAAI,gBAAgB,OAAO,EAAE;AAAA,YACtC,CAAC;AACD,mBAAO,EAAE,QAAQA,QAAO,QAAQ,MAAMA,QAAO,QAAQ,YAAY;AAAA,UACnE;AAAA,QACF,WAAW,QAAQ,YAAY,WAAW,QAAQ,GAAG;AAEnD,4BAAkB,OAAO,UAAU;AACjC,kBAAM,EAAE,QAAAA,SAAQ,KAAK,IAAI,MAAM,QAAQ,UAAU,YAAY;AAC3D,qBAAO,QAAQ,QAAQ,WAAW,UAAU,KAAK;AAAA,YACnD,CAAC;AACD,mBAAO,EAAE,QAAQA,SAAQ,KAAK;AAAA,UAChC;AAAA,QACF,OAAO;AACL,kBAAQ;AAAA,YACN,uBAAuB,QAAQ;AAAA,UACjC;AACA,4BAAkB,OAAO,WAAW,EAAE,QAAQ,MAAM;AAAA,QACtD;AAEA,cAAM,SAAS,MAAM,QAAQ,YAAY,iBAAiB,OAAO;AACjE,gBAAQ,KAAK,MAAM;AAEnB,gBAAQ,IAAI,OAAO,YAAY,MAAM,IAAI,IAAI;AAAA,MAC/C,SAAS,KAAK;AACZ,gBAAQ;AAAA,UACN,sBAAsB,QAAQ,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc,QAAQ,SAAS,GAAG;AACpC,YAAM,SAAS,QAAQ,WAAW,IAAI,QAAQ,CAAC,IAAI;AACnD,YAAM,YAAiB,aAAa,aAAQ,UAAU,CAAC;AACvD,gBAAM,uBAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAC1C,gBAAM,gBAAAC,WAAe,YAAY,KAAK,UAAU,QAAQ,MAAM,CAAC,GAAG,OAAO;AACzE,cAAQ,IAAI,oBAAoB,UAAU,EAAE;AAAA,IAC9C;AAAA,EACF,UAAE;AACA,UAAM,QAAQ,SAAS,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzC;AAGA,MAAI,QAAQ,WAAW,GAAG;AACxB,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,MAAM,GAAG;AACjB,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["import_node_fs","round","readFileAsync","result","writeFileAsync"]}

package/dist/cli.js

@@ -2,7 +2,7 @@
 import {
   evalCompare,
   runEval
-} from "./chunk-K73SNPVV.js";
+} from "./chunk-OHZX3VFE.js";
 
 // src/cli.ts
 import { readdirSync, statSync } from "fs";

package/dist/index.cjs

@@ -82,6 +82,7 @@ function scorer(config) {
 }
 
 // src/llm-scorer.ts
+var import_axl = require("@axlsdk/axl");
 function llmScorer(config) {
   const scorerInstance = {
     name: config.name,
@@ -113,10 +114,10 @@ ${JSON.stringify(annotations, null, 2)}` : "",
           { role: "system", content: config.system },
           { role: "user", content: prompt }
         ],
-        { model, temperature: config.temperature ?? 0.2 }
+        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: "json_object" } }
       );
       scorerInstance._lastCost = response.cost;
-      const parsed = JSON.parse(response.content);
+      const parsed = JSON.parse((0, import_axl.extractJson)(response.content));
       const validated = config.schema.parse(parsed);
       return validated.score;
     }
@@ -213,7 +214,7 @@ async function runEval(config, executeWorkflow, runtime) {
             totalCost += lastCost;
           }
         }
-        if (score < 0 || score > 1) {
+        if (!Number.isFinite(score) || score < 0 || score > 1) {
           if (!evalItem.scorerErrors) evalItem.scorerErrors = [];
           evalItem.scorerErrors.push(
             `Scorer "${scorer2.name}" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`

package/dist/index.cjs.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/runner.ts","../src/compare.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerContext, ScorerFn } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: { model: string; temperature?: number },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer, ScorerContext } from './scorer.js';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _lastCost?: number } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<number> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2 },\n      );\n\n      scorerInstance._lastCost = response.cost;\n\n      const parsed = JSON.parse(response.content);\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACvBO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACnCO,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAAkD;AAAA,IACtD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACiB;AACjB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,IAAI;AAAA,MAClD;AAEA,qBAAe,YAAY,SAAS;AAEpC,YAAM,SAAS,KAAK,MAAM,SAAS,OAAO;AAC1C,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AC9DO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACDA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAMA,QAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAIA,QAAO,OAAO;AAChB,gBAAM,WAAYA,QAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAIA,QAAO,OAAO;AAChB,gBAAM,WAAYA,QAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;AC1KA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQA,OAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;","names":["scorer","round"]}
+{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/runner.ts","../src/compare.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerContext, ScorerFn } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { extractJson } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _lastCost?: number } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<number> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n      );\n\n      scorerInstance._lastCost = response.cost;\n\n      const parsed = JSON.parse(extractJson(response.content));\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!Number.isFinite(score) || score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACnBO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACjDA,iBAA4B;AAWrB,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAAkD;AAAA,IACtD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACiB;AACjB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,qBAAe,YAAY,SAAS;AAEpC,YAAM,SAAS,KAAK,UAAM,wBAAY,SAAS,OAAO,CAAC;AACvD,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AC/DO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACDA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAMA,QAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAIA,QAAO,OAAO;AAChB,gBAAM,WAAYA,QAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AACrD,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAIA,QAAO,OAAO;AAChB,gBAAM,WAAYA,QAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;AC1KA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQA,OAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;","names":["scorer","round"]}

package/dist/index.d.cts

@@ -33,6 +33,10 @@ type ScorerContext = {
             }[], options: {
                 model: string;
                 temperature?: number;
+                responseFormat?: {
+                    type: string;
+                    json_schema?: unknown;
+                };
             }): Promise<{
                 content: string;
                 cost?: number;

package/dist/index.d.ts

@@ -33,6 +33,10 @@ type ScorerContext = {
             }[], options: {
                 model: string;
                 temperature?: number;
+                responseFormat?: {
+                    type: string;
+                    json_schema?: unknown;
+                };
             }): Promise<{
                 content: string;
                 cost?: number;

package/dist/index.js

@@ -1,7 +1,7 @@
 import {
   evalCompare,
   runEval
-} from "./chunk-K73SNPVV.js";
+} from "./chunk-OHZX3VFE.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
@@ -46,6 +46,7 @@ function scorer(config) {
 }
 
 // src/llm-scorer.ts
+import { extractJson } from "@axlsdk/axl";
 function llmScorer(config) {
   const scorerInstance = {
     name: config.name,
@@ -77,10 +78,10 @@ ${JSON.stringify(annotations, null, 2)}` : "",
           { role: "system", content: config.system },
           { role: "user", content: prompt }
         ],
-        { model, temperature: config.temperature ?? 0.2 }
+        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: "json_object" } }
       );
       scorerInstance._lastCost = response.cost;
-      const parsed = JSON.parse(response.content);
+      const parsed = JSON.parse(extractJson(response.content));
       const validated = config.schema.parse(parsed);
       return validated.score;
     }

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: { model: string; temperature?: number },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer, ScorerContext } from './scorer.js';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _lastCost?: number } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<number> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2 },\n      );\n\n      scorerInstance._lastCost = response.cost;\n\n      const parsed = JSON.parse(response.content);\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACvBO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACnCO,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAAkD;AAAA,IACtD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACiB;AACjB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,IAAI;AAAA,MAClD;AAEA,qBAAe,YAAY,SAAS;AAEpC,YAAM,SAAS,KAAK,MAAM,SAAS,OAAO;AAC1C,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AC9DO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { extractJson } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _lastCost?: number } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<number> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n      );\n\n      scorerInstance._lastCost = response.cost;\n\n      const parsed = JSON.parse(extractJson(response.content));\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACnBO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACjDA,SAAS,mBAAmB;AAWrB,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAAkD;AAAA,IACtD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACiB;AACjB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,qBAAe,YAAY,SAAS;AAEpC,YAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AC/DO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.13.1",
+  "version": "0.13.3",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.13.1"
+    "@axlsdk/axl": "0.13.3"
   },
   "devDependencies": {
     "@types/node": "^25.2.3",

package/dist/chunk-K73SNPVV.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/compare.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9EA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;","names":["round"]}