@axlsdk/eval 0.10.3 → 0.11.0

package/README.md

@@ -85,7 +85,7 @@ const qualityJudge = llmScorer({
 });
 ```
 
-### `runEval(config, executeFn, provider?, runtime?)`
+### `runEval(config, executeFn, provider, runtime)`
 
 Run an evaluation:
 
@@ -103,6 +103,8 @@ const results = await runEval(
     const output = await runtime.execute('my-workflow', input);
     return { output };
   },
+  undefined,  // provider (only needed for llmScorer)
+  runtime,
 );
 
 console.log(results.summary);
@@ -158,11 +160,11 @@ export default defineEval({
 });
 ```
 
-By default, the eval runner calls `runtime.execute(workflow, input)` for each dataset item, which requires the workflow to be registered on the runtime.
+When running via `runtime.eval()` or `runtime.runRegisteredEval()`, the runner calls `runtime.execute(workflow, input)` by default, which requires the workflow to be registered on the runtime.
 
-For self-contained evals, export an `executeWorkflow` function alongside the config. This is called instead of `runtime.execute()` for each dataset item.
+When running via the CLI, the eval runner resolves the execution function in order: (1) the exported `executeWorkflow` function, (2) `runtime.execute(workflow, input)` if the workflow is registered on the runtime, or (3) a passthrough that returns the input as the output (with a warning).
 
-The function receives `(input, runtime?)` — the second argument is the `AxlRuntime` instance when running via Studio or `runtime.runRegisteredEval()`. It is `undefined` when running via the CLI. Use it to call agents without needing a registered workflow:
+The function receives `(input, runtime)` — the second argument is the `AxlRuntime` instance, always provided by the CLI, Studio, and `runtime.runRegisteredEval()`. Use it to call agents without needing a registered workflow:
 
 ```typescript
 // evals/qa-quality.eval.ts — calls an agent directly via the runtime
@@ -192,15 +194,14 @@ export default defineEval({
   scorers: [notEmpty],
 });
 
-export async function executeWorkflow(input: { question: string }, runtime?: AxlRuntime) {
-  if (!runtime) throw new Error('This eval requires a runtime — run via Studio or runtime.runRegisteredEval()');
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
   const ctx = runtime.createContext();
   const output = await ctx.ask(qaAgent, input.question);
-  return { output };
+  return { output }; // cost tracked automatically via runtime.trackCost()
 }
 ```
 
-When no runtime is needed (e.g., testing a pure function), just ignore the second argument — these evals work from both the CLI and Studio:
+When no runtime is needed (e.g., testing a pure function), omit the runtime parameter — these evals work from both the CLI and Studio:
 
 ```typescript
 // evals/parser.eval.ts — tests a pure function, no runtime needed
@@ -210,9 +211,81 @@ export async function executeWorkflow(input: { raw: string }) {
 }
 ```
 
-> **Note:** The CLI (`npx axl eval`) does not provide a runtime to `executeWorkflow`. Eval files that call agents via `runtime.createContext()` must be run through Studio or `runtime.runRegisteredEval()`.
+> **Note:** The CLI always provides a runtime to `executeWorkflow`. By default it auto-detects `axl.config.*` in the working directory or falls back to a bare `AxlRuntime` (providers resolved from environment variables). Use `--config` for explicit config files.
+
+### Cost Tracking
+
+Cost is tracked automatically. You don't need to return `cost` from your `executeWorkflow` — the eval runner wraps each item with `runtime.trackCost()`, which captures cost from all `createContext()` and `execute()` calls.
+
+```typescript
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
+  const ctx = runtime.createContext();
+  const output = await ctx.ask(myAgent, input.question);
+  return { output }; // cost captured automatically
+}
+```
+
+To override the automatic cost (e.g., to exclude setup calls), return it explicitly:
+
+```typescript
+return { output, cost: ctx.totalCost };
+```
+
+You can also read `ctx.totalCost` at any point to inspect accumulated cost.
+
+### Common Patterns
+
+**Budget per item** — cap cost to prevent runaway evals:
+
+```typescript
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
+  const ctx = runtime.createContext({ budget: '$0.50' });
+  const output = await ctx.ask(myAgent, input.question);
+  return { output };
+}
+```
+
+**Timeout per item** — cancel items that take too long:
+
+```typescript
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
+  const controller = new AbortController();
+  setTimeout(() => controller.abort(), 30_000);
+  const ctx = runtime.createContext({ signal: controller.signal });
+  const output = await ctx.ask(myAgent, input.question);
+  return { output };
+}
+```
+
+**Auto-approve tools** — when testing agents that have tools with `requireApproval`:
+
+```typescript
+export async function executeWorkflow(input: { question: string }, runtime: AxlRuntime) {
+  const ctx = runtime.createContext({
+    awaitHumanHandler: async () => ({ approved: true }),
+  });
+  const output = await ctx.ask(myAgent, input.question);
+  return { output };
+}
+```
 
-> **Cost tracking:** The default path (no `executeWorkflow` export) tracks cost automatically via runtime trace events. Custom `executeWorkflow` functions must return `{ output, cost }` manually if cost tracking is needed — `createContext()` does not capture cost automatically.
+**Multi-turn evaluation** — test follow-up responses with conversation history:
+
+```typescript
+export async function executeWorkflow(
+  input: { setupPrompt: string; setupResponse: string; followUp: string },
+  runtime: AxlRuntime,
+) {
+  const ctx = runtime.createContext({
+    sessionHistory: [
+      { role: 'user', content: input.setupPrompt },
+      { role: 'assistant', content: input.setupResponse },
+    ],
+  });
+  const output = await ctx.ask(myAgent, input.followUp);
+  return { output };
+}
+```
 
 ## CLI
 
@@ -220,18 +293,39 @@ Run evaluations from the command line:
 
 ```bash
 # Run an eval file
-npx axl eval ./evals/math.eval.ts
+npx axl-eval ./evals/math.eval.ts
 
 # Run all evals in a directory
-npx axl eval ./evals/
+npx axl-eval ./evals/
 
 # Save results to a file
-npx axl eval ./evals/math.eval.ts --output ./results/baseline.json
+npx axl-eval ./evals/math.eval.ts --output ./results/baseline.json
+
+# Use a config file for runtime access
+npx axl-eval ./evals/ --config ./axl.config.ts
+
+# Use custom import conditions (monorepo source exports)
+npx axl-eval ./evals/ --config ./axl.config.ts --conditions development
 
 # Compare two results
-npx axl eval compare ./results/baseline.json ./results/candidate.json
+npx axl-eval compare ./results/baseline.json ./results/candidate.json
+
+# Fail CI if regressions detected
+npx axl-eval compare baseline.json candidate.json --fail-on-regression
 ```
 
+TypeScript config and eval files require [tsx](https://github.com/privatenumber/tsx) as a dev dependency (`npm install -D tsx`).
+
+### Runtime Resolution
+
+The CLI resolves an `AxlRuntime` to pass to `executeWorkflow`:
+
+1. **`--config <path>`** — use an explicit config file
+2. **Auto-detect** — search for `axl.config.mts` → `.ts` → `.mjs` → `.js` in cwd
+3. **Fallback** — create a bare `new AxlRuntime()` (providers resolve from environment variables)
+
+When a runtime is available, each `executeWorkflow` call is wrapped with `runtime.trackCost()` for automatic cost attribution.
+
 ## Studio Integration
 
 Eval files can also be lazy-loaded by the Studio middleware, enabling the Eval Runner panel without static imports that would create circular dependencies:

package/dist/{chunk-LMQJNZWT.js → chunk-D7KCUS6H.js}

@@ -195,4 +195,4 @@ export {
   evalCompare,
   runEval
 };
-//# sourceMappingURL=chunk-LMQJNZWT.js.map
+//# sourceMappingURL=chunk-D7KCUS6H.js.map

package/dist/chunk-D7KCUS6H.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/compare.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/** Minimal provider interface for LLM scoring (avoids depending on core axl package). */\ntype LlmScorerProvider = {\n  chat(\n    messages: { role: string; content: string }[],\n    options: { model: string; temperature?: number },\n  ): Promise<{ content: string }>;\n};\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  provider: LlmScorerProvider | undefined,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as unknown as { _provider: LlmScorerProvider })._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC/EA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAUA,eAAsB,QACpB,QACA,iBAIA,UACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI,OAAO,OAAO;AAChB,QAAC,OAAuD,YAAY;AAAA,MACtE;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;","names":["round"]}

package/dist/cli.cjs

@@ -24,9 +24,10 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/cli.ts
-var import_node_fs = require("fs");
+var import_node_fs2 = require("fs");
 var import_promises = require("fs/promises");
 var path = __toESM(require("path"), 1);
+var import_node_url2 = require("url");
 
 // src/compare.ts
 function round(n) {
@@ -221,16 +222,110 @@ async function runEval(config, executeWorkflow, provider, runtime) {
   };
 }
 
+// src/cli-utils.ts
+var import_node_path = require("path");
+var import_node_fs = require("fs");
+var import_node_url = require("url");
+var CONFIG_CANDIDATES = [
+  "axl.config.mts",
+  "axl.config.ts",
+  "axl.config.mjs",
+  "axl.config.js"
+];
+function findConfig(cwd) {
+  for (const name of CONFIG_CANDIDATES) {
+    const p = (0, import_node_path.resolve)(cwd, name);
+    if ((0, import_node_fs.existsSync)(p)) return p;
+  }
+  return void 0;
+}
+function needsEsmForcing(configPath) {
+  const ext = (0, import_node_path.extname)(configPath);
+  return ext === ".ts" || ext === ".tsx";
+}
+function needsTsxLoader(configPath) {
+  return /\.[mc]?tsx?$/.test(configPath);
+}
+function resolveRuntime(mod) {
+  const def = mod.default;
+  return def?.default ?? def ?? mod.runtime;
+}
+var tsxRegistered = false;
+async function ensureTsxLoader() {
+  if (tsxRegistered) return;
+  tsxRegistered = true;
+  let loaded = false;
+  try {
+    const tsxEsm = await import("tsx/esm/api");
+    tsxEsm.register();
+    loaded = true;
+  } catch {
+  }
+  try {
+    const tsxCjs = await import("tsx/cjs/api");
+    tsxCjs.register();
+    loaded = true;
+  } catch {
+  }
+  if (!loaded) {
+    console.warn(
+      "[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\n  Install it with: npm install -D tsx"
+    );
+  }
+}
+async function forceEsmForConfig(configPath) {
+  try {
+    const nodeModule = await import("module");
+    const configUrl = (0, import_node_url.pathToFileURL)(configPath).href;
+    const hookCode = [
+      `export async function resolve(specifier, context, nextResolve) {`,
+      `  const result = await nextResolve(specifier, context);`,
+      `  if (result.url === ${JSON.stringify(configUrl)}) result.format = 'module';`,
+      `  return result;`,
+      `}`
+    ].join("\n");
+    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
+  } catch {
+  }
+}
+async function registerConditions(conditions) {
+  try {
+    const nodeModule = await import("module");
+    const hookCode = [
+      `const extra = ${JSON.stringify(conditions)};`,
+      `export async function resolve(specifier, context, nextResolve) {`,
+      `  return nextResolve(specifier, {`,
+      `    ...context,`,
+      `    conditions: [...new Set([...context.conditions, ...extra])],`,
+      `  });`,
+      `}`
+    ].join("\n");
+    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
+  } catch {
+    console.warn("[axl-eval] Warning: --conditions requires Node.js 20.6+");
+  }
+}
+
 // src/cli.ts
+var KNOWN_FLAGS = /* @__PURE__ */ new Set(["--output", "--config", "--conditions", "--fail-on-regression"]);
 async function main() {
   const args = process.argv.slice(2);
-  if (args.length === 0 || args[0] === "--help") {
+  if (args.length === 0 || args[0] === "--help" || args[0] === "-h") {
     console.log(`
 Usage:
-  npx axl eval <path>              Run eval file(s)
-  npx axl eval <path> --output <f> Run eval and save results to JSON
-  npx axl eval compare <a> <b>     Compare two eval result files
-  npx axl eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
+  axl-eval <path>                         Run eval file(s)
+  axl-eval <path> --output <file>         Save results to JSON
+  axl-eval <path> --config <file>         Use config file for runtime
+  axl-eval <path> --conditions <list>     Node.js import conditions (comma-separated)
+  axl-eval compare <a> <b>                Compare two eval result files
+  axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
+
+Config auto-detection (when --config is not specified):
+  ${CONFIG_CANDIDATES.join(" -> ")}
+
+When a config is found, the exported AxlRuntime is passed to executeWorkflow
+and cost is tracked automatically via runtime.trackCost().
+When no config is found, a bare AxlRuntime is created (providers from env vars).
 `);
     process.exit(0);
   }
@@ -244,7 +339,7 @@ async function runCompare(args) {
   const failOnRegression = args.includes("--fail-on-regression");
   const files = args.filter((a) => !a.startsWith("--"));
   if (files.length !== 2) {
-    console.error("Usage: npx axl eval compare <baseline.json> <candidate.json>");
+    console.error("Usage: axl-eval compare <baseline.json> <candidate.json>");
     process.exit(1);
   }
   const baseline = JSON.parse(await (0, import_promises.readFile)(files[0], "utf-8"));
@@ -282,10 +377,10 @@ Compare: baseline (${comparison.baseline.id.slice(0, 8)}) -> candidate (${compar
 function collectEvalFiles(p) {
   const resolved = path.resolve(p);
   try {
-    const stat = (0, import_node_fs.statSync)(resolved);
+    const stat = (0, import_node_fs2.statSync)(resolved);
     if (stat.isDirectory()) {
-      const entries = (0, import_node_fs.readdirSync)(resolved);
-      return entries.filter((e) => e.endsWith(".eval.ts") || e.endsWith(".eval.js")).map((e) => path.join(resolved, e));
+      const entries = (0, import_node_fs2.readdirSync)(resolved);
+      return entries.filter((e) => /\.eval\.[mc]?[jt]sx?$/.test(e)).map((e) => path.join(resolved, e));
     }
     return [resolved];
   } catch {
@@ -317,16 +412,104 @@ function formatTable(result) {
   );
   return lines.join("\n");
 }
-async function runEvalCommand(args) {
+async function resolveRuntimeFromConfig(configPath) {
+  if (needsTsxLoader(configPath)) {
+    await ensureTsxLoader();
+  }
+  if (needsEsmForcing(configPath)) {
+    await forceEsmForConfig(configPath);
+  }
+  try {
+    const mod = await import((0, import_node_url2.pathToFileURL)(configPath).href);
+    const runtime = resolveRuntime(mod);
+    if (!runtime || typeof runtime.execute !== "function") {
+      console.error(`Config must export a default AxlRuntime instance.`);
+      if (runtime) {
+        const keys = Object.keys(runtime).slice(0, 5).join(", ");
+        console.error(`  Got: ${typeof runtime}${keys ? ` with keys: { ${keys} }` : ""}`);
+      }
+      console.error(
+        `Example:
+  import { AxlRuntime } from '@axlsdk/axl';
+  export default new AxlRuntime({ ... });`
+      );
+      process.exit(1);
+    }
+    return runtime;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    if (/Cannot use import statement|Unexpected reserved word|top-level await|exports is not defined/.test(
+      msg
+    )) {
+      const ext = path.extname(configPath);
+      console.error(`[axl-eval] Config failed to load due to a CJS/ESM compatibility issue.`);
+      if (ext === ".ts" || ext === ".tsx") {
+        const mtsPath = configPath.slice(0, -ext.length) + ".mts";
+        console.error(
+          `  Tip: rename to .mts to force ESM format:
+    mv ${configPath} ${mtsPath}`
+        );
+      } else {
+        console.error(`  Tip: add "type": "module" to your package.json.`);
+      }
+      console.error();
+    }
+    console.error(`Failed to load config:`, err);
+    process.exit(1);
+  }
+}
+async function getRuntime(configArg, conditions) {
+  if (conditions && conditions.length > 0) {
+    await registerConditions(conditions);
+  }
+  if (configArg) {
+    const configPath = path.resolve(process.cwd(), configArg);
+    const stat = (0, import_node_fs2.statSync)(configPath, { throwIfNoEntry: false });
+    if (!stat?.isFile()) {
+      console.error(`Config file not found: ${configPath}`);
+      process.exit(1);
+    }
+    console.error(`[axl-eval] Loading config from ${configPath}`);
+    return resolveRuntimeFromConfig(configPath);
+  }
+  const found = findConfig(process.cwd());
+  if (found) {
+    console.error(`[axl-eval] Auto-detected config: ${found}`);
+    return resolveRuntimeFromConfig(found);
+  }
+  const { AxlRuntime } = await import("@axlsdk/axl");
+  return new AxlRuntime();
+}
+function parseEvalArgs(args) {
   let outputPath;
+  let configArg;
+  let conditions = [];
   const paths = [];
   for (let i = 0; i < args.length; i++) {
-    if (args[i] === "--output" && i + 1 < args.length) {
-      outputPath = args[++i];
-    } else if (!args[i].startsWith("--")) {
-      paths.push(args[i]);
+    const arg = args[i];
+    if (arg === "--output" || arg === "--config" || arg === "--conditions") {
+      if (i + 1 >= args.length) {
+        console.error(`Error: ${arg} requires a value`);
+        process.exit(1);
+      }
+      const value = args[++i];
+      if (arg === "--output") outputPath = value;
+      else if (arg === "--config") configArg = value;
+      else
+        conditions = value.split(",").map((c) => c.trim()).filter(Boolean);
+    } else if (arg.startsWith("--")) {
+      if (!KNOWN_FLAGS.has(arg)) {
+        console.error(`Unknown flag: ${arg}`);
+        process.exit(1);
+      }
+    } else {
+      paths.push(arg);
     }
   }
+  return { outputPath, configArg, conditions, paths };
+}
+async function runEvalCommand(args) {
+  const { outputPath, configArg, conditions, paths } = parseEvalArgs(args);
   if (paths.length === 0) {
     console.error("Error: No eval file path provided");
     process.exit(1);
@@ -339,33 +522,65 @@ async function runEvalCommand(args) {
     console.error("Error: No eval files found");
     process.exit(1);
   }
+  if (evalFiles.some((f) => /\.[mc]?tsx?$/.test(f))) {
+    await ensureTsxLoader();
+  }
+  const runtime = await getRuntime(configArg, conditions);
   const results = [];
-  for (const filePath of evalFiles) {
-    try {
-      const mod = await import(path.resolve(filePath));
-      const evalConfig = mod.default ?? mod.config ?? mod;
-      if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+  try {
+    for (const filePath of evalFiles) {
+      try {
+        const mod = await import((0, import_node_url2.pathToFileURL)(path.resolve(filePath)).href);
+        const evalConfig = mod.default ?? mod.config ?? mod;
+        if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+          console.error(
+            `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
+          );
+          continue;
+        }
+        let executeWorkflow;
+        if (mod.executeWorkflow) {
+          executeWorkflow = async (input, rt) => {
+            const { result: result2, cost: trackedCost } = await runtime.trackCost(async () => {
+              return mod.executeWorkflow(input, rt);
+            });
+            return { output: result2.output, cost: result2.cost ?? trackedCost };
+          };
+        } else if (runtime.getWorkflow(evalConfig.workflow)) {
+          executeWorkflow = async (input) => {
+            const { result: result2, cost } = await runtime.trackCost(async () => {
+              return runtime.execute(evalConfig.workflow, input);
+            });
+            return { output: result2, cost };
+          };
+        } else {
+          console.warn(
+            `[axl-eval] Warning: ${filePath} does not export executeWorkflow \u2014 using input passthrough`
+          );
+          executeWorkflow = async (input) => ({ output: input });
+        }
+        const result = await runEval(evalConfig, executeWorkflow, mod.provider, runtime);
+        results.push(result);
+        console.log("\n" + formatTable(result) + "\n");
+      } catch (err) {
         console.error(
-          `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
+          `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`
         );
-        continue;
       }
-      const executeWorkflow = mod.executeWorkflow ?? (async (input) => ({ output: input }));
-      const result = await runEval(evalConfig, executeWorkflow, mod.provider);
-      results.push(result);
-      console.log("\n" + formatTable(result) + "\n");
-    } catch (err) {
-      console.error(
-        `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`
-      );
     }
+    if (outputPath && results.length > 0) {
+      const output = results.length === 1 ? results[0] : results;
+      const outputDir = path.dirname(path.resolve(outputPath));
+      await (0, import_promises.mkdir)(outputDir, { recursive: true });
+      await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
+      console.log(`Results saved to ${outputPath}`);
+    }
+  } finally {
+    await runtime.shutdown().catch(() => {
+    });
   }
-  if (outputPath && results.length > 0) {
-    const output = results.length === 1 ? results[0] : results;
-    const outputDir = path.dirname(path.resolve(outputPath));
-    await (0, import_promises.mkdir)(outputDir, { recursive: true });
-    await (0, import_promises.writeFile)(outputPath, JSON.stringify(output, null, 2), "utf-8");
-    console.log(`Results saved to ${outputPath}`);
+  if (results.length === 0) {
+    process.exit(1);
   }
 }
 main().catch((err) => {