@axlsdk/eval 0.1.0

package/dist/index.cjs

@@ -0,0 +1,334 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  dataset: () => dataset,
+  defineEval: () => defineEval,
+  evalCompare: () => evalCompare,
+  llmScorer: () => llmScorer,
+  runEval: () => runEval,
+  scorer: () => scorer
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/dataset.ts
+var import_promises = require("fs/promises");
+var path = __toESM(require("path"), 1);
+function dataset(config) {
+  return {
+    name: config.name,
+    schema: config.schema,
+    annotationsSchema: config.annotations,
+    async getItems() {
+      if (config.items && config.file) {
+        throw new Error('Dataset config error: "items" and "file" are mutually exclusive');
+      }
+      if (!config.items && !config.file) {
+        throw new Error('Dataset config error: either "items" or "file" must be provided');
+      }
+      if (config.items) {
+        return config.items.map((item) => ({
+          input: config.schema.parse(item.input),
+          annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
+        }));
+      }
+      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
+      const content = await (0, import_promises.readFile)(resolvedPath, "utf-8");
+      const items = JSON.parse(content);
+      return items.map((item) => ({
+        input: config.schema.parse(item.input),
+        annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
+      }));
+    }
+  };
+}
+
+// src/scorer.ts
+function scorer(config) {
+  return {
+    name: config.name,
+    description: config.description,
+    isLlm: false,
+    score: config.score
+  };
+}
+
+// src/llm-scorer.ts
+function llmScorer(config) {
+  const scorerInstance = {
+    name: config.name,
+    description: config.description,
+    isLlm: true,
+    async score(output, input, annotations) {
+      const injectedProvider = scorerInstance._provider;
+      if (!injectedProvider) {
+        throw new Error(
+          `LLM scorer "${config.name}" requires a provider. Run via axl.eval() or the CLI.`
+        );
+      }
+      const prompt = [
+        `Evaluate the following output.`,
+        ``,
+        `## Input`,
+        `${JSON.stringify(input, null, 2)}`,
+        annotations ? `
+## Annotations (Ground Truth)
+${JSON.stringify(annotations, null, 2)}` : "",
+        ``,
+        `## Output to Evaluate`,
+        `${JSON.stringify(output, null, 2)}`,
+        ``,
+        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`
+      ].join("\n");
+      const colonIdx = config.model.indexOf(":");
+      const model = colonIdx > -1 ? config.model.slice(colonIdx + 1) : config.model;
+      const response = await injectedProvider.chat(
+        [
+          { role: "system", content: config.system },
+          { role: "user", content: prompt }
+        ],
+        { model, temperature: config.temperature ?? 0.2 }
+      );
+      const parsed = JSON.parse(response.content);
+      const validated = config.schema.parse(parsed);
+      return validated.score;
+    }
+  };
+  return scorerInstance;
+}
+
+// src/define-eval.ts
+function defineEval(config) {
+  return config;
+}
+
+// src/runner.ts
+var import_node_crypto = require("crypto");
+function computeStats(scores) {
+  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };
+  const sorted = [...scores].sort((a, b) => a - b);
+  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const min = sorted[0];
+  const max = sorted[sorted.length - 1];
+  const p50 = sorted[Math.floor(sorted.length * 0.5)];
+  const p95 = sorted[Math.floor(sorted.length * 0.95)];
+  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };
+}
+function round(n) {
+  return Math.round(n * 1e3) / 1e3;
+}
+function parseCost(cost) {
+  const match = cost.match(/^\$?([\d.]+)$/);
+  if (!match) throw new Error(`Invalid cost format: "${cost}"`);
+  return parseFloat(match[1]);
+}
+async function runEval(config, executeWorkflow, provider) {
+  const startTime = Date.now();
+  const id = (0, import_node_crypto.randomUUID)();
+  const items = await config.dataset.getItems();
+  const concurrency = config.concurrency ?? 5;
+  const budgetLimit = config.budget ? parseCost(config.budget) : void 0;
+  if (provider) {
+    for (const scorer2 of config.scorers) {
+      if (scorer2.isLlm) {
+        scorer2._provider = provider;
+      }
+    }
+  }
+  const evalItems = [];
+  let totalCost = 0;
+  let budgetExceeded = false;
+  async function processItem(item) {
+    if (budgetExceeded) {
+      evalItems.push({
+        input: item.input,
+        annotations: item.annotations,
+        output: null,
+        error: "Budget exceeded",
+        scores: {}
+      });
+      return;
+    }
+    const evalItem = {
+      input: item.input,
+      annotations: item.annotations,
+      output: null,
+      scores: {}
+    };
+    try {
+      const result = await executeWorkflow(item.input);
+      evalItem.output = result.output;
+      if (result.cost != null) {
+        totalCost += result.cost;
+      }
+    } catch (err) {
+      evalItem.error = err instanceof Error ? err.message : String(err);
+      evalItems.push(evalItem);
+      return;
+    }
+    if (budgetLimit != null && totalCost > budgetLimit) {
+      budgetExceeded = true;
+    }
+    for (const scorer2 of config.scorers) {
+      try {
+        const score = await scorer2.score(evalItem.output, item.input, item.annotations);
+        if (score < 0 || score > 1) {
+          if (!evalItem.errors) evalItem.errors = [];
+          evalItem.errors.push(
+            `Scorer "${scorer2.name}" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`
+          );
+          evalItem.scores[scorer2.name] = -1;
+        } else {
+          evalItem.scores[scorer2.name] = round(score);
+        }
+      } catch (err) {
+        if (!evalItem.errors) evalItem.errors = [];
+        evalItem.errors.push(
+          `Scorer "${scorer2.name}" threw: ${err instanceof Error ? err.message : String(err)}`
+        );
+        evalItem.scores[scorer2.name] = -1;
+      }
+    }
+    evalItems.push(evalItem);
+  }
+  let index = 0;
+  async function runNext() {
+    while (index < items.length) {
+      const currentIndex = index++;
+      await processItem(items[currentIndex]);
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());
+  await Promise.all(workers);
+  const failures = evalItems.filter((i) => i.error).length;
+  const scorerNames = config.scorers.map((s) => s.name);
+  const scorerStats = {};
+  for (const name of scorerNames) {
+    const scores = evalItems.filter((i) => !i.error && i.scores[name] !== void 0 && i.scores[name] >= 0).map((i) => i.scores[name]);
+    scorerStats[name] = computeStats(scores);
+  }
+  return {
+    id,
+    workflow: config.workflow,
+    dataset: config.dataset.name,
+    metadata: config.metadata ?? {},
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    totalCost,
+    duration: Date.now() - startTime,
+    items: evalItems,
+    summary: { count: items.length, failures, scorers: scorerStats }
+  };
+}
+
+// src/compare.ts
+function round2(n) {
+  return Math.round(n * 100) / 100;
+}
+function evalCompare(baseline, candidate) {
+  if (baseline.dataset !== candidate.dataset) {
+    throw new Error(
+      `Cannot compare evals from different datasets: "${baseline.dataset}" vs "${candidate.dataset}"`
+    );
+  }
+  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();
+  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();
+  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {
+    throw new Error(`Cannot compare evals with different scorers`);
+  }
+  const scorers = {};
+  for (const name of baselineScorerNames) {
+    const bMean = baseline.summary.scorers[name].mean;
+    const cMean = candidate.summary.scorers[name].mean;
+    const delta = round2(cMean - bMean);
+    const deltaPercent = bMean > 0 ? round2(delta / bMean * 100) : 0;
+    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };
+  }
+  const regressions = [];
+  const improvements = [];
+  const minLength = Math.min(baseline.items.length, candidate.items.length);
+  for (let i = 0; i < minLength; i++) {
+    const bItem = baseline.items[i];
+    const cItem = candidate.items[i];
+    if (bItem.error || cItem.error) continue;
+    for (const name of baselineScorerNames) {
+      const bScore = bItem.scores[name];
+      const cScore = cItem.scores[name];
+      if (bScore === void 0 || cScore === void 0 || bScore < 0 || cScore < 0) continue;
+      const delta = round2(cScore - bScore);
+      if (delta < -0.1)
+        regressions.push({
+          input: bItem.input,
+          scorer: name,
+          baselineScore: bScore,
+          candidateScore: cScore,
+          delta
+        });
+      else if (delta > 0.1)
+        improvements.push({
+          input: bItem.input,
+          scorer: name,
+          baselineScore: bScore,
+          candidateScore: cScore,
+          delta
+        });
+    }
+  }
+  const parts = [];
+  for (const name of baselineScorerNames) {
+    const s = scorers[name];
+    if (Math.abs(s.delta) > 1e-3) {
+      const direction = s.delta > 0 ? "improves" : "regresses";
+      const sign = s.delta > 0 ? "+" : "";
+      parts.push(
+        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`
+      );
+    }
+  }
+  const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no significant changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  return {
+    baseline: { id: baseline.id, metadata: baseline.metadata },
+    candidate: { id: candidate.id, metadata: candidate.metadata },
+    scorers,
+    regressions,
+    improvements,
+    summary: summaryStr
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  dataset,
+  defineEval,
+  evalCompare,
+  llmScorer,
+  runEval,
+  scorer
+});
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/runner.ts","../src/compare.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerFn } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\n\nexport type {\n  EvalConfig,\n  EvalResult,\n  EvalItem,\n  EvalSummary,\n  EvalComparison,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<\n  TInput extends z.ZodTypeAny,\n  TAnnotations extends z.ZodTypeAny = z.ZodTypeAny,\n> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodTypeAny;\n  readonly annotationsSchema?: z.ZodTypeAny;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodTypeAny, TAnnotations extends z.ZodTypeAny>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","export type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = any, TInput = any, TAnnotations = any> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = any, TInput = any, TAnnotations = any> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(output: TOutput, input: TInput, annotations?: TAnnotations): number | Promise<number>;\n};\n\nexport function scorer<TOutput = any, TInput = any, TAnnotations = any>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer } from './scorer.js';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: any }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _provider?: any } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(output: any, input: any, annotations?: any): Promise<number> {\n      const injectedProvider = (scorerInstance as any)._provider;\n      if (!injectedProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" requires a provider. Run via axl.eval() or the CLI.`,\n        );\n      }\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const colonIdx = config.model.indexOf(':');\n      const model = colonIdx > -1 ? config.model.slice(colonIdx + 1) : config.model;\n\n      const response = await injectedProvider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2 },\n      );\n\n      const parsed = JSON.parse(response.content);\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n","import type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (input: unknown) => Promise<{ output: unknown; cost?: number }>,\n  provider?: any,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  if (provider) {\n    for (const scorer of config.scorers) {\n      if (scorer.isLlm) {\n        (scorer as any)._provider = provider;\n      }\n    }\n  }\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(evalItem.output, item.input, item.annotations);\n        if (score < 0 || score > 1) {\n          if (!evalItem.errors) evalItem.errors = [];\n          evalItem.errors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = -1;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        if (!evalItem.errors) evalItem.errors = [];\n        evalItem.errors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = -1;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] !== undefined && i.scores[name] >= 0)\n      .map((i) => i.scores[name]);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore === undefined || cScore === undefined || bScore < 0 || cScore < 0) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AA2Bf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC7CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AChBO,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAA+C;AAAA,IACnD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MAAM,QAAa,OAAY,aAAoC;AACvE,YAAM,mBAAoB,eAAuB;AACjD,UAAI,CAAC,kBAAkB;AACrB,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAC5B;AAAA,MACF;AAEA,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,OAAO,MAAM,QAAQ,GAAG;AACzC,YAAM,QAAQ,WAAW,KAAK,OAAO,MAAM,MAAM,WAAW,CAAC,IAAI,OAAO;AAExE,YAAM,WAAW,MAAM,iBAAiB;AAAA,QACtC;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,IAAI;AAAA,MAClD;AAEA,YAAM,SAAS,KAAK,MAAM,SAAS,OAAO;AAC1C,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;ACxDO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACHA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBACA,UACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAE/D,MAAI,UAAU;AACZ,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAIA,QAAO,OAAO;AAChB,QAACA,QAAe,YAAY;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,KAAK;AAC/C,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAWA,WAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAMA,QAAO,MAAM,SAAS,QAAQ,KAAK,OAAO,KAAK,WAAW;AAC9E,YAAI,QAAQ,KAAK,QAAQ,GAAG;AAC1B,cAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,mBAAS,OAAO;AAAA,YACd,WAAWA,QAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,CAAC,SAAS,OAAQ,UAAS,SAAS,CAAC;AACzC,iBAAS,OAAO;AAAA,UACd,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,MAAM,UAAa,EAAE,OAAO,IAAI,KAAK,CAAC,EAC7E,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC;AAC5B,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;;;AC5IA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,WAAW,UAAa,WAAW,UAAa,SAAS,KAAK,SAAS,EAAG;AAC9E,YAAM,QAAQA,OAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;","names":["scorer","round"]}

package/dist/index.d.cts

@@ -0,0 +1,126 @@
+import { z } from 'zod';
+
+type DatasetItem<TInput, TAnnotations = undefined> = {
+    input: TInput;
+    annotations?: TAnnotations;
+};
+type DatasetConfig<TInput extends z.ZodTypeAny, TAnnotations extends z.ZodTypeAny = z.ZodTypeAny> = {
+    name: string;
+    schema: TInput;
+    annotations?: TAnnotations;
+    items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];
+    file?: string;
+    /** Base directory for resolving relative file paths. Defaults to cwd. */
+    basePath?: string;
+};
+type Dataset<TInput = unknown, TAnnotations = unknown> = {
+    readonly name: string;
+    readonly schema: z.ZodTypeAny;
+    readonly annotationsSchema?: z.ZodTypeAny;
+    getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;
+};
+declare function dataset<TInput extends z.ZodTypeAny, TAnnotations extends z.ZodTypeAny>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
+
+type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number;
+type ScorerConfig<TOutput = any, TInput = any, TAnnotations = any> = {
+    name: string;
+    description: string;
+    score: ScorerFn<TOutput, TInput, TAnnotations>;
+};
+type Scorer<TOutput = any, TInput = any, TAnnotations = any> = {
+    readonly name: string;
+    readonly description: string;
+    readonly isLlm: boolean;
+    score(output: TOutput, input: TInput, annotations?: TAnnotations): number | Promise<number>;
+};
+declare function scorer<TOutput = any, TInput = any, TAnnotations = any>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
+
+type LlmScorerConfig = {
+    name: string;
+    description: string;
+    model: string;
+    system: string;
+    schema: z.ZodType<{
+        score: number;
+        [key: string]: any;
+    }>;
+    temperature?: number;
+};
+declare function llmScorer(config: LlmScorerConfig): Scorer;
+
+type EvalConfig = {
+    workflow: string;
+    dataset: Dataset<any, any>;
+    scorers: Scorer<any, any, any>[];
+    concurrency?: number;
+    budget?: string;
+    metadata?: Record<string, any>;
+};
+type EvalResult = {
+    id: string;
+    workflow: string;
+    dataset: string;
+    metadata: Record<string, any>;
+    timestamp: string;
+    totalCost: number;
+    duration: number;
+    items: EvalItem[];
+    summary: EvalSummary;
+};
+type EvalItem = {
+    input: any;
+    annotations?: any;
+    output: any;
+    error?: string;
+    errors?: string[];
+    scores: Record<string, number>;
+};
+type EvalSummary = {
+    count: number;
+    failures: number;
+    scorers: Record<string, {
+        mean: number;
+        min: number;
+        max: number;
+        p50: number;
+        p95: number;
+    }>;
+};
+type EvalComparison = {
+    baseline: {
+        id: string;
+        metadata: Record<string, any>;
+    };
+    candidate: {
+        id: string;
+        metadata: Record<string, any>;
+    };
+    scorers: Record<string, {
+        baselineMean: number;
+        candidateMean: number;
+        delta: number;
+        deltaPercent: number;
+    }>;
+    regressions: EvalRegression[];
+    improvements: EvalImprovement[];
+    summary: string;
+};
+type EvalRegression = {
+    input: any;
+    scorer: string;
+    baselineScore: number;
+    candidateScore: number;
+    delta: number;
+};
+type EvalImprovement = EvalRegression;
+
+declare function defineEval(config: EvalConfig): EvalConfig;
+
+declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown) => Promise<{
+    output: unknown;
+    cost?: number;
+}>, provider?: any): Promise<EvalResult>;
+
+declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
+
+export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerFn, dataset, defineEval, evalCompare, llmScorer, runEval, scorer };

package/dist/index.d.ts

@@ -0,0 +1,126 @@
+import { z } from 'zod';
+
+type DatasetItem<TInput, TAnnotations = undefined> = {
+    input: TInput;
+    annotations?: TAnnotations;
+};
+type DatasetConfig<TInput extends z.ZodTypeAny, TAnnotations extends z.ZodTypeAny = z.ZodTypeAny> = {
+    name: string;
+    schema: TInput;
+    annotations?: TAnnotations;
+    items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];
+    file?: string;
+    /** Base directory for resolving relative file paths. Defaults to cwd. */
+    basePath?: string;
+};
+type Dataset<TInput = unknown, TAnnotations = unknown> = {
+    readonly name: string;
+    readonly schema: z.ZodTypeAny;
+    readonly annotationsSchema?: z.ZodTypeAny;
+    getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;
+};
+declare function dataset<TInput extends z.ZodTypeAny, TAnnotations extends z.ZodTypeAny>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
+
+type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number;
+type ScorerConfig<TOutput = any, TInput = any, TAnnotations = any> = {
+    name: string;
+    description: string;
+    score: ScorerFn<TOutput, TInput, TAnnotations>;
+};
+type Scorer<TOutput = any, TInput = any, TAnnotations = any> = {
+    readonly name: string;
+    readonly description: string;
+    readonly isLlm: boolean;
+    score(output: TOutput, input: TInput, annotations?: TAnnotations): number | Promise<number>;
+};
+declare function scorer<TOutput = any, TInput = any, TAnnotations = any>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
+
+type LlmScorerConfig = {
+    name: string;
+    description: string;
+    model: string;
+    system: string;
+    schema: z.ZodType<{
+        score: number;
+        [key: string]: any;
+    }>;
+    temperature?: number;
+};
+declare function llmScorer(config: LlmScorerConfig): Scorer;
+
+type EvalConfig = {
+    workflow: string;
+    dataset: Dataset<any, any>;
+    scorers: Scorer<any, any, any>[];
+    concurrency?: number;
+    budget?: string;
+    metadata?: Record<string, any>;
+};
+type EvalResult = {
+    id: string;
+    workflow: string;
+    dataset: string;
+    metadata: Record<string, any>;
+    timestamp: string;
+    totalCost: number;
+    duration: number;
+    items: EvalItem[];
+    summary: EvalSummary;
+};
+type EvalItem = {
+    input: any;
+    annotations?: any;
+    output: any;
+    error?: string;
+    errors?: string[];
+    scores: Record<string, number>;
+};
+type EvalSummary = {
+    count: number;
+    failures: number;
+    scorers: Record<string, {
+        mean: number;
+        min: number;
+        max: number;
+        p50: number;
+        p95: number;
+    }>;
+};
+type EvalComparison = {
+    baseline: {
+        id: string;
+        metadata: Record<string, any>;
+    };
+    candidate: {
+        id: string;
+        metadata: Record<string, any>;
+    };
+    scorers: Record<string, {
+        baselineMean: number;
+        candidateMean: number;
+        delta: number;
+        deltaPercent: number;
+    }>;
+    regressions: EvalRegression[];
+    improvements: EvalImprovement[];
+    summary: string;
+};
+type EvalRegression = {
+    input: any;
+    scorer: string;
+    baselineScore: number;
+    candidateScore: number;
+    delta: number;
+};
+type EvalImprovement = EvalRegression;
+
+declare function defineEval(config: EvalConfig): EvalConfig;
+
+declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown) => Promise<{
+    output: unknown;
+    cost?: number;
+}>, provider?: any): Promise<EvalResult>;
+
+declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
+
+export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerFn, dataset, defineEval, evalCompare, llmScorer, runEval, scorer };