@axlsdk/eval 0.13.6 → 0.13.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. package/README.md +170 -6
  2. package/dist/chunk-CMWRFU3P.js +653 -0
  3. package/dist/chunk-CMWRFU3P.js.map +1 -0
  4. package/dist/cli.cjs +577 -73
  5. package/dist/cli.cjs.map +1 -1
  6. package/dist/cli.js +169 -23
  7. package/dist/cli.js.map +1 -1
  8. package/dist/index.cjs +413 -47
  9. package/dist/index.cjs.map +1 -1
  10. package/dist/index.d.cts +74 -2
  11. package/dist/index.d.ts +74 -2
  12. package/dist/index.js +7 -1
  13. package/dist/index.js.map +1 -1
  14. package/package.json +2 -2
  15. package/dist/chunk-F2AALK5U.js +0 -290
  16. package/dist/chunk-F2AALK5U.js.map +0 -1
package/dist/index.cjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/runner.ts","../src/compare.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer, normalizeScorerResult } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerContext, ScorerFn, ScorerResult } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\n\nexport type {\n EvalConfig,\n EvalResult,\n EvalItem,\n EvalSummary,\n EvalComparison,\n EvalRegression,\n EvalImprovement,\n ScorerDetail,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n input: TInput;\n annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n name: string;\n schema: TInput;\n annotations?: TAnnotations;\n items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n file?: string;\n /** Base directory for resolving relative file paths. Defaults to cwd. */\n basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly schema: z.ZodType;\n readonly annotationsSchema?: z.ZodType;\n getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n return {\n name: config.name,\n schema: config.schema,\n annotationsSchema: config.annotations,\n async getItems() {\n if (config.items && config.file) {\n throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n }\n if (!config.items && !config.file) {\n throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n }\n if (config.items) {\n return config.items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n }\n const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n const content = await readFile(resolvedPath, 'utf-8');\n const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n return items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n },\n };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n score: number;\n metadata?: Record<string, unknown>;\n cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n /** Resolve a provider:model URI to a provider instance and model name. */\n resolveProvider: (modelUri: string) => {\n provider: {\n chat(\n messages: { role: string; content: string }[],\n options: {\n model: string;\n temperature?: number;\n responseFormat?: { type: string; json_schema?: unknown };\n },\n ): Promise<{ content: string; cost?: number }>;\n };\n model: string;\n };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n name: string;\n description: string;\n score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly description: string;\n readonly isLlm: boolean;\n score(\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n context?: ScorerContext,\n ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n return {\n name: config.name,\n description: config.description,\n isLlm: false,\n score: config.score,\n };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n name: string;\n description: string;\n model: string;\n system: string;\n schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n // Resolve schema and its JSON representation once at construction time —\n // both are fixed for the lifetime of this scorer instance.\n const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n return {\n name: config.name,\n description: config.description,\n isLlm: true,\n async score(\n output: unknown,\n input: unknown,\n annotations?: unknown,\n context?: ScorerContext,\n ): Promise<ScorerResult> {\n if (!context?.resolveProvider) {\n throw new Error(\n `LLM scorer \"${config.name}\" has no provider. ` +\n `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n );\n }\n\n const { provider, model } = context.resolveProvider(config.model);\n\n const prompt = [\n `Evaluate the following output.`,\n ``,\n `## Input`,\n `${JSON.stringify(input, null, 2)}`,\n annotations\n ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n : '',\n ``,\n `## Output to Evaluate`,\n `${JSON.stringify(output, null, 2)}`,\n ``,\n `Respond with valid JSON matching this schema:`,\n schemaJson,\n ].join('\\n');\n\n const response = await provider.chat(\n [\n { role: 'system', content: config.system },\n { role: 'user', content: prompt },\n ],\n { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n );\n\n const responseCost = response.cost;\n\n let validated: { score: number; [key: string]: unknown };\n try {\n const parsed = JSON.parse(extractJson(response.content));\n validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n } catch (err) {\n // Attach cost to all errors so the runner can capture it even on failure.\n if (err && typeof err === 'object') {\n (err as any).cost = responseCost;\n }\n // Duck-type check instead of instanceof to handle potential dual-instance\n // scenarios where two copies of zod are present in the dependency tree.\n if (\n err &&\n typeof err === 'object' &&\n 'issues' in err &&\n Array.isArray((err as any).issues)\n ) {\n const issues = (err as any).issues as Array<{\n path: (string | number)[];\n message: string;\n }>;\n const messages = issues\n .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n .join('; ');\n const error = new Error(\n `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n );\n (error as any).cost = responseCost;\n throw error;\n }\n throw err;\n }\n\n const { score, ...rest } = validated;\n return {\n score,\n metadata: Object.keys(rest).length > 0 ? rest : undefined,\n cost: responseCost,\n };\n },\n };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n return config;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n mean: number;\n min: number;\n max: number;\n p50: number;\n p95: number;\n} {\n if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n const sorted = [...scores].sort((a, b) => a - b);\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n const min = sorted[0];\n const max = sorted[sorted.length - 1];\n const p50 = sorted[Math.floor(sorted.length * 0.5)];\n const p95 = sorted[Math.floor(sorted.length * 0.95)];\n return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n const match = cost.match(/^\\$?([\\d.]+)$/);\n if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n return parseFloat(match[1]);\n}\n\nexport async function runEval(\n config: EvalConfig,\n executeWorkflow: (\n input: unknown,\n runtime: AxlRuntime,\n ) => Promise<{ output: unknown; cost?: number }>,\n runtime: AxlRuntime,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const id = randomUUID();\n const items = await config.dataset.getItems();\n const concurrency = config.concurrency ?? 5;\n const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n // Create a scorer context that LLM scorers use to resolve providers.\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const evalItems: EvalItem[] = new Array(items.length);\n let totalCost = 0;\n let budgetExceeded = false;\n\n async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n if (budgetExceeded) {\n evalItems[itemIndex] = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Budget exceeded',\n scores: {},\n };\n return;\n }\n\n const evalItem: EvalItem = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n scores: {},\n };\n const itemStart = Date.now();\n try {\n const result = await executeWorkflow(item.input, runtime);\n evalItem.duration = Date.now() - itemStart;\n evalItem.output = result.output;\n evalItem.cost = result.cost;\n if (result.cost != null) {\n totalCost += result.cost;\n }\n } catch (err) {\n evalItem.duration = Date.now() - itemStart;\n evalItem.error = err instanceof Error ? err.message : String(err);\n evalItems[itemIndex] = evalItem;\n return;\n }\n\n if (budgetLimit != null && totalCost > budgetLimit) {\n budgetExceeded = true;\n }\n\n evalItem.scoreDetails = {};\n let itemScorerCost = 0;\n\n for (const scorer of config.scorers) {\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(\n evalItem.output,\n item.input,\n item.annotations,\n scorerContext,\n );\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n evalItem.scores[scorer.name] = round(scorerResult.score);\n evalItem.scoreDetails[scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n // Capture cost from error (LLM scorer attaches it)\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n evalItems[itemIndex] = evalItem;\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < items.length) {\n const currentIndex = index++;\n await processItem(items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n await Promise.all(workers);\n\n const failures = evalItems.filter((i) => i.error).length;\n const scorerNames = config.scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = evalItems\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n return {\n id,\n workflow: config.workflow,\n dataset: config.dataset.name,\n metadata: config.metadata ?? {},\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: evalItems,\n summary: { count: items.length, failures, scorers: scorerStats, timing },\n };\n}\n","import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n if (baseline.dataset !== candidate.dataset) {\n throw new Error(\n `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n );\n }\n\n const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n throw new Error(`Cannot compare evals with different scorers`);\n }\n\n const scorers: EvalComparison['scorers'] = {};\n for (const name of baselineScorerNames) {\n const bMean = baseline.summary.scorers[name].mean;\n const cMean = candidate.summary.scorers[name].mean;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n }\n\n const regressions: EvalRegression[] = [];\n const improvements: EvalImprovement[] = [];\n\n const minLength = Math.min(baseline.items.length, candidate.items.length);\n for (let i = 0; i < minLength; i++) {\n const bItem = baseline.items[i];\n const cItem = candidate.items[i];\n if (bItem.error || cItem.error) continue;\n for (const name of baselineScorerNames) {\n const bScore = bItem.scores[name];\n const cScore = cItem.scores[name];\n if (bScore == null || cScore == null) continue;\n const delta = round(cScore - bScore);\n if (delta < -0.1)\n regressions.push({\n itemIndex: i,\n input: bItem.input,\n scorer: name,\n baselineScore: bScore,\n candidateScore: cScore,\n delta,\n });\n else if (delta > 0.1)\n improvements.push({\n itemIndex: i,\n input: bItem.input,\n scorer: name,\n baselineScore: bScore,\n candidateScore: cScore,\n delta,\n });\n }\n }\n\n // Timing comparison (use pre-computed summary.timing when available)\n let timing: EvalComparison['timing'];\n if (baseline.summary.timing && candidate.summary.timing) {\n const bMean = baseline.summary.timing.mean;\n const cMean = candidate.summary.timing.mean;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n timing = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n }\n\n // Cost comparison\n let cost: EvalComparison['cost'];\n if (baseline.totalCost > 0 || candidate.totalCost > 0) {\n const delta = round(candidate.totalCost - baseline.totalCost);\n const deltaPercent = baseline.totalCost > 0 ? round((delta / baseline.totalCost) * 100) : 0;\n cost = {\n baselineTotal: baseline.totalCost,\n candidateTotal: candidate.totalCost,\n delta,\n deltaPercent,\n };\n }\n\n const parts: string[] = [];\n for (const name of baselineScorerNames) {\n const s = scorers[name];\n if (Math.abs(s.delta) > 0.001) {\n const direction = s.delta > 0 ? 'improves' : 'regresses';\n const sign = s.delta > 0 ? '+' : '';\n parts.push(\n `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n );\n }\n }\n if (timing && Math.abs(timing.deltaPercent) > 1) {\n const dir = timing.delta > 0 ? 'slower' : 'faster';\n parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n }\n if (cost && Math.abs(cost.deltaPercent) > 1) {\n const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n }\n const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n return {\n baseline: { id: baseline.id, metadata: baseline.metadata },\n candidate: { id: candidate.id, metadata: candidate.metadata },\n scorers,\n timing,\n cost,\n regressions,\n improvements,\n summary: summaryStr,\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACrDO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC/DA,iBAAkB;AAElB,iBAA6C;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,aAAE,OAAO,EAAE,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,aAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,cAAU,4BAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,UAAM,wBAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACAA,yBAA2B;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,SAAS,OAAO;AACzB,eAAS,OAAO,OAAO;AACvB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,SAAS,IAAI;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWA,WAAU,OAAO,SAAS;AACnC,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACvD,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;AC5MA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQA,OAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,MAAI,SAAS,QAAQ,UAAU,UAAU,QAAQ,QAAQ;AACvD,UAAM,QAAQ,SAAS,QAAQ,OAAO;AACtC,UAAM,QAAQ,UAAU,QAAQ,OAAO;AACvC,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,OAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,aAAS,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EAC5E;AAGA,MAAI;AACJ,MAAI,SAAS,YAAY,KAAK,UAAU,YAAY,GAAG;AACrD,UAAM,QAAQA,OAAM,UAAU,YAAY,SAAS,SAAS;AAC5D,UAAM,eAAe,SAAS,YAAY,IAAIA,OAAO,QAAQ,SAAS,YAAa,GAAG,IAAI;AAC1F,WAAO;AAAA,MACL,eAAe,SAAS;AAAA,MACxB,gBAAgB,UAAU;AAAA,MAC1B;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;","names":["scorer","round"]}
1
+ {"version":3,"sources":["../src/index.ts","../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts","../src/utils.ts","../src/runner.ts","../src/bootstrap.ts","../src/compare.ts","../src/rescore.ts","../src/multi-run.ts"],"sourcesContent":["export { dataset } from './dataset.js';\nexport type { Dataset, DatasetConfig, DatasetItem } from './dataset.js';\n\nexport { scorer, normalizeScorerResult } from './scorer.js';\nexport type { Scorer, ScorerConfig, ScorerContext, ScorerFn, ScorerResult } from './scorer.js';\n\nexport { llmScorer } from './llm-scorer.js';\nexport type { LlmScorerConfig } from './llm-scorer.js';\n\nexport { defineEval } from './define-eval.js';\nexport { runEval } from './runner.js';\nexport { evalCompare } from './compare.js';\nexport { pairedBootstrapCI } from './bootstrap.js';\nexport type { BootstrapCIResult } from './bootstrap.js';\nexport { rescore } from './rescore.js';\nexport type { RescoreOptions } from './rescore.js';\nexport { aggregateRuns } from './multi-run.js';\nexport type { MultiRunSummary } from './multi-run.js';\n\nexport type {\n EvalConfig,\n EvalResult,\n EvalItem,\n EvalSummary,\n EvalComparison,\n EvalCompareOptions,\n EvalRegression,\n EvalImprovement,\n ScorerDetail,\n} from './types.js';\n","import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n input: TInput;\n annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n name: string;\n schema: TInput;\n annotations?: TAnnotations;\n items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n file?: string;\n /** Base directory for resolving relative file paths. Defaults to cwd. */\n basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly schema: z.ZodType;\n readonly annotationsSchema?: z.ZodType;\n getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n return {\n name: config.name,\n schema: config.schema,\n annotationsSchema: config.annotations,\n async getItems() {\n if (config.items && config.file) {\n throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n }\n if (!config.items && !config.file) {\n throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n }\n if (config.items) {\n return config.items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n }\n const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n const content = await readFile(resolvedPath, 'utf-8');\n const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n return items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n },\n };\n}\n","/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n score: number;\n metadata?: Record<string, unknown>;\n cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n return typeof result === 'number' ? { score: result } : result;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n /** Resolve a provider:model URI to a provider instance and model name. */\n resolveProvider: (modelUri: string) => {\n provider: {\n chat(\n messages: { role: string; content: string }[],\n options: {\n model: string;\n temperature?: number;\n responseFormat?: { type: string; json_schema?: unknown };\n },\n ): Promise<{ content: string; cost?: number }>;\n };\n model: string;\n };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n) => number | ScorerResult;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n name: string;\n description: string;\n score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly description: string;\n readonly isLlm: boolean;\n score(\n output: TOutput,\n input: TInput,\n annotations?: TAnnotations,\n context?: ScorerContext,\n ): number | ScorerResult | Promise<number | ScorerResult>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n return {\n name: config.name,\n description: config.description,\n isLlm: false,\n score: config.score,\n };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n name: string;\n description: string;\n model: string;\n system: string;\n schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n // Resolve schema and its JSON representation once at construction time —\n // both are fixed for the lifetime of this scorer instance.\n const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n return {\n name: config.name,\n description: config.description,\n isLlm: true,\n async score(\n output: unknown,\n input: unknown,\n annotations?: unknown,\n context?: ScorerContext,\n ): Promise<ScorerResult> {\n if (!context?.resolveProvider) {\n throw new Error(\n `LLM scorer \"${config.name}\" has no provider. ` +\n `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n );\n }\n\n const { provider, model } = context.resolveProvider(config.model);\n\n const prompt = [\n `Evaluate the following output.`,\n ``,\n `## Input`,\n `${JSON.stringify(input, null, 2)}`,\n annotations\n ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n : '',\n ``,\n `## Output to Evaluate`,\n `${JSON.stringify(output, null, 2)}`,\n ``,\n `Respond with valid JSON matching this schema:`,\n schemaJson,\n ].join('\\n');\n\n const response = await provider.chat(\n [\n { role: 'system', content: config.system },\n { role: 'user', content: prompt },\n ],\n { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n );\n\n const responseCost = response.cost;\n\n let validated: { score: number; [key: string]: unknown };\n try {\n const parsed = JSON.parse(extractJson(response.content));\n validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n } catch (err) {\n // Attach cost to all errors so the runner can capture it even on failure.\n if (err && typeof err === 'object') {\n (err as any).cost = responseCost;\n }\n // Duck-type check instead of instanceof to handle potential dual-instance\n // scenarios where two copies of zod are present in the dependency tree.\n if (\n err &&\n typeof err === 'object' &&\n 'issues' in err &&\n Array.isArray((err as any).issues)\n ) {\n const issues = (err as any).issues as Array<{\n path: (string | number)[];\n message: string;\n }>;\n const messages = issues\n .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n .join('; ');\n const error = new Error(\n `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n );\n (error as any).cost = responseCost;\n throw error;\n }\n throw err;\n }\n\n const { score, ...rest } = validated;\n return {\n score,\n metadata: Object.keys(rest).length > 0 ? rest : undefined,\n cost: responseCost,\n };\n },\n };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n return config;\n}\n","export function computeStats(scores: number[]): {\n mean: number;\n min: number;\n max: number;\n p50: number;\n p95: number;\n} {\n if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n const sorted = [...scores].sort((a, b) => a - b);\n const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n const min = sorted[0];\n const max = sorted[sorted.length - 1];\n const p50 = sorted[Math.floor(sorted.length * 0.5)];\n const p95 = sorted[Math.floor(sorted.length * 0.95)];\n return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n const match = cost.match(/^\\$?([\\d.]+)$/);\n if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n return parseFloat(match[1]);\n}\n\nexport async function runEval(\n config: EvalConfig,\n executeWorkflow: (\n input: unknown,\n runtime: AxlRuntime,\n ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n runtime: AxlRuntime,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const id = randomUUID();\n const items = await config.dataset.getItems();\n const concurrency = config.concurrency ?? 5;\n const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n // Create a scorer context that LLM scorers use to resolve providers.\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const evalItems: EvalItem[] = new Array(items.length);\n let totalCost = 0;\n let budgetExceeded = false;\n\n async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n if (budgetExceeded) {\n evalItems[itemIndex] = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n error: 'Budget exceeded',\n scores: {},\n };\n return;\n }\n\n const evalItem: EvalItem = {\n input: item.input,\n annotations: item.annotations,\n output: null,\n scores: {},\n };\n const itemStart = Date.now();\n try {\n const result = await executeWorkflow(item.input, runtime);\n evalItem.duration = Date.now() - itemStart;\n evalItem.output = result.output;\n evalItem.cost = result.cost;\n if (result.metadata) evalItem.metadata = result.metadata;\n if (result.cost != null) {\n totalCost += result.cost;\n }\n } catch (err) {\n evalItem.duration = Date.now() - itemStart;\n evalItem.error = err instanceof Error ? err.message : String(err);\n evalItems[itemIndex] = evalItem;\n return;\n }\n\n if (budgetLimit != null && totalCost > budgetLimit) {\n budgetExceeded = true;\n }\n\n evalItem.scoreDetails = {};\n let itemScorerCost = 0;\n\n for (const scorer of config.scorers) {\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(\n evalItem.output,\n item.input,\n item.annotations,\n scorerContext,\n );\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n evalItem.scores[scorer.name] = round(scorerResult.score);\n evalItem.scoreDetails[scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n // Capture cost from error (LLM scorer attaches it)\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n evalItem.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n evalItem.scores[scorer.name] = null;\n evalItem.scoreDetails[scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n evalItem.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n evalItems[itemIndex] = evalItem;\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < items.length) {\n const currentIndex = index++;\n await processItem(items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n await Promise.all(workers);\n\n const failures = evalItems.filter((i) => i.error).length;\n const scorerNames = config.scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = evalItems\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n const scorerTypes: Record<string, string> = {};\n for (const s of config.scorers) {\n scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n }\n\n // Aggregate per-model LLM call counts across all items\n const totalModelCalls = new Map<string, number>();\n for (const item of evalItems) {\n const itemCounts = item.metadata?.modelCallCounts;\n if (itemCounts && typeof itemCounts === 'object') {\n for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n if (typeof count === 'number')\n totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n }\n } else {\n // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n const itemModels = item.metadata?.models;\n if (Array.isArray(itemModels)) {\n for (const m of itemModels) {\n if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n }\n }\n }\n }\n\n // models: unique list sorted by total calls (most-called first)\n // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n const modelsMeta: Record<string, unknown> = {};\n if (totalModelCalls.size > 0) {\n const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n modelsMeta.models = sorted.map(([m]) => m);\n modelsMeta.modelCounts = Object.fromEntries(sorted);\n }\n\n return {\n id,\n workflow: config.workflow,\n dataset: config.dataset.name,\n metadata: {\n ...config.metadata,\n scorerTypes,\n ...modelsMeta,\n },\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: evalItems,\n summary: { count: items.length, failures, scorers: scorerStats, timing },\n };\n}\n","export type BootstrapCIResult = {\n /** Lower bound of the confidence interval. */\n lower: number;\n /** Upper bound of the confidence interval. */\n upper: number;\n /** Mean of the original differences. */\n mean: number;\n /** Proportion of resamples where the mean difference is negative (probability of regression). */\n pRegression: number;\n /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n differences: number[],\n options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n const n = differences.length;\n if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n const nResamples = options?.nResamples ?? 1000;\n const alpha = options?.alpha ?? 0.05;\n\n const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n if (n === 1)\n return {\n lower: differences[0],\n upper: differences[0],\n mean,\n pRegression: mean < 0 ? 1 : 0,\n pImprovement: mean > 0 ? 1 : 0,\n };\n\n if (nResamples <= 0)\n return {\n lower: round(mean),\n upper: round(mean),\n mean: round(mean),\n pRegression: mean < 0 ? 1 : 0,\n pImprovement: mean > 0 ? 1 : 0,\n };\n\n // Seeded or random number generator\n const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n const resampleMeans: number[] = new Array(nResamples);\n for (let r = 0; r < nResamples; r++) {\n let sum = 0;\n for (let i = 0; i < n; i++) {\n sum += differences[Math.floor(rng() * n)];\n }\n resampleMeans[r] = sum / n;\n }\n\n resampleMeans.sort((a, b) => a - b);\n\n const lowerIdx = Math.floor((alpha / 2) * nResamples);\n const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n // Count resamples below/above zero for directional probabilities\n // resampleMeans is already sorted, so binary search for zero\n let belowZero = 0;\n for (let i = 0; i < nResamples; i++) {\n if (resampleMeans[i] < 0) belowZero++;\n else break;\n }\n let aboveZero = 0;\n for (let i = nResamples - 1; i >= 0; i--) {\n if (resampleMeans[i] > 0) aboveZero++;\n else break;\n }\n\n return {\n lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n mean: round(mean),\n pRegression: round(belowZero / nResamples),\n pImprovement: round(aboveZero / nResamples),\n };\n}\n\nfunction round(n: number): number {\n return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n let state = seed | 0 || 1; // Ensure non-zero\n return () => {\n state ^= state << 13;\n state ^= state >> 17;\n state ^= state << 5;\n return (state >>> 0) / 0x100000000;\n };\n}\n","import type {\n EvalResult,\n EvalComparison,\n EvalCompareOptions,\n EvalRegression,\n EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\n\nfunction round(n: number): number {\n return Math.round(n * 1000) / 1000;\n}\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n scorerName: string,\n options: EvalCompareOptions | undefined,\n metadata: Record<string, unknown>,\n): number {\n // Explicit global threshold\n if (typeof options?.thresholds === 'number') return options.thresholds;\n\n // Per-scorer threshold map\n if (options?.thresholds && typeof options.thresholds === 'object') {\n const perScorer = options.thresholds[scorerName];\n if (perScorer != null) return perScorer;\n // Fall through to auto-calibration if scorer not in map\n }\n\n // Auto-calibrate from scorerTypes metadata\n const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n if (scorerTypes && scorerName in scorerTypes) {\n return scorerTypes[scorerName] === 'llm'\n ? DEFAULT_LLM_THRESHOLD\n : DEFAULT_DETERMINISTIC_THRESHOLD;\n }\n\n // Legacy fallback for results without scorerTypes\n return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n baseline: EvalResult | EvalResult[],\n candidate: EvalResult | EvalResult[],\n options?: EvalCompareOptions,\n): EvalComparison {\n const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];\n const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];\n\n if (baselineRuns.length === 0 || candidateRuns.length === 0) {\n throw new Error('Cannot compare empty eval result arrays');\n }\n\n // Use the first run as representative for metadata, dataset, scorers\n const baselineRef = baselineRuns[0];\n const candidateRef = candidateRuns[0];\n\n if (baselineRef.dataset !== candidateRef.dataset) {\n throw new Error(\n `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n );\n }\n\n const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n throw new Error(`Cannot compare evals with different scorers`);\n }\n\n // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n const pairedDiffs: Record<string, number[]> = {};\n for (const name of baselineScorerNames) {\n pairedDiffs[name] = [];\n }\n const runCount = Math.min(baselineRuns.length, candidateRuns.length);\n for (let r = 0; r < runCount; r++) {\n const bRun = baselineRuns[r];\n const cRun = candidateRuns[r];\n const minLength = Math.min(bRun.items.length, cRun.items.length);\n for (let i = 0; i < minLength; i++) {\n const bItem = bRun.items[i];\n const cItem = cRun.items[i];\n if (bItem.error || cItem.error) continue;\n for (const name of baselineScorerNames) {\n const bScore = bItem.scores[name];\n const cScore = cItem.scores[name];\n if (bScore != null && cScore != null) {\n pairedDiffs[name].push(cScore - bScore);\n }\n }\n }\n }\n\n // Compute aggregate means across runs\n const baselineMeans: Record<string, number> = {};\n const candidateMeans: Record<string, number> = {};\n for (const name of baselineScorerNames) {\n baselineMeans[name] =\n baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n baselineRuns.length;\n candidateMeans[name] =\n candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n candidateRuns.length;\n }\n\n const scorers: EvalComparison['scorers'] = {};\n for (const name of baselineScorerNames) {\n const bMeanRaw = baselineMeans[name];\n const cMeanRaw = candidateMeans[name];\n const deltaRaw = cMeanRaw - bMeanRaw;\n const bMean = round(bMeanRaw);\n const cMean = round(cMeanRaw);\n const delta = round(deltaRaw);\n const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n const entry: EvalComparison['scorers'][string] = {\n baselineMean: bMean,\n candidateMean: cMean,\n delta,\n deltaPercent,\n };\n\n // Compute bootstrap CI when we have enough paired data\n const diffs = pairedDiffs[name];\n if (diffs.length >= 2) {\n const ci = pairedBootstrapCI(diffs);\n entry.ci = { lower: ci.lower, upper: ci.upper };\n entry.pRegression = ci.pRegression;\n entry.pImprovement = ci.pImprovement;\n entry.n = diffs.length;\n // Significant when CI excludes zero AND delta exceeds practical threshold\n const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n }\n\n scorers[name] = entry;\n }\n\n // Per-item regressions/improvements.\n // For multi-run: average each item's score across runs to get a stable per-item comparison.\n const regressions: EvalRegression[] = [];\n const improvements: EvalImprovement[] = [];\n\n const itemCount = Math.min(\n ...baselineRuns.map((r) => r.items.length),\n ...candidateRuns.map((r) => r.items.length),\n );\n\n for (let i = 0; i < itemCount; i++) {\n // Check if any run has an error for this item\n const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n if (hasBaselineError || hasCandidateError) continue;\n\n for (const name of baselineScorerNames) {\n // Average this item's score across all runs on each side\n const bScores = baselineRuns\n .map((r) => r.items[i]?.scores[name])\n .filter((s): s is number => s != null);\n const cScores = candidateRuns\n .map((r) => r.items[i]?.scores[name])\n .filter((s): s is number => s != null);\n if (bScores.length === 0 || cScores.length === 0) continue;\n\n const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n const delta = round(cAvg - bAvg);\n const threshold = resolveThreshold(name, options, baselineRef.metadata);\n if (delta < -threshold)\n regressions.push({\n itemIndex: i,\n input: baselineRef.items[i]?.input,\n scorer: name,\n baselineScore: round(bAvg),\n candidateScore: round(cAvg),\n delta,\n });\n else if (delta > threshold)\n improvements.push({\n itemIndex: i,\n input: baselineRef.items[i]?.input,\n scorer: name,\n baselineScore: round(bAvg),\n candidateScore: round(cAvg),\n delta,\n });\n }\n }\n\n // Timing comparison (average across all runs for multi-run)\n let timing: EvalComparison['timing'];\n const baselineTimings = baselineRuns\n .filter((r) => r.summary.timing)\n .map((r) => r.summary.timing!.mean);\n const candidateTimings = candidateRuns\n .filter((r) => r.summary.timing)\n .map((r) => r.summary.timing!.mean);\n if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n const delta = round(cMean - bMean);\n const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n }\n\n // Cost comparison (per-run average for multi-run)\n let cost: EvalComparison['cost'];\n const baselineAvgCost =\n baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n const candidateAvgCost =\n candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n const deltaRaw = candidateAvgCost - baselineAvgCost;\n const delta = round(deltaRaw);\n const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n cost = {\n baselineTotal: round(baselineAvgCost),\n candidateTotal: round(candidateAvgCost),\n delta,\n deltaPercent,\n };\n }\n\n const parts: string[] = [];\n for (const name of baselineScorerNames) {\n const s = scorers[name];\n if (Math.abs(s.delta) > 0.001) {\n const direction = s.delta > 0 ? 'improves' : 'regresses';\n const sign = s.delta > 0 ? '+' : '';\n let sigLabel = '';\n if (s.significant === true) sigLabel = ' (significant)';\n else if (s.significant === false) sigLabel = ' (not significant)';\n parts.push(\n `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n );\n }\n }\n if (timing && Math.abs(timing.deltaPercent) > 1) {\n const dir = timing.delta > 0 ? 'slower' : 'faster';\n parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n }\n if (cost && Math.abs(cost.deltaPercent) > 1) {\n const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n }\n const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n return {\n baseline: { id: baselineRef.id, metadata: baselineRef.metadata },\n candidate: { id: candidateRef.id, metadata: candidateRef.metadata },\n scorers,\n timing,\n cost,\n regressions,\n improvements,\n summary: summaryStr,\n };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult } from './scorer.js';\nimport { computeStats, round } from './utils.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n concurrency?: number;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n result: EvalResult,\n scorers: Scorer[],\n runtime: AxlRuntime,\n options?: RescoreOptions,\n): Promise<EvalResult> {\n const startTime = Date.now();\n const concurrency = options?.concurrency ?? 5;\n\n const scorerContext: ScorerContext = {\n resolveProvider: (uri: string) => {\n if (typeof runtime.resolveProvider !== 'function') {\n throw new Error(\n `LLM scorers require a runtime with resolveProvider(). ` +\n `Ensure you are using a real AxlRuntime instance, not a mock.`,\n );\n }\n return runtime.resolveProvider(uri);\n },\n };\n\n const rescored: EvalItem[] = new Array(result.items.length);\n let totalCost = 0;\n\n async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n // Pass through error items without scoring\n if (original.error) {\n rescored[itemIndex] = {\n input: original.input,\n annotations: original.annotations,\n output: original.output,\n error: original.error,\n scores: {},\n };\n return;\n }\n\n const item: EvalItem = {\n input: original.input,\n annotations: original.annotations,\n output: original.output,\n metadata: original.metadata,\n scores: {},\n scoreDetails: {},\n };\n\n let itemScorerCost = 0;\n\n for (const scorer of scorers) {\n const scorerStart = Date.now();\n try {\n const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n const scorerResult = normalizeScorerResult(raw);\n\n if (scorerResult.cost != null) {\n itemScorerCost += scorerResult.cost;\n totalCost += scorerResult.cost;\n }\n\n const scorerDuration = Date.now() - scorerStart;\n\n if (\n !Number.isFinite(scorerResult.score) ||\n scorerResult.score < 0 ||\n scorerResult.score > 1\n ) {\n if (!item.scorerErrors) item.scorerErrors = [];\n item.scorerErrors.push(\n `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`,\n );\n item.scores[scorer.name] = null;\n item.scoreDetails![scorer.name] = {\n score: null,\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n } else {\n item.scores[scorer.name] = round(scorerResult.score);\n item.scoreDetails![scorer.name] = {\n score: round(scorerResult.score),\n metadata: scorerResult.metadata,\n duration: scorerDuration,\n cost: scorerResult.cost,\n };\n }\n } catch (err) {\n const errCost = typeof (err as any)?.cost === 'number' ? (err as any).cost : undefined;\n if (errCost != null) {\n itemScorerCost += errCost;\n totalCost += errCost;\n }\n if (!item.scorerErrors) item.scorerErrors = [];\n item.scorerErrors.push(\n `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n );\n item.scores[scorer.name] = null;\n item.scoreDetails![scorer.name] = {\n score: null,\n duration: Date.now() - scorerStart,\n cost: errCost,\n };\n }\n }\n item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n rescored[itemIndex] = item;\n }\n\n let index = 0;\n async function runNext(): Promise<void> {\n while (index < result.items.length) {\n const currentIndex = index++;\n await rescoreItem(result.items[currentIndex], currentIndex);\n }\n }\n\n const workers = Array.from({ length: Math.min(concurrency, result.items.length) }, () =>\n runNext(),\n );\n await Promise.all(workers);\n\n const failures = rescored.filter((i) => i.error).length;\n const scorerNames = scorers.map((s) => s.name);\n const scorerStats: EvalSummary['scorers'] = {};\n for (const name of scorerNames) {\n const scores = rescored\n .filter((i) => !i.error && i.scores[name] != null)\n .map((i) => i.scores[name] as number);\n scorerStats[name] = computeStats(scores);\n }\n\n const scorerTypes: Record<string, string> = {};\n for (const s of scorers) {\n scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n }\n\n return {\n id: randomUUID(),\n workflow: result.workflow,\n dataset: result.dataset,\n metadata: (() => {\n // Strip run group membership — rescored results are independent evaluations\n // eslint-disable-next-line @typescript-eslint/no-unused-vars\n const { runGroupId: _, runIndex: __, ...rest } = result.metadata;\n return { ...rest, rescored: true, originalId: result.id, scorerTypes };\n })(),\n timestamp: new Date().toISOString(),\n totalCost,\n duration: Date.now() - startTime,\n items: rescored,\n summary: { count: result.items.length, failures, scorers: scorerStats },\n };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n runGroupId: string;\n runCount: number;\n workflow: string;\n dataset: string;\n totalCost: number;\n totalDuration: number;\n scorers: Record<string, { mean: number; std: number; min: number; max: number }>;\n timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n if (values.length <= 1) return 0;\n const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n const { workflow, dataset } = runs[0];\n\n const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n // Collect per-scorer means from each run\n const scorerNames = Object.keys(runs[0].summary.scorers);\n const scorers: MultiRunSummary['scorers'] = {};\n\n for (const name of scorerNames) {\n const means = runs.map((r) => r.summary.scorers[name]?.mean ?? 0);\n const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n const minMean = Math.min(...means);\n const maxMean = Math.max(...means);\n\n scorers[name] = {\n mean: round(meanOfMeans),\n std: round(std(means, meanOfMeans)),\n min: round(minMean),\n max: round(maxMean),\n };\n }\n\n // Timing aggregation\n let timing: MultiRunSummary['timing'];\n const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n if (timingMeans.length === runs.length && timingMeans.length > 0) {\n const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n timing = {\n mean: round(meanTiming),\n std: round(std(timingMeans, meanTiming)),\n };\n }\n\n return {\n runGroupId,\n runCount: runs.length,\n workflow,\n dataset,\n totalCost: round(totalCost),\n totalDuration: round(totalDuration),\n scorers,\n timing,\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACCA,sBAAyB;AACzB,WAAsB;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,UAAM,0BAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACrDO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AA4CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AC/DA,iBAAkB;AAElB,iBAA6C;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,aAAE,OAAO,EAAE,OAAO,aAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,aAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,cAAU,4BAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,UAAM,wBAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;;;ACJO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAM,MAAM,IAAI,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,GAAG,KAAK,MAAM,GAAG,EAAE;AACjG;AAEO,SAAS,MAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;;;ACdA,yBAA2B;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,SAAK,+BAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,SAAS,OAAO;AACzB,eAAS,OAAO,OAAO;AACvB,UAAI,OAAO,SAAU,UAAS,WAAW,OAAO;AAChD,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,SAAS,IAAI;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,aAAS,eAAe,CAAC;AACzB,QAAI,iBAAiB;AAErB,eAAWA,WAAU,OAAO,SAAS;AACnC,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO;AAAA,UACvB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AACA,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,mBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,mBAAS,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACvD,mBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,YACnC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AAEZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAOA,QAAO,IAAI,IAAI;AAC/B,iBAAS,aAAaA,QAAO,IAAI,IAAI;AAAA,UACnC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,aAAS,aAAa,iBAAiB,IAAI,iBAAiB;AAC5D,cAAU,SAAS,IAAI;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,GAAG,YAAY;AAAA,IACrD;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAEhE,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAGA,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,IACL;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,aAAa,OAAO;AAAA,EACzE;AACF;;;AC9MO,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAOC,OAAM,IAAI;AAAA,MACjB,OAAOA,OAAM,IAAI;AAAA,MACjB,MAAMA,OAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAOA,OAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAOA,OAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAMA,OAAM,IAAI;AAAA,IAChB,aAAaA,OAAM,YAAY,UAAU;AAAA,IACzC,cAAcA,OAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;AC5FA,SAASC,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,eAAe,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACnE,QAAM,gBAAgB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAEvE,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAGA,QAAM,cAAc,aAAa,CAAC;AAClC,QAAM,eAAe,cAAc,CAAC;AAEpC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,QAAM,WAAW,KAAK,IAAI,aAAa,QAAQ,cAAc,MAAM;AACnE,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAEtE,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,IACF;AAGA,UAAM,QAAQ,YAAY,IAAI;AAC9B,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AACxB,YAAM,IAAI,MAAM;AAEhB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAE/J,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,YAAY,IAAI,UAAU,YAAY,SAAS;AAAA,IAC/D,WAAW,EAAE,IAAI,aAAa,IAAI,UAAU,aAAa,SAAS;AAAA,IAClE;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;ACjQA,IAAAC,sBAA2B;AAU3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAE5C,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAE/E,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAEA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAEA,QAAI,iBAAiB;AAErB,eAAWC,WAAU,SAAS;AAC5B,YAAM,cAAc,KAAK,IAAI;AAC7B,UAAI;AACF,cAAM,MAAM,MAAMA,QAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,cAAM,eAAe,sBAAsB,GAAG;AAE9C,YAAI,aAAa,QAAQ,MAAM;AAC7B,4BAAkB,aAAa;AAC/B,uBAAa,aAAa;AAAA,QAC5B;AAEA,cAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,YACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,cAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,eAAK,aAAa;AAAA,YAChB,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACnH;AACA,eAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO;AAAA,YACP,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF,OAAO;AACL,eAAK,OAAOA,QAAO,IAAI,IAAI,MAAM,aAAa,KAAK;AACnD,eAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,YAChC,OAAO,MAAM,aAAa,KAAK;AAAA,YAC/B,UAAU,aAAa;AAAA,YACvB,UAAU;AAAA,YACV,MAAM,aAAa;AAAA,UACrB;AAAA,QACF;AAAA,MACF,SAAS,KAAK;AACZ,cAAM,UAAU,OAAQ,KAAa,SAAS,WAAY,IAAY,OAAO;AAC7E,YAAI,WAAW,MAAM;AACnB,4BAAkB;AAClB,uBAAa;AAAA,QACf;AACA,YAAI,CAAC,KAAK,aAAc,MAAK,eAAe,CAAC;AAC7C,aAAK,aAAa;AAAA,UAChB,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,aAAK,aAAcA,QAAO,IAAI,IAAI;AAAA,UAChC,OAAO;AAAA,UACP,UAAU,KAAK,IAAI,IAAI;AAAA,UACvB,MAAM;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,SAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,aAAS,SAAS,IAAI;AAAA,EACxB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,OAAO,MAAM,QAAQ;AAClC,YAAM,eAAe;AACrB,YAAM,YAAY,OAAO,MAAM,YAAY,GAAG,YAAY;AAAA,IAC5D;AAAA,EACF;AAEA,QAAM,UAAU,MAAM;AAAA,IAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,MAAM,EAAE;AAAA,IAAG,MACjF,QAAQ;AAAA,EACV;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,QAAI,gCAAW;AAAA,IACf,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAGf,YAAM,EAAE,YAAY,GAAG,UAAU,IAAI,GAAG,KAAK,IAAI,OAAO;AACxD,aAAO,EAAE,GAAG,MAAM,UAAU,MAAM,YAAY,OAAO,IAAI,YAAY;AAAA,IACvE,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;ACtKA,IAAAC,sBAA2B;AAc3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,kBAAyB,gCAAW;AACzE,QAAM,EAAE,UAAU,SAAAC,SAAQ,IAAI,KAAK,CAAC;AAEpC,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,QAAQ,KAAK,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AAChE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAEjC,YAAQ,IAAI,IAAI;AAAA,MACd,MAAM,MAAM,WAAW;AAAA,MACvB,KAAK,MAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAK,MAAM,OAAO;AAAA,MAClB,KAAK,MAAM,OAAO;AAAA,IACpB;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAM,MAAM,UAAU;AAAA,MACtB,KAAK,MAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA,SAAAA;AAAA,IACA,WAAW,MAAM,SAAS;AAAA,IAC1B,eAAe,MAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;","names":["scorer","round","round","import_node_crypto","scorer","import_node_crypto","dataset"]}
package/dist/index.d.cts CHANGED
@@ -116,6 +116,8 @@ type EvalItem = {
116
116
  cost?: number;
117
117
  scorerCost?: number;
118
118
  scoreDetails?: Record<string, ScorerDetail>;
119
+ /** Execution metadata forwarded from the runtime (models, tokens, agentCalls, etc). */
120
+ metadata?: Record<string, unknown>;
119
121
  };
120
122
  type EvalSummary = {
121
123
  count: number;
@@ -149,6 +151,15 @@ type EvalComparison = {
149
151
  candidateMean: number;
150
152
  delta: number;
151
153
  deltaPercent: number;
154
+ ci?: {
155
+ lower: number;
156
+ upper: number;
157
+ };
158
+ significant?: boolean;
159
+ pRegression?: number;
160
+ pImprovement?: number;
161
+ /** Number of paired item differences used for CI computation. */
162
+ n?: number;
152
163
  }>;
153
164
  timing?: {
154
165
  baselineMean: number;
@@ -175,14 +186,75 @@ type EvalRegression = {
175
186
  delta: number;
176
187
  };
177
188
  type EvalImprovement = EvalRegression;
189
+ type EvalCompareOptions = {
190
+ /** Global threshold or per-scorer map. Default: auto-calibrate from scorerTypes metadata. */
191
+ thresholds?: Record<string, number> | number;
192
+ };
178
193
 
179
194
  declare function defineEval(config: EvalConfig): EvalConfig;
180
195
 
181
196
  declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, runtime: AxlRuntime) => Promise<{
182
197
  output: unknown;
183
198
  cost?: number;
199
+ metadata?: Record<string, unknown>;
184
200
  }>, runtime: AxlRuntime): Promise<EvalResult>;
185
201
 
186
- declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
202
+ declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
203
+
204
+ type BootstrapCIResult = {
205
+ /** Lower bound of the confidence interval. */
206
+ lower: number;
207
+ /** Upper bound of the confidence interval. */
208
+ upper: number;
209
+ /** Mean of the original differences. */
210
+ mean: number;
211
+ /** Proportion of resamples where the mean difference is negative (probability of regression). */
212
+ pRegression: number;
213
+ /** Proportion of resamples where the mean difference is positive (probability of improvement). */
214
+ pImprovement: number;
215
+ };
216
+ /**
217
+ * Compute a bootstrap confidence interval on paired differences.
218
+ *
219
+ * Resamples the differences array with replacement, computes the mean of each
220
+ * resample, and returns the percentile-based confidence interval.
221
+ */
222
+ declare function pairedBootstrapCI(differences: number[], options?: {
223
+ nResamples?: number;
224
+ alpha?: number;
225
+ seed?: number;
226
+ }): BootstrapCIResult;
227
+
228
+ type RescoreOptions = {
229
+ concurrency?: number;
230
+ };
231
+ /**
232
+ * Re-run scorers on the saved outputs of an existing eval result.
233
+ * Preserves original input/output/annotations. Only re-runs scoring.
234
+ */
235
+ declare function rescore(result: EvalResult, scorers: Scorer[], runtime: AxlRuntime, options?: RescoreOptions): Promise<EvalResult>;
236
+
237
+ type MultiRunSummary = {
238
+ runGroupId: string;
239
+ runCount: number;
240
+ workflow: string;
241
+ dataset: string;
242
+ totalCost: number;
243
+ totalDuration: number;
244
+ scorers: Record<string, {
245
+ mean: number;
246
+ std: number;
247
+ min: number;
248
+ max: number;
249
+ }>;
250
+ timing?: {
251
+ mean: number;
252
+ std: number;
253
+ };
254
+ };
255
+ /**
256
+ * Aggregate multiple eval runs into a summary with mean +/- std per scorer.
257
+ */
258
+ declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
187
259
 
188
- export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, runEval, scorer };
260
+ export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
package/dist/index.d.ts CHANGED
@@ -116,6 +116,8 @@ type EvalItem = {
116
116
  cost?: number;
117
117
  scorerCost?: number;
118
118
  scoreDetails?: Record<string, ScorerDetail>;
119
+ /** Execution metadata forwarded from the runtime (models, tokens, agentCalls, etc). */
120
+ metadata?: Record<string, unknown>;
119
121
  };
120
122
  type EvalSummary = {
121
123
  count: number;
@@ -149,6 +151,15 @@ type EvalComparison = {
149
151
  candidateMean: number;
150
152
  delta: number;
151
153
  deltaPercent: number;
154
+ ci?: {
155
+ lower: number;
156
+ upper: number;
157
+ };
158
+ significant?: boolean;
159
+ pRegression?: number;
160
+ pImprovement?: number;
161
+ /** Number of paired item differences used for CI computation. */
162
+ n?: number;
152
163
  }>;
153
164
  timing?: {
154
165
  baselineMean: number;
@@ -175,14 +186,75 @@ type EvalRegression = {
175
186
  delta: number;
176
187
  };
177
188
  type EvalImprovement = EvalRegression;
189
+ type EvalCompareOptions = {
190
+ /** Global threshold or per-scorer map. Default: auto-calibrate from scorerTypes metadata. */
191
+ thresholds?: Record<string, number> | number;
192
+ };
178
193
 
179
194
  declare function defineEval(config: EvalConfig): EvalConfig;
180
195
 
181
196
  declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, runtime: AxlRuntime) => Promise<{
182
197
  output: unknown;
183
198
  cost?: number;
199
+ metadata?: Record<string, unknown>;
184
200
  }>, runtime: AxlRuntime): Promise<EvalResult>;
185
201
 
186
- declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
202
+ declare function evalCompare(baseline: EvalResult | EvalResult[], candidate: EvalResult | EvalResult[], options?: EvalCompareOptions): EvalComparison;
203
+
204
+ type BootstrapCIResult = {
205
+ /** Lower bound of the confidence interval. */
206
+ lower: number;
207
+ /** Upper bound of the confidence interval. */
208
+ upper: number;
209
+ /** Mean of the original differences. */
210
+ mean: number;
211
+ /** Proportion of resamples where the mean difference is negative (probability of regression). */
212
+ pRegression: number;
213
+ /** Proportion of resamples where the mean difference is positive (probability of improvement). */
214
+ pImprovement: number;
215
+ };
216
+ /**
217
+ * Compute a bootstrap confidence interval on paired differences.
218
+ *
219
+ * Resamples the differences array with replacement, computes the mean of each
220
+ * resample, and returns the percentile-based confidence interval.
221
+ */
222
+ declare function pairedBootstrapCI(differences: number[], options?: {
223
+ nResamples?: number;
224
+ alpha?: number;
225
+ seed?: number;
226
+ }): BootstrapCIResult;
227
+
228
+ type RescoreOptions = {
229
+ concurrency?: number;
230
+ };
231
+ /**
232
+ * Re-run scorers on the saved outputs of an existing eval result.
233
+ * Preserves original input/output/annotations. Only re-runs scoring.
234
+ */
235
+ declare function rescore(result: EvalResult, scorers: Scorer[], runtime: AxlRuntime, options?: RescoreOptions): Promise<EvalResult>;
236
+
237
+ type MultiRunSummary = {
238
+ runGroupId: string;
239
+ runCount: number;
240
+ workflow: string;
241
+ dataset: string;
242
+ totalCost: number;
243
+ totalDuration: number;
244
+ scorers: Record<string, {
245
+ mean: number;
246
+ std: number;
247
+ min: number;
248
+ max: number;
249
+ }>;
250
+ timing?: {
251
+ mean: number;
252
+ std: number;
253
+ };
254
+ };
255
+ /**
256
+ * Aggregate multiple eval runs into a summary with mean +/- std per scorer.
257
+ */
258
+ declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
187
259
 
188
- export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, runEval, scorer };
260
+ export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type EvalCompareOptions, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, aggregateRuns, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
package/dist/index.js CHANGED
@@ -1,9 +1,12 @@
1
1
  import {
2
+ aggregateRuns,
2
3
  evalCompare,
3
4
  normalizeScorerResult,
5
+ pairedBootstrapCI,
6
+ rescore,
4
7
  runEval,
5
8
  scorer
6
- } from "./chunk-F2AALK5U.js";
9
+ } from "./chunk-CMWRFU3P.js";
7
10
 
8
11
  // src/dataset.ts
9
12
  import { readFile } from "fs/promises";
@@ -111,11 +114,14 @@ function defineEval(config) {
111
114
  return config;
112
115
  }
113
116
  export {
117
+ aggregateRuns,
114
118
  dataset,
115
119
  defineEval,
116
120
  evalCompare,
117
121
  llmScorer,
118
122
  normalizeScorerResult,
123
+ pairedBootstrapCI,
124
+ rescore,
119
125
  runEval,
120
126
  scorer
121
127
  };
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n input: TInput;\n annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n name: string;\n schema: TInput;\n annotations?: TAnnotations;\n items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n file?: string;\n /** Base directory for resolving relative file paths. Defaults to cwd. */\n basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly schema: z.ZodType;\n readonly annotationsSchema?: z.ZodType;\n getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n return {\n name: config.name,\n schema: config.schema,\n annotationsSchema: config.annotations,\n async getItems() {\n if (config.items && config.file) {\n throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n }\n if (!config.items && !config.file) {\n throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n }\n if (config.items) {\n return config.items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n }\n const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n const content = await readFile(resolvedPath, 'utf-8');\n const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n return items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n },\n };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n name: string;\n description: string;\n model: string;\n system: string;\n schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n // Resolve schema and its JSON representation once at construction time —\n // both are fixed for the lifetime of this scorer instance.\n const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n return {\n name: config.name,\n description: config.description,\n isLlm: true,\n async score(\n output: unknown,\n input: unknown,\n annotations?: unknown,\n context?: ScorerContext,\n ): Promise<ScorerResult> {\n if (!context?.resolveProvider) {\n throw new Error(\n `LLM scorer \"${config.name}\" has no provider. ` +\n `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n );\n }\n\n const { provider, model } = context.resolveProvider(config.model);\n\n const prompt = [\n `Evaluate the following output.`,\n ``,\n `## Input`,\n `${JSON.stringify(input, null, 2)}`,\n annotations\n ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n : '',\n ``,\n `## Output to Evaluate`,\n `${JSON.stringify(output, null, 2)}`,\n ``,\n `Respond with valid JSON matching this schema:`,\n schemaJson,\n ].join('\\n');\n\n const response = await provider.chat(\n [\n { role: 'system', content: config.system },\n { role: 'user', content: prompt },\n ],\n { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n );\n\n const responseCost = response.cost;\n\n let validated: { score: number; [key: string]: unknown };\n try {\n const parsed = JSON.parse(extractJson(response.content));\n validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n } catch (err) {\n // Attach cost to all errors so the runner can capture it even on failure.\n if (err && typeof err === 'object') {\n (err as any).cost = responseCost;\n }\n // Duck-type check instead of instanceof to handle potential dual-instance\n // scenarios where two copies of zod are present in the dependency tree.\n if (\n err &&\n typeof err === 'object' &&\n 'issues' in err &&\n Array.isArray((err as any).issues)\n ) {\n const issues = (err as any).issues as Array<{\n path: (string | number)[];\n message: string;\n }>;\n const messages = issues\n .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n .join('; ');\n const error = new Error(\n `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n );\n (error as any).cost = responseCost;\n throw error;\n }\n throw err;\n }\n\n const { score, ...rest } = validated;\n return {\n score,\n metadata: Object.keys(rest).length > 0 ? rest : undefined,\n cost: responseCost,\n };\n },\n };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n return config;\n}\n"],"mappings":";;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC7DA,SAAS,SAAS;AAElB,SAAS,aAAa,uBAAuB;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}
1
+ {"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n input: TInput;\n annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n name: string;\n schema: TInput;\n annotations?: TAnnotations;\n items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n file?: string;\n /** Base directory for resolving relative file paths. Defaults to cwd. */\n basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n readonly name: string;\n readonly schema: z.ZodType;\n readonly annotationsSchema?: z.ZodType;\n getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n return {\n name: config.name,\n schema: config.schema,\n annotationsSchema: config.annotations,\n async getItems() {\n if (config.items && config.file) {\n throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n }\n if (!config.items && !config.file) {\n throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n }\n if (config.items) {\n return config.items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n }\n const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n const content = await readFile(resolvedPath, 'utf-8');\n const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n return items.map((item) => ({\n input: config.schema.parse(item.input),\n annotations:\n item.annotations && config.annotations\n ? config.annotations.parse(item.annotations)\n : item.annotations,\n }));\n },\n };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n name: string;\n description: string;\n model: string;\n system: string;\n schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n // Resolve schema and its JSON representation once at construction time —\n // both are fixed for the lifetime of this scorer instance.\n const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n return {\n name: config.name,\n description: config.description,\n isLlm: true,\n async score(\n output: unknown,\n input: unknown,\n annotations?: unknown,\n context?: ScorerContext,\n ): Promise<ScorerResult> {\n if (!context?.resolveProvider) {\n throw new Error(\n `LLM scorer \"${config.name}\" has no provider. ` +\n `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n );\n }\n\n const { provider, model } = context.resolveProvider(config.model);\n\n const prompt = [\n `Evaluate the following output.`,\n ``,\n `## Input`,\n `${JSON.stringify(input, null, 2)}`,\n annotations\n ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n : '',\n ``,\n `## Output to Evaluate`,\n `${JSON.stringify(output, null, 2)}`,\n ``,\n `Respond with valid JSON matching this schema:`,\n schemaJson,\n ].join('\\n');\n\n const response = await provider.chat(\n [\n { role: 'system', content: config.system },\n { role: 'user', content: prompt },\n ],\n { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n );\n\n const responseCost = response.cost;\n\n let validated: { score: number; [key: string]: unknown };\n try {\n const parsed = JSON.parse(extractJson(response.content));\n validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n } catch (err) {\n // Attach cost to all errors so the runner can capture it even on failure.\n if (err && typeof err === 'object') {\n (err as any).cost = responseCost;\n }\n // Duck-type check instead of instanceof to handle potential dual-instance\n // scenarios where two copies of zod are present in the dependency tree.\n if (\n err &&\n typeof err === 'object' &&\n 'issues' in err &&\n Array.isArray((err as any).issues)\n ) {\n const issues = (err as any).issues as Array<{\n path: (string | number)[];\n message: string;\n }>;\n const messages = issues\n .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n .join('; ');\n const error = new Error(\n `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n );\n (error as any).cost = responseCost;\n throw error;\n }\n throw err;\n }\n\n const { score, ...rest } = validated;\n return {\n score,\n metadata: Object.keys(rest).length > 0 ? rest : undefined,\n cost: responseCost,\n };\n },\n };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n return config;\n}\n"],"mappings":";;;;;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC7DA,SAAS,SAAS;AAElB,SAAS,aAAa,uBAAuB;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@axlsdk/eval",
3
- "version": "0.13.6",
3
+ "version": "0.13.7",
4
4
  "description": "Evaluation framework for Axl agentic workflows",
5
5
  "type": "module",
6
6
  "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
25
25
  "dist"
26
26
  ],
27
27
  "dependencies": {
28
- "@axlsdk/axl": "0.13.6"
28
+ "@axlsdk/axl": "0.13.7"
29
29
  },
30
30
  "peerDependencies": {
31
31
  "zod": "^4.0.0"