@axlsdk/eval 0.13.4 → 0.13.6

package/dist/index.d.cts

@@ -22,6 +22,14 @@ type Dataset<TInput = unknown, TAnnotations = unknown> = {
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
 
+/** Result from a scorer that includes metadata beyond the numeric score. */
+type ScorerResult = {
+    score: number;
+    metadata?: Record<string, unknown>;
+    cost?: number;
+};
+/** Normalize a scorer return value to a ScorerResult. */
+declare function normalizeScorerResult(result: number | ScorerResult): ScorerResult;
 /** Context passed to scorers by the eval runner. */
 type ScorerContext = {
     /** Resolve a provider:model URI to a provider instance and model name. */
@@ -45,7 +53,7 @@ type ScorerContext = {
         model: string;
     };
 };
-type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number;
+type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number | ScorerResult;
 type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     name: string;
     description: string;
@@ -55,7 +63,7 @@ type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly description: string;
     readonly isLlm: boolean;
-    score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | Promise<number>;
+    score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | ScorerResult | Promise<number | ScorerResult>;
 };
 declare function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
 
@@ -91,6 +99,12 @@ type EvalResult = {
     items: EvalItem[];
     summary: EvalSummary;
 };
+type ScorerDetail = {
+    score: number | null;
+    metadata?: Record<string, unknown>;
+    duration?: number;
+    cost?: number;
+};
 type EvalItem = {
     input: unknown;
     annotations?: unknown;
@@ -98,6 +112,10 @@ type EvalItem = {
     error?: string;
     scorerErrors?: string[];
     scores: Record<string, number | null>;
+    duration?: number;
+    cost?: number;
+    scorerCost?: number;
+    scoreDetails?: Record<string, ScorerDetail>;
 };
 type EvalSummary = {
     count: number;
@@ -109,6 +127,13 @@ type EvalSummary = {
         p50: number;
         p95: number;
     }>;
+    timing?: {
+        mean: number;
+        min: number;
+        max: number;
+        p50: number;
+        p95: number;
+    };
 };
 type EvalComparison = {
     baseline: {
@@ -125,11 +150,24 @@ type EvalComparison = {
         delta: number;
         deltaPercent: number;
     }>;
+    timing?: {
+        baselineMean: number;
+        candidateMean: number;
+        delta: number;
+        deltaPercent: number;
+    };
+    cost?: {
+        baselineTotal: number;
+        candidateTotal: number;
+        delta: number;
+        deltaPercent: number;
+    };
     regressions: EvalRegression[];
     improvements: EvalImprovement[];
     summary: string;
 };
 type EvalRegression = {
+    itemIndex: number;
     input: unknown;
     scorer: string;
     baselineScore: number;
@@ -147,4 +185,4 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
 
 declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
 
-export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerContext, type ScorerFn, dataset, defineEval, evalCompare, llmScorer, runEval, scorer };
+export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, runEval, scorer };

package/dist/index.d.ts

@@ -22,6 +22,14 @@ type Dataset<TInput = unknown, TAnnotations = unknown> = {
 };
 declare function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(config: DatasetConfig<TInput, TAnnotations>): Dataset<z.infer<TInput>, z.infer<TAnnotations>>;
 
+/** Result from a scorer that includes metadata beyond the numeric score. */
+type ScorerResult = {
+    score: number;
+    metadata?: Record<string, unknown>;
+    cost?: number;
+};
+/** Normalize a scorer return value to a ScorerResult. */
+declare function normalizeScorerResult(result: number | ScorerResult): ScorerResult;
 /** Context passed to scorers by the eval runner. */
 type ScorerContext = {
     /** Resolve a provider:model URI to a provider instance and model name. */
@@ -45,7 +53,7 @@ type ScorerContext = {
         model: string;
     };
 };
-type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number;
+type ScorerFn<TOutput, TInput, TAnnotations> = (output: TOutput, input: TInput, annotations?: TAnnotations) => number | ScorerResult;
 type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     name: string;
     description: string;
@@ -55,7 +63,7 @@ type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {
     readonly name: string;
     readonly description: string;
     readonly isLlm: boolean;
-    score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | Promise<number>;
+    score(output: TOutput, input: TInput, annotations?: TAnnotations, context?: ScorerContext): number | ScorerResult | Promise<number | ScorerResult>;
 };
 declare function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(config: ScorerConfig<TOutput, TInput, TAnnotations>): Scorer<TOutput, TInput, TAnnotations>;
 
@@ -91,6 +99,12 @@ type EvalResult = {
     items: EvalItem[];
     summary: EvalSummary;
 };
+type ScorerDetail = {
+    score: number | null;
+    metadata?: Record<string, unknown>;
+    duration?: number;
+    cost?: number;
+};
 type EvalItem = {
     input: unknown;
     annotations?: unknown;
@@ -98,6 +112,10 @@ type EvalItem = {
     error?: string;
     scorerErrors?: string[];
     scores: Record<string, number | null>;
+    duration?: number;
+    cost?: number;
+    scorerCost?: number;
+    scoreDetails?: Record<string, ScorerDetail>;
 };
 type EvalSummary = {
     count: number;
@@ -109,6 +127,13 @@ type EvalSummary = {
         p50: number;
         p95: number;
     }>;
+    timing?: {
+        mean: number;
+        min: number;
+        max: number;
+        p50: number;
+        p95: number;
+    };
 };
 type EvalComparison = {
     baseline: {
@@ -125,11 +150,24 @@ type EvalComparison = {
         delta: number;
         deltaPercent: number;
     }>;
+    timing?: {
+        baselineMean: number;
+        candidateMean: number;
+        delta: number;
+        deltaPercent: number;
+    };
+    cost?: {
+        baselineTotal: number;
+        candidateTotal: number;
+        delta: number;
+        deltaPercent: number;
+    };
     regressions: EvalRegression[];
     improvements: EvalImprovement[];
     summary: string;
 };
 type EvalRegression = {
+    itemIndex: number;
     input: unknown;
     scorer: string;
     baselineScore: number;
@@ -147,4 +185,4 @@ declare function runEval(config: EvalConfig, executeWorkflow: (input: unknown, r
 
 declare function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison;
 
-export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerContext, type ScorerFn, dataset, defineEval, evalCompare, llmScorer, runEval, scorer };
+export { type Dataset, type DatasetConfig, type DatasetItem, type EvalComparison, type EvalConfig, type EvalImprovement, type EvalItem, type EvalRegression, type EvalResult, type EvalSummary, type LlmScorerConfig, type Scorer, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, dataset, defineEval, evalCompare, llmScorer, normalizeScorerResult, runEval, scorer };

package/dist/index.js

@@ -1,7 +1,9 @@
 import {
   evalCompare,
-  runEval
-} from "./chunk-OHZX3VFE.js";
+  normalizeScorerResult,
+  runEval,
+  scorer
+} from "./chunk-F2AALK5U.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";
@@ -35,23 +37,13 @@ function dataset(config) {
   };
 }
 
-// src/scorer.ts
-function scorer(config) {
-  return {
-    name: config.name,
-    description: config.description,
-    isLlm: false,
-    score: config.score
-  };
-}
-
 // src/llm-scorer.ts
 import { z } from "zod";
 import { extractJson, zodToJsonSchema } from "@axlsdk/axl";
 function llmScorer(config) {
   const schema = config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });
   const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);
-  const scorerInstance = {
+  return {
     name: config.name,
     description: config.description,
     isLlm: true,
@@ -84,23 +76,34 @@ ${JSON.stringify(annotations, null, 2)}` : "",
         ],
         { model, temperature: config.temperature ?? 0.2, responseFormat: { type: "json_object" } }
       );
-      scorerInstance._lastCost = response.cost;
-      const parsed = JSON.parse(extractJson(response.content));
+      const responseCost = response.cost;
       let validated;
       try {
+        const parsed = JSON.parse(extractJson(response.content));
         validated = schema.parse(parsed);
       } catch (err) {
+        if (err && typeof err === "object") {
+          err.cost = responseCost;
+        }
         if (err && typeof err === "object" && "issues" in err && Array.isArray(err.issues)) {
           const issues = err.issues;
           const messages = issues.map((i) => `${i.path.length ? `${i.path.join(".")}: ` : ""}${i.message}`).join("; ");
-          throw new Error(`LLM scorer "${config.name}" returned an invalid response: ${messages}`);
+          const error = new Error(
+            `LLM scorer "${config.name}" returned an invalid response: ${messages}`
+          );
+          error.cost = responseCost;
+          throw error;
         }
         throw err;
       }
-      return validated.score;
+      const { score, ...rest } = validated;
+      return {
+        score,
+        metadata: Object.keys(rest).length > 0 ? rest : void 0,
+        cost: responseCost
+      };
     }
   };
-  return scorerInstance;
 }
 
 // src/define-eval.ts
@@ -112,6 +115,7 @@ export {
   defineEval,
   evalCompare,
   llmScorer,
+  normalizeScorerResult,
   runEval,
   scorer
 };

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => {\n    provider: {\n      chat(\n        messages: { role: string; content: string }[],\n        options: {\n          model: string;\n          temperature?: number;\n          responseFormat?: { type: string; json_schema?: unknown };\n        },\n      ): Promise<{ content: string; cost?: number }>;\n    };\n    model: string;\n  };\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | Promise<number>;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  const scorerInstance: Scorer & { _lastCost?: number } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<number> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n      );\n\n      scorerInstance._lastCost = response.cost;\n\n      const parsed = JSON.parse(extractJson(response.content));\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          throw new Error(`LLM scorer \"${config.name}\" returned an invalid response: ${messages}`);\n        }\n        throw err;\n      }\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;ACnBO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;ACnDA,SAAS,SAAS;AAElB,SAAS,aAAa,uBAAuB;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,QAAM,iBAAkD;AAAA,IACtD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACiB;AACjB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,qBAAe,YAAY,SAAS;AAEpC,YAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,UAAI;AACJ,UAAI;AACF,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAGZ,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,IAAI,MAAM,eAAe,OAAO,IAAI,mCAAmC,QAAQ,EAAE;AAAA,QACzF;AACA,cAAM;AAAA,MACR;AACA,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;AC5FO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}
+{"version":3,"sources":["../src/dataset.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<TInput extends z.ZodType, TAnnotations extends z.ZodType = z.ZodType> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodType;\n  readonly annotationsSchema?: z.ZodType;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodType, TAnnotations extends z.ZodType>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","import { z } from 'zod';\nimport type { Scorer, ScorerContext, ScorerResult } from './scorer.js';\nimport { extractJson, zodToJsonSchema } from '@axlsdk/axl';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema?: z.ZodType<{ score: number; [key: string]: unknown }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  // Resolve schema and its JSON representation once at construction time —\n  // both are fixed for the lifetime of this scorer instance.\n  const schema: z.ZodType<{ score: number; [key: string]: unknown }> =\n    config.schema ?? z.object({ score: z.number().min(0).max(1), reasoning: z.string() });\n  const schemaJson = JSON.stringify(zodToJsonSchema(schema), null, 2);\n\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(\n      output: unknown,\n      input: unknown,\n      annotations?: unknown,\n      context?: ScorerContext,\n    ): Promise<ScorerResult> {\n      if (!context?.resolveProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" has no provider. ` +\n            `Ensure you are running via runEval() with a real AxlRuntime instance.`,\n        );\n      }\n\n      const { provider, model } = context.resolveProvider(config.model);\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching this schema:`,\n        schemaJson,\n      ].join('\\n');\n\n      const response = await provider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2, responseFormat: { type: 'json_object' } },\n      );\n\n      const responseCost = response.cost;\n\n      let validated: { score: number; [key: string]: unknown };\n      try {\n        const parsed = JSON.parse(extractJson(response.content));\n        validated = schema.parse(parsed) as { score: number; [key: string]: unknown };\n      } catch (err) {\n        // Attach cost to all errors so the runner can capture it even on failure.\n        if (err && typeof err === 'object') {\n          (err as any).cost = responseCost;\n        }\n        // Duck-type check instead of instanceof to handle potential dual-instance\n        // scenarios where two copies of zod are present in the dependency tree.\n        if (\n          err &&\n          typeof err === 'object' &&\n          'issues' in err &&\n          Array.isArray((err as any).issues)\n        ) {\n          const issues = (err as any).issues as Array<{\n            path: (string | number)[];\n            message: string;\n          }>;\n          const messages = issues\n            .map((i) => `${i.path.length ? `${i.path.join('.')}: ` : ''}${i.message}`)\n            .join('; ');\n          const error = new Error(\n            `LLM scorer \"${config.name}\" returned an invalid response: ${messages}`,\n          );\n          (error as any).cost = responseCost;\n          throw error;\n        }\n        throw err;\n      }\n\n      const { score, ...rest } = validated;\n      return {\n        score,\n        metadata: Object.keys(rest).length > 0 ? rest : undefined,\n        cost: responseCost,\n      };\n    },\n  };\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AAwBf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC7DA,SAAS,SAAS;AAElB,SAAS,aAAa,uBAAuB;AAWtC,SAAS,UAAU,QAAiC;AAGzD,QAAM,SACJ,OAAO,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,WAAW,EAAE,OAAO,EAAE,CAAC;AACtF,QAAM,aAAa,KAAK,UAAU,gBAAgB,MAAM,GAAG,MAAM,CAAC;AAElE,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MACJ,QACA,OACA,aACA,SACuB;AACvB,UAAI,CAAC,SAAS,iBAAiB;AAC7B,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAE5B;AAAA,MACF;AAEA,YAAM,EAAE,UAAU,MAAM,IAAI,QAAQ,gBAAgB,OAAO,KAAK;AAEhE,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,MAAM,SAAS;AAAA,QAC9B;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,KAAK,gBAAgB,EAAE,MAAM,cAAc,EAAE;AAAA,MAC3F;AAEA,YAAM,eAAe,SAAS;AAE9B,UAAI;AACJ,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,YAAY,SAAS,OAAO,CAAC;AACvD,oBAAY,OAAO,MAAM,MAAM;AAAA,MACjC,SAAS,KAAK;AAEZ,YAAI,OAAO,OAAO,QAAQ,UAAU;AAClC,UAAC,IAAY,OAAO;AAAA,QACtB;AAGA,YACE,OACA,OAAO,QAAQ,YACf,YAAY,OACZ,MAAM,QAAS,IAAY,MAAM,GACjC;AACA,gBAAM,SAAU,IAAY;AAI5B,gBAAM,WAAW,OACd,IAAI,CAAC,MAAM,GAAG,EAAE,KAAK,SAAS,GAAG,EAAE,KAAK,KAAK,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,EACxE,KAAK,IAAI;AACZ,gBAAM,QAAQ,IAAI;AAAA,YAChB,eAAe,OAAO,IAAI,mCAAmC,QAAQ;AAAA,UACvE;AACA,UAAC,MAAc,OAAO;AACtB,gBAAM;AAAA,QACR;AACA,cAAM;AAAA,MACR;AAEA,YAAM,EAAE,OAAO,GAAG,KAAK,IAAI;AAC3B,aAAO;AAAA,QACL;AAAA,QACA,UAAU,OAAO,KAAK,IAAI,EAAE,SAAS,IAAI,OAAO;AAAA,QAChD,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACF;;;ACxGO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.13.4",
+  "version": "0.13.6",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.13.4"
+    "@axlsdk/axl": "0.13.6"
   },
   "peerDependencies": {
     "zod": "^4.0.0"

package/dist/chunk-OHZX3VFE.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/compare.ts","../src/runner.ts"],"sourcesContent":["import type { EvalResult, EvalComparison, EvalRegression, EvalImprovement } from './types.js';\n\nfunction round(n: number): number {\n  return Math.round(n * 100) / 100;\n}\n\nexport function evalCompare(baseline: EvalResult, candidate: EvalResult): EvalComparison {\n  if (baseline.dataset !== candidate.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baseline.dataset}\" vs \"${candidate.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baseline.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidate.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMean = baseline.summary.scorers[name].mean;\n    const cMean = candidate.summary.scorers[name].mean;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round((delta / bMean) * 100) : 0;\n    scorers[name] = { baselineMean: bMean, candidateMean: cMean, delta, deltaPercent };\n  }\n\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const minLength = Math.min(baseline.items.length, candidate.items.length);\n  for (let i = 0; i < minLength; i++) {\n    const bItem = baseline.items[i];\n    const cItem = candidate.items[i];\n    if (bItem.error || cItem.error) continue;\n    for (const name of baselineScorerNames) {\n      const bScore = bItem.scores[name];\n      const cScore = cItem.scores[name];\n      if (bScore == null || cScore == null) continue;\n      const delta = round(cScore - bScore);\n      if (delta < -0.1)\n        regressions.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n      else if (delta > 0.1)\n        improvements.push({\n          input: bItem.input,\n          scorer: name,\n          baselineScore: bScore,\n          candidateScore: cScore,\n          delta,\n        });\n    }\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})`,\n      );\n    }\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no significant changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  return {\n    baseline: { id: baseline.id, metadata: baseline.metadata },\n    candidate: { id: candidate.id, metadata: candidate.metadata },\n    scorers,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number }>,\n  runtime: AxlRuntime,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  const concurrency = config.concurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n  };\n\n  const evalItems: EvalItem[] = [];\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0]): Promise<void> {\n    if (budgetExceeded) {\n      evalItems.push({\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    try {\n      const result = await executeWorkflow(item.input, runtime);\n      evalItem.output = result.output;\n      if (result.cost != null) {\n        totalCost += result.cost;\n      }\n    } catch (err) {\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      evalItems.push(evalItem);\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    for (const scorer of config.scorers) {\n      try {\n        const score = await scorer.score(\n          evalItem.output,\n          item.input,\n          item.annotations,\n          scorerContext,\n        );\n        // Accumulate LLM scorer cost if available\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!Number.isFinite(score) || score < 0 || score > 1) {\n          if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n          evalItem.scorerErrors.push(\n            `Scorer \"${scorer.name}\" returned out-of-range score ${score} for input ${JSON.stringify(item.input)}`,\n          );\n          evalItem.scores[scorer.name] = null;\n        } else {\n          evalItem.scores[scorer.name] = round(score);\n        }\n      } catch (err) {\n        // Accumulate cost even on failure — the LLM call may have succeeded\n        // before a downstream error (e.g., invalid JSON, schema validation)\n        if (scorer.isLlm) {\n          const lastCost = (scorer as unknown as { _lastCost?: number })._lastCost;\n          if (lastCost != null) {\n            totalCost += lastCost;\n          }\n        }\n        if (!evalItem.scorerErrors) evalItem.scorerErrors = [];\n        evalItem.scorerErrors.push(\n          `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`,\n        );\n        evalItem.scores[scorer.name] = null;\n      }\n    }\n    evalItems.push(evalItem);\n  }\n\n  let index = 0;\n  async function runNext(): Promise<void> {\n    while (index < items.length) {\n      const currentIndex = index++;\n      await processItem(items[currentIndex]);\n    }\n  }\n\n  const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => runNext());\n  await Promise.all(workers);\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    scorerStats[name] = computeStats(scores);\n  }\n\n  return {\n    id,\n    workflow: config.workflow,\n    dataset: config.dataset.name,\n    metadata: config.metadata ?? {},\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: { count: items.length, failures, scorers: scorerStats },\n  };\n}\n"],"mappings":";AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAG,IAAI;AAC/B;AAEO,SAAS,YAAY,UAAsB,WAAuC;AACvF,MAAI,SAAS,YAAY,UAAU,SAAS;AAC1C,UAAM,IAAI;AAAA,MACR,kDAAkD,SAAS,OAAO,SAAS,UAAU,OAAO;AAAA,IAC9F;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,SAAS,QAAQ,OAAO,EAAE,KAAK;AACvE,QAAM,uBAAuB,OAAO,KAAK,UAAU,QAAQ,OAAO,EAAE,KAAK;AACzE,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,QAAQ,SAAS,QAAQ,QAAQ,IAAI,EAAE;AAC7C,UAAM,QAAQ,UAAU,QAAQ,QAAQ,IAAI,EAAE;AAC9C,UAAM,QAAQ,MAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAI,MAAO,QAAQ,QAAS,GAAG,IAAI;AAChE,YAAQ,IAAI,IAAI,EAAE,cAAc,OAAO,eAAe,OAAO,OAAO,aAAa;AAAA,EACnF;AAEA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK,IAAI,SAAS,MAAM,QAAQ,UAAU,MAAM,MAAM;AACxE,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,SAAS,MAAM,CAAC;AAC9B,UAAM,QAAQ,UAAU,MAAM,CAAC;AAC/B,QAAI,MAAM,SAAS,MAAM,MAAO;AAChC,eAAW,QAAQ,qBAAqB;AACtC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAM,SAAS,MAAM,OAAO,IAAI;AAChC,UAAI,UAAU,QAAQ,UAAU,KAAM;AACtC,YAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,UAAI,QAAQ;AACV,oBAAY,KAAK;AAAA,UACf,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,OAAO,MAAM;AAAA,UACb,QAAQ;AAAA,UACR,eAAe;AAAA,UACf,gBAAgB;AAAA,UAChB;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC;AAAA,MAC/H;AAAA,IACF;AAAA,EACF;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,wBAAwB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAEhK,SAAO;AAAA,IACL,UAAU,EAAE,IAAI,SAAS,IAAI,UAAU,SAAS,SAAS;AAAA,IACzD,WAAW,EAAE,IAAI,UAAU,IAAI,UAAU,UAAU,SAAS;AAAA,IAC5D;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;;;AC9EA,SAAS,kBAAkB;AAE3B,SAAS,aAAa,QAMpB;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEA,SAASA,OAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAEA,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAEA,eAAsB,QACpB,QACA,iBAIA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAC5C,QAAM,cAAc,OAAO,eAAe;AAC1C,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,EACF;AAEA,QAAM,YAAwB,CAAC;AAC/B,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAwC;AACjE,QAAI,gBAAgB;AAClB,gBAAU,KAAK;AAAA,QACb,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX,CAAC;AACD;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,QAAI;AACF,YAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,eAAS,SAAS,OAAO;AACzB,UAAI,OAAO,QAAQ,MAAM;AACvB,qBAAa,OAAO;AAAA,MACtB;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAChE,gBAAU,KAAK,QAAQ;AACvB;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAEA,eAAW,UAAU,OAAO,SAAS;AACnC,UAAI;AACF,cAAM,QAAQ,MAAM,OAAO;AAAA,UACzB,SAAS;AAAA,UACT,KAAK;AAAA,UACL,KAAK;AAAA,UACL;AAAA,QACF;AAEA,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,KAAK,QAAQ,GAAG;AACrD,cAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,mBAAS,aAAa;AAAA,YACpB,WAAW,OAAO,IAAI,iCAAiC,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,UACtG;AACA,mBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,QACjC,OAAO;AACL,mBAAS,OAAO,OAAO,IAAI,IAAIA,OAAM,KAAK;AAAA,QAC5C;AAAA,MACF,SAAS,KAAK;AAGZ,YAAI,OAAO,OAAO;AAChB,gBAAM,WAAY,OAA6C;AAC/D,cAAI,YAAY,MAAM;AACpB,yBAAa;AAAA,UACf;AAAA,QACF;AACA,YAAI,CAAC,SAAS,aAAc,UAAS,eAAe,CAAC;AACrD,iBAAS,aAAa;AAAA,UACpB,WAAW,OAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,QACpF;AACA,iBAAS,OAAO,OAAO,IAAI,IAAI;AAAA,MACjC;AAAA,IACF;AACA,cAAU,KAAK,QAAQ;AAAA,EACzB;AAEA,MAAI,QAAQ;AACZ,iBAAe,UAAyB;AACtC,WAAO,QAAQ,MAAM,QAAQ;AAC3B,YAAM,eAAe;AACrB,YAAM,YAAY,MAAM,YAAY,CAAC;AAAA,IACvC;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC;AAC3F,QAAM,QAAQ,IAAI,OAAO;AAEzB,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,gBAAY,IAAI,IAAI,aAAa,MAAM;AAAA,EACzC;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,OAAO;AAAA,IACjB,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU,OAAO,YAAY,CAAC;AAAA,IAC9B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACjE;AACF;","names":["round"]}