@axlsdk/eval 0.1.0

package/dist/index.js

@@ -0,0 +1,104 @@
+import {
+  evalCompare,
+  runEval
+} from "./chunk-F5UZX6FP.js";
+
+// src/dataset.ts
+import { readFile } from "fs/promises";
+import * as path from "path";
+function dataset(config) {
+  return {
+    name: config.name,
+    schema: config.schema,
+    annotationsSchema: config.annotations,
+    async getItems() {
+      if (config.items && config.file) {
+        throw new Error('Dataset config error: "items" and "file" are mutually exclusive');
+      }
+      if (!config.items && !config.file) {
+        throw new Error('Dataset config error: either "items" or "file" must be provided');
+      }
+      if (config.items) {
+        return config.items.map((item) => ({
+          input: config.schema.parse(item.input),
+          annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
+        }));
+      }
+      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file);
+      const content = await readFile(resolvedPath, "utf-8");
+      const items = JSON.parse(content);
+      return items.map((item) => ({
+        input: config.schema.parse(item.input),
+        annotations: item.annotations && config.annotations ? config.annotations.parse(item.annotations) : item.annotations
+      }));
+    }
+  };
+}
+
+// src/scorer.ts
+function scorer(config) {
+  return {
+    name: config.name,
+    description: config.description,
+    isLlm: false,
+    score: config.score
+  };
+}
+
+// src/llm-scorer.ts
+function llmScorer(config) {
+  const scorerInstance = {
+    name: config.name,
+    description: config.description,
+    isLlm: true,
+    async score(output, input, annotations) {
+      const injectedProvider = scorerInstance._provider;
+      if (!injectedProvider) {
+        throw new Error(
+          `LLM scorer "${config.name}" requires a provider. Run via axl.eval() or the CLI.`
+        );
+      }
+      const prompt = [
+        `Evaluate the following output.`,
+        ``,
+        `## Input`,
+        `${JSON.stringify(input, null, 2)}`,
+        annotations ? `
+## Annotations (Ground Truth)
+${JSON.stringify(annotations, null, 2)}` : "",
+        ``,
+        `## Output to Evaluate`,
+        `${JSON.stringify(output, null, 2)}`,
+        ``,
+        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`
+      ].join("\n");
+      const colonIdx = config.model.indexOf(":");
+      const model = colonIdx > -1 ? config.model.slice(colonIdx + 1) : config.model;
+      const response = await injectedProvider.chat(
+        [
+          { role: "system", content: config.system },
+          { role: "user", content: prompt }
+        ],
+        { model, temperature: config.temperature ?? 0.2 }
+      );
+      const parsed = JSON.parse(response.content);
+      const validated = config.schema.parse(parsed);
+      return validated.score;
+    }
+  };
+  return scorerInstance;
+}
+
+// src/define-eval.ts
+function defineEval(config) {
+  return config;
+}
+export {
+  dataset,
+  defineEval,
+  evalCompare,
+  llmScorer,
+  runEval,
+  scorer
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/dataset.ts","../src/scorer.ts","../src/llm-scorer.ts","../src/define-eval.ts"],"sourcesContent":["import type { z } from 'zod';\nimport { readFile } from 'node:fs/promises';\nimport * as path from 'node:path';\n\nexport type DatasetItem<TInput, TAnnotations = undefined> = {\n  input: TInput;\n  annotations?: TAnnotations;\n};\n\nexport type DatasetConfig<\n  TInput extends z.ZodTypeAny,\n  TAnnotations extends z.ZodTypeAny = z.ZodTypeAny,\n> = {\n  name: string;\n  schema: TInput;\n  annotations?: TAnnotations;\n  items?: DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n  file?: string;\n  /** Base directory for resolving relative file paths. Defaults to cwd. */\n  basePath?: string;\n};\n\nexport type Dataset<TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly schema: z.ZodTypeAny;\n  readonly annotationsSchema?: z.ZodTypeAny;\n  getItems(): Promise<DatasetItem<TInput, TAnnotations>[]>;\n};\n\nexport function dataset<TInput extends z.ZodTypeAny, TAnnotations extends z.ZodTypeAny>(\n  config: DatasetConfig<TInput, TAnnotations>,\n): Dataset<z.infer<TInput>, z.infer<TAnnotations>> {\n  return {\n    name: config.name,\n    schema: config.schema,\n    annotationsSchema: config.annotations,\n    async getItems() {\n      if (config.items && config.file) {\n        throw new Error('Dataset config error: \"items\" and \"file\" are mutually exclusive');\n      }\n      if (!config.items && !config.file) {\n        throw new Error('Dataset config error: either \"items\" or \"file\" must be provided');\n      }\n      if (config.items) {\n        return config.items.map((item) => ({\n          input: config.schema.parse(item.input),\n          annotations:\n            item.annotations && config.annotations\n              ? config.annotations.parse(item.annotations)\n              : item.annotations,\n        }));\n      }\n      const resolvedPath = path.resolve(config.basePath ?? process.cwd(), config.file!);\n      const content = await readFile(resolvedPath, 'utf-8');\n      const items = JSON.parse(content) as DatasetItem<z.infer<TInput>, z.infer<TAnnotations>>[];\n      return items.map((item) => ({\n        input: config.schema.parse(item.input),\n        annotations:\n          item.annotations && config.annotations\n            ? config.annotations.parse(item.annotations)\n            : item.annotations,\n      }));\n    },\n  };\n}\n","export type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number;\n\nexport type ScorerConfig<TOutput = any, TInput = any, TAnnotations = any> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n};\n\nexport type Scorer<TOutput = any, TInput = any, TAnnotations = any> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(output: TOutput, input: TInput, annotations?: TAnnotations): number | Promise<number>;\n};\n\nexport function scorer<TOutput = any, TInput = any, TAnnotations = any>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n  };\n}\n","import type { z } from 'zod';\nimport type { Scorer } from './scorer.js';\n\nexport type LlmScorerConfig = {\n  name: string;\n  description: string;\n  model: string;\n  system: string;\n  schema: z.ZodType<{ score: number; [key: string]: any }>;\n  temperature?: number;\n};\n\nexport function llmScorer(config: LlmScorerConfig): Scorer {\n  const scorerInstance: Scorer & { _provider?: any } = {\n    name: config.name,\n    description: config.description,\n    isLlm: true,\n    async score(output: any, input: any, annotations?: any): Promise<number> {\n      const injectedProvider = (scorerInstance as any)._provider;\n      if (!injectedProvider) {\n        throw new Error(\n          `LLM scorer \"${config.name}\" requires a provider. Run via axl.eval() or the CLI.`,\n        );\n      }\n\n      const prompt = [\n        `Evaluate the following output.`,\n        ``,\n        `## Input`,\n        `${JSON.stringify(input, null, 2)}`,\n        annotations\n          ? `\\n## Annotations (Ground Truth)\\n${JSON.stringify(annotations, null, 2)}`\n          : '',\n        ``,\n        `## Output to Evaluate`,\n        `${JSON.stringify(output, null, 2)}`,\n        ``,\n        `Respond with valid JSON matching the required schema with a score field (0-1) and reasoning.`,\n      ].join('\\n');\n\n      const colonIdx = config.model.indexOf(':');\n      const model = colonIdx > -1 ? config.model.slice(colonIdx + 1) : config.model;\n\n      const response = await injectedProvider.chat(\n        [\n          { role: 'system', content: config.system },\n          { role: 'user', content: prompt },\n        ],\n        { model, temperature: config.temperature ?? 0.2 },\n      );\n\n      const parsed = JSON.parse(response.content);\n      const validated = config.schema.parse(parsed);\n      return validated.score;\n    },\n  };\n\n  return scorerInstance;\n}\n","import type { EvalConfig } from './types.js';\n\nexport function defineEval(config: EvalConfig): EvalConfig {\n  return config;\n}\n"],"mappings":";;;;;;AACA,SAAS,gBAAgB;AACzB,YAAY,UAAU;AA2Bf,SAAS,QACd,QACiD;AACjD,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,QAAQ,OAAO;AAAA,IACf,mBAAmB,OAAO;AAAA,IAC1B,MAAM,WAAW;AACf,UAAI,OAAO,SAAS,OAAO,MAAM;AAC/B,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,CAAC,OAAO,SAAS,CAAC,OAAO,MAAM;AACjC,cAAM,IAAI,MAAM,iEAAiE;AAAA,MACnF;AACA,UAAI,OAAO,OAAO;AAChB,eAAO,OAAO,MAAM,IAAI,CAAC,UAAU;AAAA,UACjC,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,UACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,QACb,EAAE;AAAA,MACJ;AACA,YAAM,eAAoB,aAAQ,OAAO,YAAY,QAAQ,IAAI,GAAG,OAAO,IAAK;AAChF,YAAM,UAAU,MAAM,SAAS,cAAc,OAAO;AACpD,YAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,aAAO,MAAM,IAAI,CAAC,UAAU;AAAA,QAC1B,OAAO,OAAO,OAAO,MAAM,KAAK,KAAK;AAAA,QACrC,aACE,KAAK,eAAe,OAAO,cACvB,OAAO,YAAY,MAAM,KAAK,WAAW,IACzC,KAAK;AAAA,MACb,EAAE;AAAA,IACJ;AAAA,EACF;AACF;;;AC7CO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,EAChB;AACF;;;AChBO,SAAS,UAAU,QAAiC;AACzD,QAAM,iBAA+C;AAAA,IACnD,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,MAAM,MAAM,QAAa,OAAY,aAAoC;AACvE,YAAM,mBAAoB,eAAuB;AACjD,UAAI,CAAC,kBAAkB;AACrB,cAAM,IAAI;AAAA,UACR,eAAe,OAAO,IAAI;AAAA,QAC5B;AAAA,MACF;AAEA,YAAM,SAAS;AAAA,QACb;AAAA,QACA;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,OAAO,MAAM,CAAC,CAAC;AAAA,QACjC,cACI;AAAA;AAAA,EAAoC,KAAK,UAAU,aAAa,MAAM,CAAC,CAAC,KACxE;AAAA,QACJ;AAAA,QACA;AAAA,QACA,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,QAClC;AAAA,QACA;AAAA,MACF,EAAE,KAAK,IAAI;AAEX,YAAM,WAAW,OAAO,MAAM,QAAQ,GAAG;AACzC,YAAM,QAAQ,WAAW,KAAK,OAAO,MAAM,MAAM,WAAW,CAAC,IAAI,OAAO;AAExE,YAAM,WAAW,MAAM,iBAAiB;AAAA,QACtC;AAAA,UACE,EAAE,MAAM,UAAU,SAAS,OAAO,OAAO;AAAA,UACzC,EAAE,MAAM,QAAQ,SAAS,OAAO;AAAA,QAClC;AAAA,QACA,EAAE,OAAO,aAAa,OAAO,eAAe,IAAI;AAAA,MAClD;AAEA,YAAM,SAAS,KAAK,MAAM,SAAS,OAAO;AAC1C,YAAM,YAAY,OAAO,OAAO,MAAM,MAAM;AAC5C,aAAO,UAAU;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AACT;;;ACxDO,SAAS,WAAW,QAAgC;AACzD,SAAO;AACT;","names":[]}

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@axlsdk/eval",
+  "version": "0.1.0",
+  "description": "Evaluation framework for Axl agentic workflows",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    }
+  },
+  "bin": {
+    "axl-eval": "./dist/cli.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "lint": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@axlsdk/axl": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^25.2.3",
+    "tsup": "^8.3.0",
+    "typescript": "^5.7.0",
+    "vitest": "^3.0.0",
+    "zod": "^3.24.0"
+  },
+  "keywords": [
+    "ai",
+    "evaluation",
+    "scoring",
+    "agents"
+  ],
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/axl-sdk/axl.git",
+    "directory": "packages/axl-eval"
+  },
+  "homepage": "https://github.com/axl-sdk/axl#readme",
+  "bugs": {
+    "url": "https://github.com/axl-sdk/axl/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}