@m4trix/evals 0.1.0 → 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Pascal Lohscheidt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,163 @@
+[![CircleCI](https://dl.circleci.com/status-badge/img/gh/Pascal-Lohscheidt/build-ai/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/Pascal-Lohscheidt/build-ai/tree/main)
+[![npm version](https://img.shields.io/npm/v/@m4trix%2Fevals)](https://www.npmjs.com/package/@m4trix/evals)
+[![license](https://img.shields.io/npm/l/@m4trix%2Fevals)](https://www.npmjs.com/package/@m4trix/evals)
+
+# @m4trix/evals
+
+`@m4trix/evals` helps you define datasets, test cases, and evaluators for repeatable AI evaluation runs.
+
+## Quick Start
+
+From the repository root:
+
+```bash
+pnpm install
+pnpm run evals:build
+```
+
+Run the bundled example project:
+
+```bash
+cd examples/evals-example
+pnpm run eval:run
+```
+
+Generate a dataset case file from the example:
+
+```bash
+pnpm run eval:generate
+```
+
+## Set Up Your First Eval
+
+Create files under your project (for example, `src/evals/`) with these suffixes:
+
+- `*.dataset.ts`
+- `*.evaluator.ts`
+- `*.test-case.ts`
+
+Optional: create `m4trix-eval.config.ts` at your project root to customize discovery and output paths.
+
+```ts
+import { defineConfigFunction, type ConfigType } from '@m4trix/evals';
+
+export default defineConfigFunction((): ConfigType => ({
+  discovery: {
+    rootDir: 'src/evals',
+    datasetFilePatterns: ['.dataset.ts'],
+    evaluatorFilePatterns: ['.evaluator.ts'],
+    testCaseFilePatterns: ['.test-case.ts'],
+    excludeDirectories: ['node_modules', 'dist'],
+  },
+  artifactDirectory: 'src/evals/.eval-results',
+}));
+```
+
+### 1) Dataset
+
+```ts
+import { Dataset } from '@m4trix/evals';
+
+export const myDataset = Dataset.define({
+  name: 'My Dataset',
+  includedTags: ['demo'],
+});
+```
+
+### 2) Evaluator
+
+```ts
+import { Evaluator, latencyMetric, percentScore, tokenCountMetric } from '@m4trix/evals';
+import { Schema as S } from 'effect';
+
+const inputSchema = S.Struct({ prompt: S.String });
+
+export const myEvaluator = Evaluator.define({
+  name: 'My Evaluator',
+  inputSchema,
+  outputSchema: S.Unknown,
+  scoreSchema: S.Struct({ scores: S.Array(S.Unknown) }),
+}).evaluate(async (input) => {
+  const start = Date.now();
+  const latencyMs = Date.now() - start;
+
+  return {
+    scores: [
+      percentScore.make({ value: 85 }, { definePassed: (d) => d.value >= 50 }),
+    ],
+    metrics: [
+      tokenCountMetric.make({
+        input: input.prompt.length,
+        output: input.prompt.length,
+        inputCached: 0,
+        outputCached: 0,
+      }),
+      latencyMetric.make({ ms: latencyMs }),
+    ],
+  };
+});
+```
+
+### 3) Test Case
+
+```ts
+import { TestCase } from '@m4trix/evals';
+import { Schema as S } from 'effect';
+
+export const myTestCase = TestCase.describe({
+  name: 'my test case',
+  tags: ['demo'],
+  inputSchema: S.Struct({ prompt: S.String }),
+  input: { prompt: 'Hello from my first eval' },
+});
+```
+
+### 4) Run
+
+```bash
+eval-agents-simple run --dataset "My Dataset" --evaluator "My Evaluator"
+```
+
+You can also use patterns:
+
+```bash
+eval-agents-simple run --dataset "*My*" --evaluator "*My*"
+```
+
+## CLI Commands
+
+- `eval-agents`: interactive CLI
+- `eval-agents-simple run --dataset "<name or pattern>" --evaluator "<name or pattern>"`
+- `eval-agents-simple generate --dataset "<dataset name>"`
+
+## Default Discovery and Artifacts
+
+By default, the runner uses `process.cwd()` as discovery root and scans for:
+
+- Datasets: `.dataset.ts`, `.dataset.tsx`, `.dataset.js`, `.dataset.mjs`
+- Evaluators: `.evaluator.ts`, `.evaluator.tsx`, `.evaluator.js`, `.evaluator.mjs`
+- Test cases: `.test-case.ts`, `.test-case.tsx`, `.test-case.js`, `.test-case.mjs`
+
+Results are written to `.eval-results`.
+
+## Config File
+
+When present, `m4trix-eval.config.ts` is loaded automatically from `process.cwd()`.
+
+- Config API: `defineConfigFunction(() => ConfigType)`
+- Supported exports: default object, or default function that returns config
+- Discovery keys:
+  - `datasetFilePatterns` (or `datasetSuffixes`)
+  - `evaluatorFilePatterns` (or `evaluatorSuffixes`)
+  - `testCaseFilePatterns` (or `testCaseSuffixes`)
+  - `rootDir`, `excludeDirectories`
+
+Precedence is:
+
+1. built-in defaults
+2. `m4trix-eval.config.ts`
+3. explicit `createRunner({...})` overrides
+
+## License
+
+MIT

package/dist/cli-simple.cjs

@@ -3,11 +3,33 @@
 
 var crypto = require('crypto');
 var effect = require('effect');
-var promises = require('fs/promises');
+var fs = require('fs');
 var path = require('path');
+var jitiModule = require('jiti');
+var promises = require('fs/promises');
 var url = require('url');
 
 var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
+function _interopNamespace(e) {
+  if (e && e.__esModule) return e;
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var jitiModule__namespace = /*#__PURE__*/_interopNamespace(jitiModule);
+
 // src/runner/config.ts
 var defaultRunnerConfig = {
   discovery: {
@@ -29,10 +51,102 @@ var defaultRunnerConfig = {
   },
   artifactDirectory: ".eval-results"
 };
+function toRunnerConfigOverrides(config) {
+  if (!config) {
+    return void 0;
+  }
+  const rawDiscovery = config.discovery;
+  const discovery = {};
+  if (rawDiscovery?.rootDir !== void 0) {
+    discovery.rootDir = rawDiscovery.rootDir;
+  }
+  if (rawDiscovery?.datasetFilePatterns !== void 0) {
+    discovery.datasetSuffixes = rawDiscovery.datasetFilePatterns;
+  } else if (rawDiscovery?.datasetSuffixes !== void 0) {
+    discovery.datasetSuffixes = rawDiscovery.datasetSuffixes;
+  }
+  if (rawDiscovery?.evaluatorFilePatterns !== void 0) {
+    discovery.evaluatorSuffixes = rawDiscovery.evaluatorFilePatterns;
+  } else if (rawDiscovery?.evaluatorSuffixes !== void 0) {
+    discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;
+  }
+  if (rawDiscovery?.testCaseFilePatterns !== void 0) {
+    discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;
+  } else if (rawDiscovery?.testCaseSuffixes !== void 0) {
+    discovery.testCaseSuffixes = rawDiscovery.testCaseSuffixes;
+  }
+  if (rawDiscovery?.excludeDirectories !== void 0) {
+    discovery.excludeDirectories = rawDiscovery.excludeDirectories;
+  }
+  const overrides = {};
+  if (config.artifactDirectory !== void 0) {
+    overrides.artifactDirectory = config.artifactDirectory;
+  }
+  if (Object.keys(discovery).length > 0) {
+    overrides.discovery = discovery;
+  }
+  return overrides;
+}
 function withRunnerConfig(overrides) {
-  {
+  if (!overrides) {
     return defaultRunnerConfig;
   }
+  const discovery = overrides.discovery ? {
+    ...defaultRunnerConfig.discovery,
+    ...overrides.discovery
+  } : defaultRunnerConfig.discovery;
+  return {
+    ...defaultRunnerConfig,
+    ...overrides,
+    discovery
+  };
+}
+var CONFIG_FILE_NAME = "m4trix-eval.config.ts";
+var cachedLoader;
+function getJitiLoader() {
+  if (cachedLoader) {
+    return cachedLoader;
+  }
+  const createJiti2 = jitiModule__namespace.createJiti ?? jitiModule__namespace.default;
+  if (typeof createJiti2 !== "function") {
+    throw new Error("Failed to initialize jiti for m4trix eval config loading.");
+  }
+  cachedLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
+    interopDefault: true,
+    moduleCache: true
+  });
+  return cachedLoader;
+}
+function resolveConfigModuleExport(loadedModule) {
+  if (loadedModule && typeof loadedModule === "object" && "default" in loadedModule) {
+    return loadedModule.default;
+  }
+  return loadedModule;
+}
+function resolveConfigValue(value) {
+  if (value === void 0 || value === null) {
+    return void 0;
+  }
+  if (typeof value === "function") {
+    return value();
+  }
+  if (typeof value !== "object") {
+    throw new Error(
+      "Invalid m4trix eval config export. Expected an object or defineConfigFunction(() => config)."
+    );
+  }
+  return value;
+}
+function loadRunnerConfigFile(cwd = process.cwd()) {
+  const configPath = path.resolve(cwd, CONFIG_FILE_NAME);
+  if (!fs.existsSync(configPath)) {
+    return void 0;
+  }
+  const loader = getJitiLoader();
+  const loaded = loader(configPath);
+  const exportedValue = resolveConfigModuleExport(loaded);
+  const config = resolveConfigValue(exportedValue);
+  return toRunnerConfigOverrides(config);
 }
 var jitiLoader;
 function toId(prefix, filePath, name) {
@@ -85,12 +199,12 @@ function hasOneSuffix(filePath, suffixes) {
 async function loadModuleExports(filePath) {
   if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) {
     if (!jitiLoader) {
-      const jitiModule = await import('jiti');
-      const createJiti = jitiModule.createJiti ?? jitiModule.default;
-      if (!createJiti) {
+      const jitiModule2 = await import('jiti');
+      const createJiti2 = jitiModule2.createJiti ?? jitiModule2.default;
+      if (!createJiti2) {
         throw new Error("Failed to initialize jiti TypeScript loader");
       }
-      jitiLoader = createJiti((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
+      jitiLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
         interopDefault: true,
         moduleCache: true
       });
@@ -498,8 +612,18 @@ function createNameMatcher(pattern) {
   }
   return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
 }
+function mergeRunnerOverrides(base, next) {
+  if (!base) {
+    return next;
+  }
+  {
+    return base;
+  }
+}
 function createRunner(overrides) {
-  return new EffectRunner(withRunnerConfig());
+  const fileOverrides = loadRunnerConfigFile();
+  const merged = mergeRunnerOverrides(fileOverrides, overrides);
+  return new EffectRunner(withRunnerConfig(merged));
 }
 var EffectRunner = class {
   constructor(config) {
@@ -902,7 +1026,7 @@ async function runSimpleEvalCommand(runner, datasetName, evaluatorPattern) {
     );
   }
   let spinnerTimer;
-  const done = new Promise((resolve3) => {
+  const done = new Promise((resolve4) => {
     const unsubscribe = runner.subscribeRunEvents((event) => {
       if (event.type === "TestCaseProgress") {
         completedCount = event.completedTestCases;
@@ -952,7 +1076,7 @@ async function runSimpleEvalCommand(runner, datasetName, evaluatorPattern) {
         runFinished = true;
         clearLine();
         unsubscribe();
-        resolve3(event);
+        resolve4(event);
       }
     });
   });