@m4trix/evals 0.1.0 → 0.3.0

package/dist/index.cjs

@@ -1,12 +1,34 @@
 'use strict';
 
-var crypto = require('crypto');
 var effect = require('effect');
-var promises = require('fs/promises');
+var crypto = require('crypto');
+var fs = require('fs');
 var path = require('path');
+var jitiModule = require('jiti');
+var promises = require('fs/promises');
 var url = require('url');
 
 var _documentCurrentScript = typeof document !== 'undefined' ? document.currentScript : null;
+function _interopNamespace(e) {
+  if (e && e.__esModule) return e;
+  var n = Object.create(null);
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k);
+        Object.defineProperty(n, k, d.get ? d : {
+          enumerable: true,
+          get: function () { return e[k]; }
+        });
+      }
+    });
+  }
+  n.default = e;
+  return Object.freeze(n);
+}
+
+var jitiModule__namespace = /*#__PURE__*/_interopNamespace(jitiModule);
+
 // src/cli/data.mock.json
 var data_mock_default = {
   datasets: [
@@ -544,6 +566,9 @@ var binaryScore = Score.of({
 });
 
 // src/runner/config.ts
+function defineConfig(factory) {
+  return factory;
+}
 var defaultRunnerConfig = {
   discovery: {
     rootDir: process.cwd(),
@@ -564,6 +589,42 @@ var defaultRunnerConfig = {
   },
   artifactDirectory: ".eval-results"
 };
+function toRunnerConfigOverrides(config) {
+  if (!config) {
+    return void 0;
+  }
+  const rawDiscovery = config.discovery;
+  const discovery = {};
+  if (rawDiscovery?.rootDir !== void 0) {
+    discovery.rootDir = rawDiscovery.rootDir;
+  }
+  if (rawDiscovery?.datasetFilePatterns !== void 0) {
+    discovery.datasetSuffixes = rawDiscovery.datasetFilePatterns;
+  } else if (rawDiscovery?.datasetSuffixes !== void 0) {
+    discovery.datasetSuffixes = rawDiscovery.datasetSuffixes;
+  }
+  if (rawDiscovery?.evaluatorFilePatterns !== void 0) {
+    discovery.evaluatorSuffixes = rawDiscovery.evaluatorFilePatterns;
+  } else if (rawDiscovery?.evaluatorSuffixes !== void 0) {
+    discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;
+  }
+  if (rawDiscovery?.testCaseFilePatterns !== void 0) {
+    discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;
+  } else if (rawDiscovery?.testCaseSuffixes !== void 0) {
+    discovery.testCaseSuffixes = rawDiscovery.testCaseSuffixes;
+  }
+  if (rawDiscovery?.excludeDirectories !== void 0) {
+    discovery.excludeDirectories = rawDiscovery.excludeDirectories;
+  }
+  const overrides = {};
+  if (config.artifactDirectory !== void 0) {
+    overrides.artifactDirectory = config.artifactDirectory;
+  }
+  if (Object.keys(discovery).length > 0) {
+    overrides.discovery = discovery;
+  }
+  return overrides;
+}
 function withRunnerConfig(overrides) {
   if (!overrides) {
     return defaultRunnerConfig;
@@ -578,6 +639,55 @@ function withRunnerConfig(overrides) {
     discovery
   };
 }
+var CONFIG_FILE_NAME = "m4trix-eval.config.ts";
+var cachedLoader;
+function getJitiLoader() {
+  if (cachedLoader) {
+    return cachedLoader;
+  }
+  const createJiti2 = jitiModule__namespace.createJiti ?? jitiModule__namespace.default;
+  if (typeof createJiti2 !== "function") {
+    throw new Error(
+      "Failed to initialize jiti for m4trix eval config loading."
+    );
+  }
+  cachedLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
+    interopDefault: true,
+    moduleCache: true
+  });
+  return cachedLoader;
+}
+function resolveConfigModuleExport(loadedModule) {
+  if (loadedModule && typeof loadedModule === "object" && "default" in loadedModule) {
+    return loadedModule.default;
+  }
+  return loadedModule;
+}
+function resolveConfigValue(value) {
+  if (value === void 0 || value === null) {
+    return void 0;
+  }
+  if (typeof value === "function") {
+    return value();
+  }
+  if (typeof value !== "object") {
+    throw new Error(
+      "Invalid m4trix eval config export. Expected an object or defineConfig(() => config)."
+    );
+  }
+  return value;
+}
+function loadRunnerConfigFile(cwd = process.cwd()) {
+  const configPath = path.resolve(cwd, CONFIG_FILE_NAME);
+  if (!fs.existsSync(configPath)) {
+    return void 0;
+  }
+  const loader = getJitiLoader();
+  const loaded = loader(configPath);
+  const exportedValue = resolveConfigModuleExport(loaded);
+  const config = resolveConfigValue(exportedValue);
+  return toRunnerConfigOverrides(config);
+}
 var jitiLoader;
 function toId(prefix, filePath, name) {
   const stable = name && name.trim().length > 0 ? name : filePath;
@@ -629,12 +739,12 @@ function hasOneSuffix(filePath, suffixes) {
 async function loadModuleExports(filePath) {
   if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) {
     if (!jitiLoader) {
-      const jitiModule = await import('jiti');
-      const createJiti = jitiModule.createJiti ?? jitiModule.default;
-      if (!createJiti) {
+      const jitiModule2 = await import('jiti');
+      const createJiti2 = jitiModule2.createJiti ?? jitiModule2.default;
+      if (!createJiti2) {
         throw new Error("Failed to initialize jiti TypeScript loader");
       }
-      jitiLoader = createJiti((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
+      jitiLoader = createJiti2((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('out.js', document.baseURI).href)), {
         interopDefault: true,
         moduleCache: true
       });
@@ -965,8 +1075,27 @@ function createNameMatcher(pattern) {
   }
   return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
 }
+function mergeRunnerOverrides(base, next) {
+  if (!base) {
+    return next;
+  }
+  if (!next) {
+    return base;
+  }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
+}
 function createRunner(overrides) {
-  return new EffectRunner(withRunnerConfig(overrides));
+  const fileOverrides = loadRunnerConfigFile();
+  const merged = mergeRunnerOverrides(fileOverrides, overrides);
+  return new EffectRunner(withRunnerConfig(merged));
 }
 var EffectRunner = class {
   constructor(config) {
@@ -1163,6 +1292,10 @@ var EffectRunner = class {
   }
 };
 
+Object.defineProperty(exports, "S", {
+  enumerable: true,
+  get: function () { return effect.Schema; }
+});
 exports.Dataset = Dataset;
 exports.Evaluator = Evaluator;
 exports.Metric = Metric;
@@ -1171,6 +1304,7 @@ exports.TestCase = TestCase;
 exports.binaryScore = binaryScore;
 exports.createRunner = createRunner;
 exports.defaultRunnerConfig = defaultRunnerConfig;
+exports.defineConfig = defineConfig;
 exports.getMetricById = getMetricById;
 exports.getScoreById = getScoreById;
 exports.latencyMetric = latencyMetric;