@m4trix/evals 0.1.0 → 0.3.0

package/dist/cli.js

@@ -5,8 +5,10 @@ import { useApp, useInput, Box, Text } from 'ink';
 import { jsx, jsxs, Fragment } from 'react/jsx-runtime';
 import { randomUUID } from 'crypto';
 import { Effect, PubSub, Queue, Fiber } from 'effect';
+import { existsSync } from 'fs';
+import { resolve, relative, join, dirname } from 'path';
+import * as jitiModule from 'jiti';
 import { mkdir, appendFile, readdir } from 'fs/promises';
-import { relative, join, dirname, resolve } from 'path';
 import { pathToFileURL } from 'url';
 
 var SEP = "  ";
@@ -1292,10 +1294,104 @@ var defaultRunnerConfig = {
   },
   artifactDirectory: ".eval-results"
 };
+function toRunnerConfigOverrides(config) {
+  if (!config) {
+    return void 0;
+  }
+  const rawDiscovery = config.discovery;
+  const discovery = {};
+  if (rawDiscovery?.rootDir !== void 0) {
+    discovery.rootDir = rawDiscovery.rootDir;
+  }
+  if (rawDiscovery?.datasetFilePatterns !== void 0) {
+    discovery.datasetSuffixes = rawDiscovery.datasetFilePatterns;
+  } else if (rawDiscovery?.datasetSuffixes !== void 0) {
+    discovery.datasetSuffixes = rawDiscovery.datasetSuffixes;
+  }
+  if (rawDiscovery?.evaluatorFilePatterns !== void 0) {
+    discovery.evaluatorSuffixes = rawDiscovery.evaluatorFilePatterns;
+  } else if (rawDiscovery?.evaluatorSuffixes !== void 0) {
+    discovery.evaluatorSuffixes = rawDiscovery.evaluatorSuffixes;
+  }
+  if (rawDiscovery?.testCaseFilePatterns !== void 0) {
+    discovery.testCaseSuffixes = rawDiscovery.testCaseFilePatterns;
+  } else if (rawDiscovery?.testCaseSuffixes !== void 0) {
+    discovery.testCaseSuffixes = rawDiscovery.testCaseSuffixes;
+  }
+  if (rawDiscovery?.excludeDirectories !== void 0) {
+    discovery.excludeDirectories = rawDiscovery.excludeDirectories;
+  }
+  const overrides = {};
+  if (config.artifactDirectory !== void 0) {
+    overrides.artifactDirectory = config.artifactDirectory;
+  }
+  if (Object.keys(discovery).length > 0) {
+    overrides.discovery = discovery;
+  }
+  return overrides;
+}
 function withRunnerConfig(overrides) {
-  {
+  if (!overrides) {
     return defaultRunnerConfig;
   }
+  const discovery = overrides.discovery ? {
+    ...defaultRunnerConfig.discovery,
+    ...overrides.discovery
+  } : defaultRunnerConfig.discovery;
+  return {
+    ...defaultRunnerConfig,
+    ...overrides,
+    discovery
+  };
+}
+var CONFIG_FILE_NAME = "m4trix-eval.config.ts";
+var cachedLoader;
+function getJitiLoader() {
+  if (cachedLoader) {
+    return cachedLoader;
+  }
+  const createJiti2 = jitiModule.createJiti ?? jitiModule.default;
+  if (typeof createJiti2 !== "function") {
+    throw new Error(
+      "Failed to initialize jiti for m4trix eval config loading."
+    );
+  }
+  cachedLoader = createJiti2(import.meta.url, {
+    interopDefault: true,
+    moduleCache: true
+  });
+  return cachedLoader;
+}
+function resolveConfigModuleExport(loadedModule) {
+  if (loadedModule && typeof loadedModule === "object" && "default" in loadedModule) {
+    return loadedModule.default;
+  }
+  return loadedModule;
+}
+function resolveConfigValue(value) {
+  if (value === void 0 || value === null) {
+    return void 0;
+  }
+  if (typeof value === "function") {
+    return value();
+  }
+  if (typeof value !== "object") {
+    throw new Error(
+      "Invalid m4trix eval config export. Expected an object or defineConfig(() => config)."
+    );
+  }
+  return value;
+}
+function loadRunnerConfigFile(cwd = process.cwd()) {
+  const configPath = resolve(cwd, CONFIG_FILE_NAME);
+  if (!existsSync(configPath)) {
+    return void 0;
+  }
+  const loader = getJitiLoader();
+  const loaded = loader(configPath);
+  const exportedValue = resolveConfigModuleExport(loaded);
+  const config = resolveConfigValue(exportedValue);
+  return toRunnerConfigOverrides(config);
 }
 var jitiLoader;
 function toId(prefix, filePath, name) {
@@ -1348,12 +1444,12 @@ function hasOneSuffix(filePath, suffixes) {
 async function loadModuleExports(filePath) {
   if (filePath.endsWith(".ts") || filePath.endsWith(".tsx")) {
     if (!jitiLoader) {
-      const jitiModule = await import('jiti');
-      const createJiti = jitiModule.createJiti ?? jitiModule.default;
-      if (!createJiti) {
+      const jitiModule2 = await import('jiti');
+      const createJiti2 = jitiModule2.createJiti ?? jitiModule2.default;
+      if (!createJiti2) {
         throw new Error("Failed to initialize jiti TypeScript loader");
       }
-      jitiLoader = createJiti(import.meta.url, {
+      jitiLoader = createJiti2(import.meta.url, {
         interopDefault: true,
         moduleCache: true
       });
@@ -1758,8 +1854,18 @@ function createNameMatcher(pattern) {
   }
   return (value) => value.toLowerCase() === normalizedPattern.toLowerCase();
 }
+function mergeRunnerOverrides(base, next) {
+  if (!base) {
+    return next;
+  }
+  {
+    return base;
+  }
+}
 function createRunner(overrides) {
-  return new EffectRunner(withRunnerConfig());
+  const fileOverrides = loadRunnerConfigFile();
+  const merged = mergeRunnerOverrides(fileOverrides, overrides);
+  return new EffectRunner(withRunnerConfig(merged));
 }
 var EffectRunner = class {
   constructor(config) {