@axlsdk/eval 0.16.1 → 0.17.0

package/dist/cli.cjs

@@ -24,7 +24,7 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 
 // src/cli.ts
-var import_node_fs2 = require("fs");
+var import_node_fs = require("fs");
 var import_promises = require("fs/promises");
 var path = __toESM(require("path"), 1);
 
@@ -114,13 +114,16 @@ function resolveThreshold(scorerName, options, metadata) {
   return LEGACY_THRESHOLD;
 }
 function evalCompare(baseline, candidate, options) {
-  const baselineRuns = Array.isArray(baseline) ? baseline : [baseline];
-  const candidateRuns = Array.isArray(candidate) ? candidate : [candidate];
-  if (baselineRuns.length === 0 || candidateRuns.length === 0) {
+  const baselineRunsRaw = Array.isArray(baseline) ? baseline : [baseline];
+  const candidateRunsRaw = Array.isArray(candidate) ? candidate : [candidate];
+  if (baselineRunsRaw.length === 0 || candidateRunsRaw.length === 0) {
     throw new Error("Cannot compare empty eval result arrays");
   }
-  const baselineRef = baselineRuns[0];
-  const candidateRef = candidateRuns[0];
+  const runCount = Math.min(baselineRunsRaw.length, candidateRunsRaw.length);
+  const baselineRuns = baselineRunsRaw.slice(0, runCount);
+  const candidateRuns = candidateRunsRaw.slice(0, runCount);
+  const baselineRef = baselineRunsRaw[0];
+  const candidateRef = candidateRunsRaw[0];
   if (baselineRef.dataset !== candidateRef.dataset) {
     throw new Error(
       `Cannot compare evals from different datasets: "${baselineRef.dataset}" vs "${candidateRef.dataset}"`
@@ -135,7 +138,6 @@ function evalCompare(baseline, candidate, options) {
   for (const name of baselineScorerNames) {
     pairedDiffs[name] = [];
   }
-  const runCount = Math.min(baselineRuns.length, candidateRuns.length);
   for (let r = 0; r < runCount; r++) {
     const bRun = baselineRuns[r];
     const cRun = candidateRuns[r];
@@ -272,9 +274,21 @@ function evalCompare(baseline, candidate, options) {
     parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);
   }
   const summaryStr = `candidate ${parts.length > 0 ? parts.join(", ") : "no meaningful changes"} with ${regressions.length} regressions and ${improvements.length} improvements`;
+  const baselinePartial = detectPartial(baselineRunsRaw);
+  const candidatePartial = detectPartial(candidateRunsRaw);
   return {
-    baseline: { id: baselineRef.id, metadata: baselineRef.metadata },
-    candidate: { id: candidateRef.id, metadata: candidateRef.metadata },
+    baseline: {
+      id: baselineRef.id,
+      metadata: baselineRef.metadata,
+      runCount,
+      ...baselinePartial ? { partial: baselinePartial } : {}
+    },
+    candidate: {
+      id: candidateRef.id,
+      metadata: candidateRef.metadata,
+      runCount,
+      ...candidatePartial ? { partial: candidatePartial } : {}
+    },
     scorers,
     timing,
     cost,
@@ -283,6 +297,20 @@ function evalCompare(baseline, candidate, options) {
     summary: summaryStr
   };
 }
+function detectPartial(runs) {
+  if (runs.length === 0) return void 0;
+  let attempted;
+  for (const r of runs) {
+    const v = r.metadata?.batchAttempted;
+    if (typeof v === "number" && Number.isFinite(v)) {
+      attempted = v;
+      break;
+    }
+  }
+  if (attempted === void 0) return void 0;
+  if (runs.length >= attempted) return void 0;
+  return { completed: runs.length, attempted };
+}
 
 // src/scorer.ts
 function normalizeScorerResult(result) {
@@ -786,66 +814,43 @@ function aggregateRuns(runs) {
 }
 
 // src/cli-utils.ts
-var import_node_path = require("path");
-var import_node_fs = require("fs");
-var import_node_url = require("url");
-var CONFIG_CANDIDATES = [
-  "axl.config.mts",
-  "axl.config.ts",
-  "axl.config.mjs",
-  "axl.config.js"
-];
-function findConfig(cwd) {
-  for (const name of CONFIG_CANDIDATES) {
-    const p = (0, import_node_path.resolve)(cwd, name);
-    if ((0, import_node_fs.existsSync)(p)) return p;
+var import_axl = require("@axlsdk/axl");
+
+// src/cli-validate.ts
+function validateEvalConfig(cfg) {
+  if (!cfg || typeof cfg !== "object") {
+    return `does not export a valid eval config \u2014 got ${cfg === null ? "null" : typeof cfg}.`;
+  }
+  const c = cfg;
+  const missing = [];
+  if (!c.workflow) missing.push("workflow");
+  if (!c.dataset) missing.push("dataset");
+  if (!c.scorers) missing.push("scorers");
+  if (missing.length > 0) {
+    const keys = Object.keys(c).slice(0, 10);
+    const got = keys.length > 0 ? ` Got: { ${keys.join(", ")} }.` : "";
+    return `does not export a valid eval config (missing ${missing.join(", ")}).${got}`;
+  }
+  if (typeof c.workflow !== "string") {
+    return `exports a non-string workflow (got ${typeof c.workflow}) \u2014 workflow must be the registered workflow name.`;
+  }
+  if (typeof c.dataset !== "object" || Array.isArray(c.dataset)) {
+    return `exports a non-object dataset (got ${Array.isArray(c.dataset) ? "array" : typeof c.dataset}) \u2014 use the dataset() factory.`;
+  }
+  if (typeof c.dataset.getItems !== "function") {
+    return `exports a dataset without a getItems() method \u2014 use the dataset() factory.`;
+  }
+  if (!Array.isArray(c.scorers) || c.scorers.length === 0) {
+    return `exports an empty scorers array \u2014 at least one scorer is required.`;
+  }
+  const badScorerIdx = c.scorers.findIndex(
+    (s) => !s || typeof s.score !== "function"
+  );
+  if (badScorerIdx !== -1) {
+    return `exports a scorers array with a non-scorer entry at index ${badScorerIdx} \u2014 each scorer must have a score() method (use the scorer() factory).`;
   }
   return void 0;
 }
-function needsTsxLoader(filePath) {
-  return /\.[mc]?tsx?$/.test(filePath);
-}
-function resolveRuntime(mod) {
-  const def = mod.default;
-  return def?.default ?? def ?? mod.runtime;
-}
-var tsImportFn;
-async function importModule(filePath, parentURL) {
-  if (needsTsxLoader(filePath)) {
-    if (tsImportFn === void 0) {
-      try {
-        const mod = await import("tsx/esm/api");
-        tsImportFn = mod.tsImport ?? null;
-      } catch {
-        tsImportFn = null;
-        console.warn(
-          "[axl-eval] Warning: tsx is not installed. TypeScript files require tsx.\n  Install it with: npm install -D tsx"
-        );
-      }
-    }
-    if (tsImportFn) {
-      return await tsImportFn((0, import_node_url.pathToFileURL)(filePath).href, parentURL);
-    }
-  }
-  return await import((0, import_node_url.pathToFileURL)(filePath).href);
-}
-async function registerConditions(conditions) {
-  try {
-    const nodeModule = await import("module");
-    const hookCode = [
-      `const extra = ${JSON.stringify(conditions)};`,
-      `export async function resolve(specifier, context, nextResolve) {`,
-      `  return nextResolve(specifier, {`,
-      `    ...context,`,
-      `    conditions: [...new Set([...context.conditions, ...extra])],`,
-      `  });`,
-      `}`
-    ].join("\n");
-    nodeModule.register(`data:text/javascript,${encodeURIComponent(hookCode)}`);
-  } catch {
-    console.warn("[axl-eval] Warning: --conditions requires Node.js 20.6+");
-  }
-}
 
 // src/cli.ts
 var import_meta = {};
@@ -878,7 +883,7 @@ Usage:
   axl-eval compare <a> <b> --fail-on-regression  Exit 1 if regressions
 
 Config auto-detection (when --config is not specified):
-  ${CONFIG_CANDIDATES.join(" -> ")}
+  ${import_axl.CONFIG_CANDIDATES.join(" -> ")}
 
 When a config is found, the exported AxlRuntime is passed to executeWorkflow
 and cost is tracked automatically via runtime.trackCost().
@@ -1000,10 +1005,16 @@ async function runRescore(args) {
   const results = Array.isArray(raw) ? raw : [raw];
   const runtime = await getRuntime(configArg, conditions);
   try {
-    const mod = await importModule(path.resolve(evalFilePath), import_meta.url);
-    const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
-    if (!evalConfig.scorers || evalConfig.scorers.length === 0) {
-      console.error(`Error: ${evalFilePath} does not export scorers`);
+    const mod = await (0, import_axl.importModule)(path.resolve(evalFilePath), import_meta.url);
+    const evalConfig = (0, import_axl.pickDefault)(mod);
+    if (!evalConfig || typeof evalConfig !== "object") {
+      console.error(
+        `Error: ${evalFilePath} does not export an eval config object \u2014 got ${evalConfig === null ? "null" : typeof evalConfig}.`
+      );
+      process.exit(1);
+    }
+    if (!Array.isArray(evalConfig.scorers) || evalConfig.scorers.length === 0) {
+      console.error(`Error: ${evalFilePath} does not export a non-empty scorers array.`);
       process.exit(1);
     }
     const rescored = [];
@@ -1026,11 +1037,12 @@ async function runRescore(args) {
   }
 }
 function collectEvalFiles(p) {
+  if (p.includes("*")) return (0, import_axl.expandGlob)(p, process.cwd());
   const resolved = path.resolve(p);
   try {
-    const stat = (0, import_node_fs2.statSync)(resolved);
+    const stat = (0, import_node_fs.statSync)(resolved);
     if (stat.isDirectory()) {
-      const entries = (0, import_node_fs2.readdirSync)(resolved);
+      const entries = (0, import_node_fs.readdirSync)(resolved);
       return entries.filter((e) => /\.eval\.[mc]?[jt]sx?$/.test(e)).map((e) => path.join(resolved, e));
     }
     return [resolved];
@@ -1129,8 +1141,8 @@ function formatMultiRunTable(summary) {
 }
 async function resolveRuntimeFromConfig(configPath) {
   try {
-    const mod = await importModule(configPath, import_meta.url);
-    const runtime = resolveRuntime(mod);
+    const mod = await (0, import_axl.importModule)(configPath, import_meta.url);
+    const runtime = (0, import_axl.resolveRuntime)(mod);
     if (!runtime || typeof runtime.execute !== "function") {
       console.error(`Config must export a default AxlRuntime instance.`);
       if (runtime) {
@@ -1167,11 +1179,11 @@ async function resolveRuntimeFromConfig(configPath) {
 }
 async function getRuntime(configArg, conditions) {
   if (conditions && conditions.length > 0) {
-    await registerConditions(conditions);
+    await (0, import_axl.registerConditions)(conditions);
   }
   if (configArg) {
     const configPath = path.resolve(process.cwd(), configArg);
-    const stat = (0, import_node_fs2.statSync)(configPath, { throwIfNoEntry: false });
+    const stat = (0, import_node_fs.statSync)(configPath, { throwIfNoEntry: false });
     if (!stat?.isFile()) {
       console.error(`Config file not found: ${configPath}`);
       process.exit(1);
@@ -1179,7 +1191,7 @@ async function getRuntime(configArg, conditions) {
     console.error(`[axl-eval] Loading config from ${configPath}`);
     return resolveRuntimeFromConfig(configPath);
   }
-  const found = findConfig(process.cwd());
+  const found = (0, import_axl.findConfig)(process.cwd());
   if (found) {
     console.error(`[axl-eval] Auto-detected config: ${found}`);
     return resolveRuntimeFromConfig(found);
@@ -1236,26 +1248,38 @@ async function runEvalCommand(args) {
   }
   const runtime = await getRuntime(configArg, conditions);
   const results = [];
+  let failedFiles = 0;
   try {
     for (const filePath of evalFiles) {
       try {
-        const mod = await importModule(path.resolve(filePath), import_meta.url);
-        const evalConfig = mod.default?.default ?? mod.default ?? mod.config ?? mod;
-        if (!evalConfig.workflow || !evalConfig.dataset || !evalConfig.scorers) {
+        const mod = await (0, import_axl.importModule)(path.resolve(filePath), import_meta.url);
+        const evalConfig = (0, import_axl.pickDefault)(mod);
+        const validationError = validateEvalConfig(evalConfig);
+        if (validationError) {
+          console.error(`Error: ${filePath} ${validationError}`);
+          failedFiles++;
+          continue;
+        }
+        const exported = (0, import_axl.pickExport)(mod, "executeWorkflow");
+        let customExecute;
+        if (typeof exported === "function") {
+          customExecute = exported;
+        } else if (exported !== void 0) {
           console.error(
-            `Error: ${filePath} does not export a valid eval config (missing workflow, dataset, or scorers)`
+            `[axl-eval] Error: ${filePath} exports executeWorkflow but it is ${typeof exported}, not a function.`
           );
+          failedFiles++;
           continue;
         }
         let executeWorkflow;
-        if (mod.executeWorkflow) {
+        if (customExecute) {
           executeWorkflow = async (input, rt) => {
             const {
               result,
               cost: trackedCost,
               metadata
             } = await runtime.trackExecution(async () => {
-              return mod.executeWorkflow(input, rt);
+              return customExecute(input, rt);
             });
             return {
               output: result.output,
@@ -1271,26 +1295,69 @@ async function runEvalCommand(args) {
             return { output: result, cost, metadata };
           };
         } else {
-          console.warn(
-            `[axl-eval] Warning: ${filePath} does not export executeWorkflow \u2014 using input passthrough`
+          const exportShape = (() => {
+            const top = Object.keys(mod).filter((k) => k !== "default");
+            const fromDefault = mod.default && typeof mod.default === "object" ? Object.keys(mod.default) : [];
+            const all = [.../* @__PURE__ */ new Set([...top, ...fromDefault])];
+            return all.length > 0 ? `Found exports: [${all.join(", ")}].` : "";
+          })();
+          console.error(
+            `[axl-eval] Error: ${filePath} does not export an executeWorkflow function and no workflow named "${evalConfig.workflow}" is registered on the runtime.
+` + (exportShape ? `  ${exportShape}
+` : "") + `  Add either:
+    export async function executeWorkflow(input, runtime) { ... }
+  or register the workflow on your runtime via runtime.registerWorkflow(workflow).
+  Tip: if your eval previously worked, your package.json may be missing
+    "type": "module" \u2014 tsx loads .ts as CJS, putting named exports on
+    mod.default.<name>. The CLI handles that interop, but if you renamed
+    or removed the export the file path is your fix.`
           );
-          executeWorkflow = async (input) => ({ output: input });
+          failedFiles++;
+          continue;
         }
         const runOptions = captureTraces ? { captureTraces: true } : void 0;
         if (runs > 1) {
           const { randomUUID: randomUUID4 } = await import("crypto");
           const runGroupId = randomUUID4();
           const runResults = [];
+          let runFailure;
           for (let r = 0; r < runs; r++) {
             console.error(`[axl-eval] Run ${r + 1}/${runs}...`);
-            const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
-            result.metadata.runGroupId = runGroupId;
-            result.metadata.runIndex = r;
-            runResults.push(result);
-            results.push(result);
+            try {
+              const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
+              result.metadata.runGroupId = runGroupId;
+              result.metadata.runIndex = r;
+              runResults.push(result);
+            } catch (err) {
+              runFailure = err instanceof Error ? err : new Error(String(err));
+              console.error(`[axl-eval] Run ${r + 1}/${runs} failed: ${runFailure.message}`);
+              break;
+            }
+          }
+          if (runResults.length === 0) {
+            console.error(
+              `[axl-eval] Aborting ${filePath}: no runs completed (${runFailure?.message ?? "unknown failure"}).`
+            );
+            failedFiles++;
+            continue;
+          }
+          const partial = runResults.length < runs;
+          if (partial) {
+            const failureMsg = runFailure ? runFailure.message || String(runFailure) || void 0 : void 0;
+            for (const r of runResults) {
+              r.metadata.fromPartialBatch = true;
+              r.metadata.batchCompleted = runResults.length;
+              r.metadata.batchAttempted = runs;
+              if (failureMsg) r.metadata.batchFailure = failureMsg;
+            }
+            console.error(
+              `[axl-eval] PARTIAL: ${runResults.length} of ${runs} runs completed for ${filePath}.`
+            );
+            failedFiles++;
           }
           const summary = aggregateRuns(runResults);
           console.log("\n" + formatMultiRunTable(summary) + "\n");
+          for (const r of runResults) results.push(r);
         } else {
           const result = await runEval(evalConfig, executeWorkflow, runtime, runOptions);
           results.push(result);
@@ -1300,6 +1367,7 @@ async function runEvalCommand(args) {
         console.error(
           `Error running eval ${filePath}: ${err instanceof Error ? err.message : String(err)}`
         );
+        failedFiles++;
       }
     }
     if (outputPath && results.length > 0) {
@@ -1313,7 +1381,7 @@ async function runEvalCommand(args) {
     await runtime.shutdown().catch(() => {
     });
   }
-  if (results.length === 0) {
+  if (failedFiles > 0 || results.length === 0) {
     process.exit(1);
   }
 }