@m4trix/evals 0.4.0 → 0.7.0

package/dist/cli.js

@@ -1656,12 +1656,12 @@ function normalizeResult(result) {
   const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
   return { scores, metrics };
 }
-function readOutputDefinition(testCase) {
+function readOutput(testCase) {
   const candidate = testCase;
-  if (typeof candidate.getOutputDefinition !== "function") {
+  if (typeof candidate.getOutput !== "function") {
     return void 0;
   }
-  return candidate.getOutputDefinition();
+  return candidate.getOutput();
 }
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
@@ -1691,7 +1691,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     const started = Date.now();
     const evaluatorScores = [];
     let testCaseError;
-    const outputDefinition = readOutputDefinition(testCaseItem.testCase);
+    const output = readOutput(testCaseItem.testCase);
     for (const { id: evaluatorId, evaluator } of task.evaluators) {
       const evaluateFn = evaluator.getEvaluateFn();
       if (!evaluateFn) {
@@ -1706,7 +1706,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
             evaluateFn({
               input: testCaseItem.testCase.getInput(),
               ctx,
-              output: outputDefinition
+              output
             })
           )
         );
@@ -1739,7 +1739,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
       passed: testCasePassed,
       durationMs: Date.now() - started,
       evaluatorScores,
-      outputDefinition,
+      output,
       errorMessage: testCaseError
     };
     updateSnapshot(task.runId, (snapshot) => ({
@@ -1873,9 +1873,18 @@ function mergeRunnerOverrides(base, next) {
   if (!base) {
     return next;
   }
-  {
+  if (!next) {
     return base;
   }
+  const discovery = base.discovery || next.discovery ? {
+    ...base.discovery ?? {},
+    ...next.discovery ?? {}
+  } : void 0;
+  return {
+    ...base,
+    ...next,
+    discovery
+  };
 }
 function createRunner(overrides) {
   const fileOverrides = loadRunnerConfigFile();