@m4trix/evals 0.2.0 → 0.4.0

package/dist/cli.js

@@ -1352,7 +1352,9 @@ function getJitiLoader() {
   }
   const createJiti2 = jitiModule.createJiti ?? jitiModule.default;
   if (typeof createJiti2 !== "function") {
-    throw new Error("Failed to initialize jiti for m4trix eval config loading.");
+    throw new Error(
+      "Failed to initialize jiti for m4trix eval config loading."
+    );
   }
   cachedLoader = createJiti2(import.meta.url, {
     interopDefault: true,
@@ -1375,7 +1377,7 @@ function resolveConfigValue(value) {
   }
   if (typeof value !== "object") {
     throw new Error(
-      "Invalid m4trix eval config export. Expected an object or defineConfigFunction(() => config)."
+      "Invalid m4trix eval config export. Expected an object or defineConfig(() => config)."
     );
   }
   return value;
@@ -1654,6 +1656,13 @@ function normalizeResult(result) {
   const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
   return { scores, metrics };
 }
+function readOutputDefinition(testCase) {
+  const candidate = testCase;
+  if (typeof candidate.getOutputDefinition !== "function") {
+    return void 0;
+  }
+  return candidate.getOutputDefinition();
+}
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -1682,6 +1691,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     const started = Date.now();
     const evaluatorScores = [];
     let testCaseError;
+    const outputDefinition = readOutputDefinition(testCaseItem.testCase);
     for (const { id: evaluatorId, evaluator } of task.evaluators) {
       const evaluateFn = evaluator.getEvaluateFn();
       if (!evaluateFn) {
@@ -1692,7 +1702,13 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
           () => Promise.resolve(evaluator.resolveContext())
         );
         const result = yield* Effect.promise(
-          () => Promise.resolve(evaluateFn(testCaseItem.testCase.getInput(), ctx))
+          () => Promise.resolve(
+            evaluateFn({
+              input: testCaseItem.testCase.getInput(),
+              ctx,
+              output: outputDefinition
+            })
+          )
         );
         const { scores, metrics } = normalizeResult(result);
         const passed = computeEvaluatorPassed(evaluator, result, scores);
@@ -1723,6 +1739,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
       passed: testCasePassed,
       durationMs: Date.now() - started,
       evaluatorScores,
+      outputDefinition,
       errorMessage: testCaseError
     };
     updateSnapshot(task.runId, (snapshot) => ({