@m4trix/evals 0.3.0 → 0.5.0

package/dist/cli.js

@@ -1656,6 +1656,13 @@ function normalizeResult(result) {
   const metrics = Array.isArray(obj.metrics) ? obj.metrics : void 0;
   return { scores, metrics };
 }
+function readOutput(testCase) {
+  const candidate = testCase;
+  if (typeof candidate.getOutput !== "function") {
+    return void 0;
+  }
+  return candidate.getOutput();
+}
 function nowIsoForFile() {
   return (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-");
 }
@@ -1684,6 +1691,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
     const started = Date.now();
     const evaluatorScores = [];
     let testCaseError;
+    const output = readOutput(testCaseItem.testCase);
     for (const { id: evaluatorId, evaluator } of task.evaluators) {
       const evaluateFn = evaluator.getEvaluateFn();
       if (!evaluateFn) {
@@ -1694,7 +1702,13 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
           () => Promise.resolve(evaluator.resolveContext())
         );
         const result = yield* Effect.promise(
-          () => Promise.resolve(evaluateFn(testCaseItem.testCase.getInput(), ctx))
+          () => Promise.resolve(
+            evaluateFn({
+              input: testCaseItem.testCase.getInput(),
+              ctx,
+              output
+            })
+          )
         );
         const { scores, metrics } = normalizeResult(result);
         const passed = computeEvaluatorPassed(evaluator, result, scores);
@@ -1725,6 +1739,7 @@ var executeRunTask = (task, publishEvent, persistenceQueue, updateSnapshot) => E
       passed: testCasePassed,
       durationMs: Date.now() - started,
       evaluatorScores,
+      output,
       errorMessage: testCaseError
     };
     updateSnapshot(task.runId, (snapshot) => ({