@wix/evalforge-evaluator 0.132.0 → 0.133.0

package/build/index.mjs

@@ -4686,6 +4686,57 @@ async function runScenario(config, evalRunId2, scenario, evalData, template, res
   };
 }
 
+// src/evaluation-loop.ts
+import { randomUUID as randomUUID5 } from "crypto";
+async function runEvaluationLoop(scenarioItems, evalData, callbacks) {
+  const runsPerScenario = evalData.evalRun.runsPerScenario ?? 1;
+  let completedExecutions = 0;
+  const totalExecutions = scenarioItems.length * runsPerScenario;
+  const targetId = evalData.evalRun.presetId ?? evalData.agent?.id ?? evalData.evalRun.id;
+  const targetName = evalData.presetName ?? evalData.agent?.name ?? "";
+  for (const { scenario, template, resolvedAssertions } of scenarioItems) {
+    for (let iterationIndex = 0; iterationIndex < runsPerScenario; iterationIndex++) {
+      const iterLabel = runsPerScenario > 1 ? ` [run ${iterationIndex + 1}/${runsPerScenario}]` : "";
+      callbacks.onProgress(completedExecutions + 1, totalExecutions, iterLabel);
+      let scenarioResult = null;
+      try {
+        scenarioResult = await callbacks.runScenario(
+          scenario,
+          template,
+          resolvedAssertions
+        );
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        console.error(
+          `[Evaluator] Scenario iteration failed, recording as error result: "${scenario.name}"${iterLabel} \u2014 ${errorMsg}`
+        );
+        const errorResult = {
+          id: randomUUID5(),
+          targetId,
+          targetName,
+          scenarioId: scenario.id,
+          scenarioName: scenario.name,
+          assertionResults: [],
+          passed: 0,
+          failed: 0,
+          passRate: 0,
+          duration: 0,
+          outputText: `Execution error: ${errorMsg}`,
+          startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+          iterationIndex
+        };
+        await callbacks.addResult(errorResult);
+      }
+      if (scenarioResult !== null) {
+        await callbacks.addResult({ ...scenarioResult, iterationIndex });
+      }
+      completedExecutions++;
+    }
+  }
+  return { completedExecutions, totalExecutions };
+}
+
 // src/error-reporter.ts
 import { EvalStatus } from "@wix/evalforge-types";
 function formatError(error, phase, context) {
@@ -4866,69 +4917,57 @@ async function runEvaluation(projectId2, evalRunId2) {
       `[${ExecutionPhase.VALIDATION}] Eval run has no agent: set agentId for evaluation runs. (agentId: ${evalData.evalRun.agentId || "not set"})`
     );
   }
-  let completedScenarios = 0;
-  const totalScenarios = scenarioItems.length;
-  for (const { scenario, template, resolvedAssertions } of scenarioItems) {
-    state.currentPhase = ExecutionPhase.EXECUTE_SKILL;
-    state.currentContext = {
-      projectId: projectId2,
-      evalRunId: evalRunId2,
-      scenarioId: scenario.id,
-      scenarioName: scenario.name,
-      presetId: evalData.evalRun.presetId,
-      presetName: evalData.presetName,
-      agentId: agent?.id,
-      agentName: agent?.name,
-      progress: `${completedScenarios + 1}/${totalScenarios}`
-    };
-    const skillNames = evalData.skills.map((s) => s.name).join(", ");
-    console.log(
-      "[Evaluator] Running scenario with preset:",
-      evalData.presetName,
-      skillNames ? `(${skillNames})` : "",
-      agent ? `with agent: ${agent.name}` : "",
-      `(${completedScenarios + 1}/${totalScenarios})`
-    );
-    try {
-      const result = await runScenario(
-        config,
-        evalRunId2,
-        scenario,
-        evalData,
-        template,
-        resolvedAssertions
-      );
-      console.log("[Evaluator] Scenario completed, adding result");
-      state.currentPhase = ExecutionPhase.ADD_RESULT;
-      state.currentContext = {
-        ...state.currentContext,
-        resultId: result.id
-      };
-      await api.addResult(projectId2, evalRunId2, result);
-      completedScenarios++;
-    } catch (err) {
-      const errorMsg = err instanceof Error ? err.message : String(err);
-      const errorStack = err instanceof Error ? err.stack : void 0;
-      console.error(
-        "[Evaluator] Failed to run scenario with preset:",
-        evalData.presetName,
-        "Error:",
-        errorMsg
-      );
-      if (errorStack) {
-        console.error("[Evaluator] Stack trace:", errorStack);
+  const skillNames = evalData.skills.map((s) => s.name).join(", ");
+  const { completedExecutions, totalExecutions } = await runEvaluationLoop(
+    scenarioItems,
+    evalData,
+    {
+      runScenario: (scenario, template, resolvedAssertions) => {
+        state.currentPhase = ExecutionPhase.EXECUTE_SKILL;
+        state.currentContext = {
+          projectId: projectId2,
+          evalRunId: evalRunId2,
+          scenarioId: scenario.id,
+          scenarioName: scenario.name,
+          presetId: evalData.evalRun.presetId,
+          presetName: evalData.presetName,
+          agentId: agent?.id,
+          agentName: agent?.name
+        };
+        return runScenario(
+          config,
+          evalRunId2,
+          scenario,
+          evalData,
+          template,
+          resolvedAssertions
+        );
+      },
+      addResult: async (result) => {
+        state.currentPhase = ExecutionPhase.ADD_RESULT;
+        state.currentContext = {
+          ...state.currentContext,
+          resultId: result.id
+        };
+        await api.addResult(projectId2, evalRunId2, result);
+      },
+      onProgress: (completed, total, iterLabel) => {
+        console.log(
+          "[Evaluator] Running scenario with preset:",
+          evalData.presetName,
+          skillNames ? `(${skillNames})` : "",
+          agent ? `with agent: ${agent.name}` : "",
+          `(${completed}/${total})${iterLabel}`
+        );
       }
-      throw new Error(
-        `[${state.currentPhase}] Failed to execute preset "${evalData.presetName}" on scenario "${scenario.name}": ${errorMsg}`
-      );
     }
-  }
+  );
   state.currentPhase = ExecutionPhase.UPDATE_STATUS;
   state.currentContext = {
     projectId: projectId2,
     evalRunId: evalRunId2,
-    completedScenarios,
-    totalScenarios
+    completedExecutions,
+    totalExecutions
   };
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {