@wix/evalforge-evaluator 0.154.0 → 0.156.0

package/build/index.mjs

@@ -659,6 +659,14 @@ async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchGitHubFolder2)
   }
 }
 
+// src/run-scenario/agents/timeout.ts
+var UNLIMITED_RUN_TIMEOUT_MS = 60 * 6e4;
+function resolveTimeoutMs(maxTurns, maxDurationMs) {
+  if (maxDurationMs !== void 0) return maxDurationMs;
+  if (maxTurns === void 0) return UNLIMITED_RUN_TIMEOUT_MS;
+  return Math.max(3e5, maxTurns * 6e4);
+}
+
 // src/run-scenario/agents/claude-code/execute.ts
 import { randomUUID } from "crypto";
 
@@ -1291,7 +1299,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
-  const SDK_TIMEOUT_MS = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
+  const SDK_TIMEOUT_MS = resolveTimeoutMs(maxTurns, options.maxDurationMs);
   let timeoutHandle;
   let timedOut = false;
   const HEARTBEAT_INTERVAL_MS = 1e4;
@@ -3104,7 +3112,7 @@ async function executeWithOpenCode(skills, scenario, options) {
     model: options.model
   });
   const maxTurns = options.maxTurns || void 0;
-  const sdkTimeoutMs = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
+  const sdkTimeoutMs = resolveTimeoutMs(maxTurns, options.maxDurationMs);
   const { env, providerID, modelID } = await buildOpenCodeEnv({
     model: options.model,
     temperature: options.temperature,
@@ -3726,7 +3734,10 @@ async function executeWithAiSdk(context) {
     emitStartEvent(traceContext, startTime);
   }
   const effectiveMaxTurns = cfg.maxTurns === 0 ? void 0 : cfg.maxTurns ?? DEFAULT_MAX_TOOL_STEPS;
-  const SDK_TIMEOUT_MS = cfg.maxDurationMs ?? Math.max(3e5, (effectiveMaxTurns ?? 25) * 6e4);
+  const SDK_TIMEOUT_MS = resolveTimeoutMs(
+    effectiveMaxTurns,
+    cfg.maxDurationMs
+  );
   const abortController = new AbortController();
   const timeoutHandle = setTimeout(() => {
     abortController.abort(
@@ -4995,6 +5006,7 @@ async function runEvaluationLoop(scenarioItems, evalData, callbacks) {
   const runsPerScenario = evalData.evalRun.runsPerScenario ?? 1;
   let completedExecutions = 0;
   let erroredExecutions = 0;
+  let firstErrorMessage;
   const totalExecutions = scenarioItems.length * runsPerScenario;
   const targetId = evalData.evalRun.presetId ?? evalData.agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName ?? evalData.agent?.name ?? "";
@@ -5032,6 +5044,9 @@ async function runEvaluationLoop(scenarioItems, evalData, callbacks) {
         };
         await callbacks.addResult(errorResult);
         erroredExecutions++;
+        if (firstErrorMessage === void 0) {
+          firstErrorMessage = errorResult.outputText;
+        }
       }
       if (scenarioResult !== null) {
         await callbacks.addResult({ ...scenarioResult, iterationIndex });
@@ -5039,7 +5054,12 @@ async function runEvaluationLoop(scenarioItems, evalData, callbacks) {
       completedExecutions++;
     }
   }
-  return { completedExecutions, totalExecutions, erroredExecutions };
+  return {
+    completedExecutions,
+    totalExecutions,
+    erroredExecutions,
+    firstErrorMessage
+  };
 }
 
 // src/error-reporter.ts
@@ -5223,7 +5243,12 @@ async function runEvaluation(projectId2, evalRunId2) {
     );
   }
   const skillNames = evalData.skills.map((s) => s.name).join(", ");
-  const { completedExecutions, totalExecutions, erroredExecutions } = await runEvaluationLoop(scenarioItems, evalData, {
+  const {
+    completedExecutions,
+    totalExecutions,
+    erroredExecutions,
+    firstErrorMessage
+  } = await runEvaluationLoop(scenarioItems, evalData, {
     runScenario: (scenario, template, resolvedAssertions) => {
       state.currentPhase = ExecutionPhase.EXECUTE_SKILL;
       state.currentContext = {
@@ -5272,10 +5297,14 @@ async function runEvaluation(projectId2, evalRunId2) {
   };
   const allFailed = totalExecutions > 0 && erroredExecutions === totalExecutions;
   const finalStatus = allFailed ? EvalStatus2.FAILED : EvalStatus2.COMPLETED;
+  const jobErrorOnAllFailed = allFailed ? truncateForJobError(
+    firstErrorMessage ?? `All ${totalExecutions} executions errored without an error message`
+  ) : void 0;
   try {
     await api.updateEvalRun(projectId2, evalRunId2, {
       status: finalStatus,
-      completedAt: (/* @__PURE__ */ new Date()).toISOString()
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      ...allFailed ? { jobError: jobErrorOnAllFailed, jobStatus: "FAILED" } : {}
     });
   } catch (updateErr) {
     throw new Error(
@@ -5283,6 +5312,11 @@ async function runEvaluation(projectId2, evalRunId2) {
     );
   }
 }
+var JOB_ERROR_MAX_LENGTH = 1e3;
+function truncateForJobError(message) {
+  if (message.length <= JOB_ERROR_MAX_LENGTH) return message;
+  return `${message.slice(0, JOB_ERROR_MAX_LENGTH)}\u2026 [truncated]`;
+}
 var projectId = process.argv[2];
 var evalRunId = process.argv[3];
 console.error(