@wix/evalforge-evaluator 0.155.0 → 0.157.0

package/build/index.mjs

@@ -659,6 +659,14 @@ async function writeSkillToFilesystem(cwd, skill, fetchFn = fetchGitHubFolder2)
   }
 }
 
+// src/run-scenario/agents/timeout.ts
+var UNLIMITED_RUN_TIMEOUT_MS = 60 * 6e4;
+function resolveTimeoutMs(maxTurns, maxDurationMs) {
+  if (maxDurationMs !== void 0) return maxDurationMs;
+  if (maxTurns === void 0) return UNLIMITED_RUN_TIMEOUT_MS;
+  return Math.max(3e5, maxTurns * 6e4);
+}
+
 // src/run-scenario/agents/claude-code/execute.ts
 import { randomUUID } from "crypto";
 
@@ -1291,7 +1299,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
-  const SDK_TIMEOUT_MS = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
+  const SDK_TIMEOUT_MS = resolveTimeoutMs(maxTurns, options.maxDurationMs);
   let timeoutHandle;
   let timedOut = false;
   const HEARTBEAT_INTERVAL_MS = 1e4;
@@ -3104,7 +3112,7 @@ async function executeWithOpenCode(skills, scenario, options) {
     model: options.model
   });
   const maxTurns = options.maxTurns || void 0;
-  const sdkTimeoutMs = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
+  const sdkTimeoutMs = resolveTimeoutMs(maxTurns, options.maxDurationMs);
   const { env, providerID, modelID } = await buildOpenCodeEnv({
     model: options.model,
     temperature: options.temperature,
@@ -3396,8 +3404,8 @@ defaultRegistry.register(openCodeAdapter);
 // src/run-scenario/agents/simple-agent/execute.ts
 import {
   APICallError,
-  generateText,
-  stepCountIs
+  stepCountIs,
+  streamText
 } from "ai";
 import { createAnthropic } from "@ai-sdk/anthropic";
 import { createGoogleGenerativeAI } from "@ai-sdk/google";
@@ -3726,12 +3734,15 @@ async function executeWithAiSdk(context) {
     emitStartEvent(traceContext, startTime);
   }
   const effectiveMaxTurns = cfg.maxTurns === 0 ? void 0 : cfg.maxTurns ?? DEFAULT_MAX_TOOL_STEPS;
-  const SDK_TIMEOUT_MS = cfg.maxDurationMs ?? Math.max(3e5, (effectiveMaxTurns ?? 25) * 6e4);
+  const SDK_TIMEOUT_MS = resolveTimeoutMs(
+    effectiveMaxTurns,
+    cfg.maxDurationMs
+  );
   const abortController = new AbortController();
   const timeoutHandle = setTimeout(() => {
     abortController.abort(
       new Error(
-        `Simple Agent generateText timed out after ${SDK_TIMEOUT_MS}ms (model=${modelId}, scenario=${scenario.name})`
+        `Simple Agent streamText timed out after ${SDK_TIMEOUT_MS}ms (model=${modelId}, scenario=${scenario.name})`
       )
     );
   }, SDK_TIMEOUT_MS);
@@ -3777,7 +3788,7 @@ async function executeWithAiSdk(context) {
     const stepTimestamps = [];
     const { triggerPromptImages } = context;
     const hasImages = triggerPromptImages && triggerPromptImages.length > 0;
-    const result = await generateText({
+    const result = streamText({
       ...topLevelExtras,
       model,
       abortSignal: abortController.signal,
@@ -3835,14 +3846,19 @@ async function executeWithAiSdk(context) {
         }
       }
     });
+    const [resolvedText, resolvedSteps, resolvedUsage] = await Promise.all([
+      result.text,
+      result.steps,
+      result.usage
+    ]);
     const durationMs = Date.now() - startTime;
     const usage = {
-      inputTokens: result.usage.inputTokens ?? 0,
-      outputTokens: result.usage.outputTokens ?? 0,
-      totalTokens: result.usage.totalTokens ?? 0
+      inputTokens: resolvedUsage.inputTokens ?? 0,
+      outputTokens: resolvedUsage.outputTokens ?? 0,
+      totalTokens: resolvedUsage.totalTokens ?? 0
     };
     const llmTrace = buildLLMTrace2(
-      result.steps,
+      resolvedSteps,
       durationMs,
       usage,
       modelId,
@@ -3855,12 +3871,12 @@ async function executeWithAiSdk(context) {
     }
     const conversation = buildConversation3(
       scenario.triggerPrompt,
-      result.steps,
+      resolvedSteps,
       startTime,
       stepTimestamps
     );
     return {
-      outputText: result.text,
+      outputText: resolvedText,
       durationMs,
       usage,
       llmTrace,