@wix/evalforge-evaluator 0.122.0 → 0.124.0

package/build/index.js

@@ -2606,8 +2606,8 @@ var import_promises9 = require("fs/promises");
 var import_path10 = require("path");
 var KILL_GRACE_PERIOD_MS = 5e3;
 var IDLE_TIMEOUT_MS = 12e4;
-var TOOL_RUNNING_IDLE_TIMEOUT_MS = 36e4;
 var IDLE_CHECK_INTERVAL_MS = 15e3;
+var MAX_IDLE_RETRIES = 3;
 function extractToolAction(toolName, args) {
   if (!toolName) return "Using tool...";
   if ((toolName === "Task" || toolName === "dispatch_agent") && args?.description) {
@@ -2733,246 +2733,126 @@ function killProcess(child, resolved) {
     }
   }, KILL_GRACE_PERIOD_MS);
 }
-async function executeWithOpenCode(skills, scenario, options) {
-  const skillNames = skills.map((s) => s.name).join(", ");
-  console.log("[executeWithOpenCode] Starting execution", {
-    skillCount: skills.length,
-    skillNames,
-    scenarioId: scenario.id,
-    scenarioName: scenario.name,
-    cwd: options.cwd,
-    aiGatewayUrl: options.aiGatewayUrl,
-    hasAiGatewayHeaders: !!options.aiGatewayHeaders,
-    model: options.model
-  });
-  const startTime = /* @__PURE__ */ new Date();
-  const maxTurns = options.maxTurns ?? 10;
-  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
-  const { env, providerID, modelID } = await buildOpenCodeEnv({
-    model: options.model,
-    temperature: options.temperature,
-    maxTurns,
-    aiGatewayUrl: options.aiGatewayUrl,
-    aiGatewayHeaders: options.aiGatewayHeaders,
-    mcps: options.mcps,
-    cwd: options.cwd
-  });
-  const traceContext = options.traceContext;
-  let traceStepNumber = 0;
-  let lastAction = "Starting...";
-  let lastToolName;
-  let lastFilePath;
-  let isToolRunning = false;
-  if (traceContext) {
-    emitTraceEvent(
-      {
-        evalRunId: traceContext.evalRunId,
-        scenarioId: traceContext.scenarioId,
-        scenarioName: traceContext.scenarioName,
-        targetId: traceContext.targetId,
-        targetName: traceContext.targetName,
-        stepNumber: 0,
-        type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
-        outputPreview: JSON.stringify({
-          event: "pre-cli-execution",
-          model: `${providerID}/${modelID}`,
-          maxTurns,
-          timestamp: (/* @__PURE__ */ new Date()).toISOString()
-        }),
-        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-        isComplete: false
-      },
-      traceContext.tracePushUrl,
-      traceContext.routeHeader,
-      traceContext.authToken
-    );
-  }
-  let systemPrompt;
-  if (options.systemPrompt === null || options.systemPrompt === "") {
-  } else if (options.systemPrompt != null) {
-    systemPrompt = options.systemPrompt;
-  } else {
-    systemPrompt = import_evalforge_types8.DEFAULT_EVALUATOR_SYSTEM_PROMPT;
-  }
-  if (systemPrompt) {
-    await writeSystemPromptRule(options.cwd, systemPrompt);
+function summarizeToolActions(events) {
+  const actions = [];
+  for (const { event: evt } of events) {
+    if (evt.type === "tool_use") {
+      const tu = evt;
+      const tool = tu.part.tool;
+      const input = tu.part.state.input;
+      const filePath = input?.file_path || input?.path || input?.target_file;
+      if (filePath) {
+        actions.push(`- ${tool}: ${String(filePath)}`);
+      } else if (input?.command) {
+        actions.push(`- ${tool}: ${String(input.command).slice(0, 80)}`);
+      } else {
+        actions.push(`- ${tool}`);
+      }
+    }
   }
-  const args = [
-    "run",
-    "--format",
-    "json",
-    "--thinking",
-    "--variant",
-    "high",
-    "--model",
-    `${providerID}/${modelID}`,
-    "--dir",
-    options.cwd,
-    // NOTE: Trigger prompt is passed as a positional CLI arg. On Linux a single
-    // arg is capped at 128 KB (MAX_ARG_STRLEN); on macOS the combined args+env
-    // share a ~1 MB limit. Prompts exceeding this would fail with E2BIG.
-    // In practice eval prompts are well under this limit.
-    scenario.triggerPrompt
-  ];
-  console.log("[executeWithOpenCode] Spawning: opencode", args.slice(0, 5));
-  return new Promise((resolve2, reject) => {
+  return actions.length > 0 ? actions.join("\n") : "(no tool actions recorded)";
+}
+function buildRecoveryPrompt(originalPrompt, events) {
+  const toolSummary = summarizeToolActions(events);
+  return `You are continuing a task that was interrupted due to a session error.
+
+ORIGINAL TASK:
+${originalPrompt}
+
+ACTIONS ALREADY COMPLETED IN THE PREVIOUS SESSION:
+${toolSummary}
+
+INSTRUCTIONS:
+1. Review the actions listed above that were already completed in the previous session
+2. Check the filesystem to verify what was already done
+3. Continue with any remaining work needed to fulfill the original task
+4. Do NOT redo work that is already done \u2014 only continue from where the previous session left off`;
+}
+function spawnOpenCodeProcess(opts) {
+  const {
+    args,
+    env,
+    cwd,
+    skillNames,
+    scenarioName,
+    sdkTimeoutMs,
+    traceContext,
+    initialStepNumber
+  } = opts;
+  return new Promise((resolve2) => {
     let resolved = false;
     let stderr = "";
     let lineBuffer = "";
     let lastOutputTime = Date.now();
-    const allEvents = [];
+    let traceStepNumber = initialStepNumber;
+    let lastAction = "Starting...";
+    let lastToolName;
+    let lastFilePath;
+    const events = [];
     const timers = {};
     const cleanup = () => {
       if (timers.timeout) clearTimeout(timers.timeout);
       if (timers.idleCheck) clearInterval(timers.idleCheck);
       if (timers.heartbeat) clearInterval(timers.heartbeat);
     };
-    const finalize = (success, error) => {
+    const finalize = (success, isIdleTimeout, error) => {
       if (resolved) return;
       resolved = true;
       cleanup();
-      if (!success) {
-        if (traceContext) {
-          emitTraceEvent(
-            {
-              evalRunId: traceContext.evalRunId,
-              scenarioId: traceContext.scenarioId,
-              scenarioName: traceContext.scenarioName,
-              targetId: traceContext.targetId,
-              targetName: traceContext.targetName,
-              stepNumber: traceStepNumber + 1,
-              type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
-              outputPreview: JSON.stringify({
-                event: "cli-execution-failed",
-                error: error?.message ?? "Unknown error"
-              }).slice(0, 2e3),
-              timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-              isComplete: true
-            },
-            traceContext.tracePushUrl,
-            traceContext.routeHeader,
-            traceContext.authToken
-          );
-        }
-        reject(
-          error ?? new Error(
-            `OpenCode CLI execution failed (exit code unknown).
-Stderr: ${stderr.slice(0, 1e3)}`
-          )
-        );
-        return;
-      }
-      const endTime = /* @__PURE__ */ new Date();
-      const totalDurationMs = endTime.getTime() - startTime.getTime();
-      let outputText = "";
-      for (const { event: evt } of allEvents) {
-        if (evt.type === "text") {
-          outputText += evt.part.text;
-        }
-      }
-      if (!outputText) {
-        reject(
-          new Error(
-            `Agent produced no text output. Model: ${providerID}/${modelID}, Events: ${allEvents.length}`
-          )
-        );
-        return;
-      }
-      let inputTokens = 0;
-      let outputTokens = 0;
-      let costUsd = 0;
-      for (const { event: evt } of allEvents) {
-        if (evt.type === "step_finish") {
-          const sf = evt;
-          inputTokens += sf.part.tokens.input;
-          outputTokens += sf.part.tokens.output;
-          costUsd += sf.part.cost;
-        }
-      }
-      if (traceContext) {
-        emitTraceEvent(
-          {
-            evalRunId: traceContext.evalRunId,
-            scenarioId: traceContext.scenarioId,
-            scenarioName: traceContext.scenarioName,
-            targetId: traceContext.targetId,
-            targetName: traceContext.targetName,
-            stepNumber: traceStepNumber + 1,
-            type: import_evalforge_types8.LiveTraceEventType.COMPLETION,
-            outputPreview: "Scenario execution completed",
-            timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-            isComplete: true
-          },
-          traceContext.tracePushUrl,
-          traceContext.routeHeader,
-          traceContext.authToken
-        );
-      }
-      const modelStr = options.model || `${providerID}/${modelID}`;
-      const llmTrace = buildLLMTrace(
-        allEvents,
-        totalDurationMs,
-        modelStr,
-        providerID,
-        startTime
-      );
-      const conversation = buildConversation2(allEvents);
       resolve2({
-        result: {
-          outputText,
-          durationMs: totalDurationMs,
-          usage: {
-            inputTokens,
-            outputTokens,
-            totalTokens: inputTokens + outputTokens
-          },
-          costUsd
-        },
-        llmTrace,
-        conversation
+        events,
+        success,
+        isIdleTimeout,
+        error,
+        finalStepNumber: traceStepNumber
       });
     };
     let child;
     try {
       child = (0, import_child_process.spawn)("opencode", args, {
-        cwd: options.cwd,
+        cwd,
         env,
         stdio: ["ignore", "pipe", "pipe"],
         detached: true
       });
     } catch (spawnError) {
-      reject(
-        new Error(
+      resolve2({
+        events: [],
+        success: false,
+        isIdleTimeout: false,
+        error: new Error(
           `Failed to spawn opencode: ${spawnError instanceof Error ? spawnError.message : String(spawnError)}`
-        )
-      );
+        ),
+        finalStepNumber: traceStepNumber
+      });
       return;
     }
     timers.timeout = setTimeout(() => {
       if (!resolved) {
-        console.error(`[OpenCode] Process timed out after ${SDK_TIMEOUT_MS}ms`);
+        console.error(`[OpenCode] Process timed out after ${sdkTimeoutMs}ms`);
         killProcess(child, resolved);
         finalize(
+          false,
           false,
           new Error(
-            `OpenCode execution timed out after ${SDK_TIMEOUT_MS}ms. Skills: ${skillNames}, Scenario: ${scenario.name}, MaxTurns: ${maxTurns}`
+            `OpenCode execution timed out after ${sdkTimeoutMs}ms. Skills: ${skillNames}, Scenario: ${scenarioName}`
           )
         );
       }
-    }, SDK_TIMEOUT_MS);
+    }, sdkTimeoutMs);
     timers.idleCheck = setInterval(() => {
       if (resolved) return;
       const idleTime = Date.now() - lastOutputTime;
-      const effectiveTimeout = isToolRunning ? TOOL_RUNNING_IDLE_TIMEOUT_MS : IDLE_TIMEOUT_MS;
-      if (idleTime >= effectiveTimeout) {
+      if (idleTime >= IDLE_TIMEOUT_MS) {
         console.warn(
-          `[OpenCode] Process appears stuck - no output for ${Math.round(idleTime / 1e3)}s (tool running: ${isToolRunning}). Killing process.`
+          `[OpenCode] Process appears stuck - no output for ${Math.round(idleTime / 1e3)}s. Killing process.`
         );
         killProcess(child, resolved);
         finalize(
           false,
+          true,
           new Error(
-            `OpenCode process stuck - no output for ${Math.round(idleTime / 1e3)} seconds (idle timeout, tool running: ${isToolRunning}). Skills: ${skillNames}, Scenario: ${scenario.name}`
+            `OpenCode process stuck - no output for ${Math.round(idleTime / 1e3)} seconds (idle timeout). Skills: ${skillNames}, Scenario: ${scenarioName}`
           )
         );
       }
@@ -3032,14 +2912,7 @@ Stderr: ${stderr.slice(0, 1e3)}`
         if (!line.trim()) continue;
         const evt = tryParseJson(line);
         if (!evt || !evt.type) continue;
-        allEvents.push({ event: evt, receivedAt: Date.now() });
-        if (evt.type === "tool_use") {
-          const tu = evt;
-          const status = tu.part.state.status;
-          isToolRunning = status !== "completed" && status !== "error";
-        } else {
-          isToolRunning = false;
-        }
+        events.push({ event: evt, receivedAt: Date.now() });
         if (traceContext) {
           traceStepNumber++;
           const traceEvt = createTraceEventFromNdjson(
@@ -3084,16 +2957,17 @@ Stderr: ${stderr.slice(0, 1e3)}`
       if (lineBuffer.trim()) {
         const evt = tryParseJson(lineBuffer);
         if (evt && evt.type) {
-          allEvents.push({ event: evt, receivedAt: Date.now() });
+          events.push({ event: evt, receivedAt: Date.now() });
         }
       }
       console.log(
-        `[executeWithOpenCode] Process exited with code ${code}, ${allEvents.length} events collected`
+        `[executeWithOpenCode] Process exited with code ${code}, ${events.length} events collected`
       );
       if (code === 0) {
-        finalize(true);
+        finalize(true, false);
       } else {
         finalize(
+          false,
           false,
           new Error(
             `OpenCode CLI exited with code ${code}.
@@ -3103,10 +2977,240 @@ Stderr: ${stderr.slice(0, 1e3)}`
       }
     });
     child.on("error", (error) => {
-      finalize(false, new Error(`OpenCode CLI spawn error: ${error.message}`));
+      finalize(
+        false,
+        false,
+        new Error(`OpenCode CLI spawn error: ${error.message}`)
+      );
     });
   });
 }
+async function executeWithOpenCode(skills, scenario, options) {
+  const skillNames = skills.map((s) => s.name).join(", ");
+  console.log("[executeWithOpenCode] Starting execution", {
+    skillCount: skills.length,
+    skillNames,
+    scenarioId: scenario.id,
+    scenarioName: scenario.name,
+    cwd: options.cwd,
+    aiGatewayUrl: options.aiGatewayUrl,
+    hasAiGatewayHeaders: !!options.aiGatewayHeaders,
+    model: options.model
+  });
+  const startTime = /* @__PURE__ */ new Date();
+  const maxTurns = options.maxTurns ?? 10;
+  const sdkTimeoutMs = Math.max(3e5, maxTurns * 6e4);
+  const { env, providerID, modelID } = await buildOpenCodeEnv({
+    model: options.model,
+    temperature: options.temperature,
+    maxTurns,
+    aiGatewayUrl: options.aiGatewayUrl,
+    aiGatewayHeaders: options.aiGatewayHeaders,
+    mcps: options.mcps,
+    cwd: options.cwd
+  });
+  const traceContext = options.traceContext;
+  if (traceContext) {
+    emitTraceEvent(
+      {
+        evalRunId: traceContext.evalRunId,
+        scenarioId: traceContext.scenarioId,
+        scenarioName: traceContext.scenarioName,
+        targetId: traceContext.targetId,
+        targetName: traceContext.targetName,
+        stepNumber: 0,
+        type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
+        outputPreview: JSON.stringify({
+          event: "pre-cli-execution",
+          model: `${providerID}/${modelID}`,
+          maxTurns,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        }),
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        isComplete: false
+      },
+      traceContext.tracePushUrl,
+      traceContext.routeHeader,
+      traceContext.authToken
+    );
+  }
+  let systemPrompt;
+  if (options.systemPrompt === null || options.systemPrompt === "") {
+  } else if (options.systemPrompt != null) {
+    systemPrompt = options.systemPrompt;
+  } else {
+    systemPrompt = import_evalforge_types8.DEFAULT_EVALUATOR_SYSTEM_PROMPT;
+  }
+  if (systemPrompt) {
+    await writeSystemPromptRule(options.cwd, systemPrompt);
+  }
+  const baseArgs = [
+    "run",
+    "--format",
+    "json",
+    "--thinking",
+    "--variant",
+    "high",
+    "--model",
+    `${providerID}/${modelID}`,
+    "--dir",
+    options.cwd
+  ];
+  const accumulatedEvents = [];
+  let traceStepNumber = 0;
+  let lastAttemptResult;
+  for (let attempt = 1; attempt <= MAX_IDLE_RETRIES; attempt++) {
+    const prompt = attempt === 1 ? scenario.triggerPrompt : buildRecoveryPrompt(scenario.triggerPrompt, accumulatedEvents);
+    if (attempt > 1) {
+      console.log(
+        `[OpenCode] Retry attempt ${attempt}/${MAX_IDLE_RETRIES} \u2014 starting fresh session with recovery context`
+      );
+      if (traceContext) {
+        emitTraceEvent(
+          {
+            evalRunId: traceContext.evalRunId,
+            scenarioId: traceContext.scenarioId,
+            scenarioName: traceContext.scenarioName,
+            targetId: traceContext.targetId,
+            targetName: traceContext.targetName,
+            stepNumber: traceStepNumber + 1,
+            type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
+            outputPreview: JSON.stringify({
+              event: "idle-timeout-retry",
+              attempt,
+              maxRetries: MAX_IDLE_RETRIES,
+              eventsFromPreviousAttempts: accumulatedEvents.length
+            }),
+            timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+            isComplete: false
+          },
+          traceContext.tracePushUrl,
+          traceContext.routeHeader,
+          traceContext.authToken
+        );
+      }
+    }
+    const args = [...baseArgs, prompt];
+    console.log(
+      `[executeWithOpenCode] Spawning attempt ${attempt}: opencode`,
+      args.slice(0, 5)
+    );
+    lastAttemptResult = await spawnOpenCodeProcess({
+      args,
+      env,
+      cwd: options.cwd,
+      skillNames,
+      scenarioName: scenario.name,
+      sdkTimeoutMs,
+      traceContext,
+      initialStepNumber: traceStepNumber
+    });
+    accumulatedEvents.push(...lastAttemptResult.events);
+    traceStepNumber = lastAttemptResult.finalStepNumber;
+    if (lastAttemptResult.success) {
+      break;
+    }
+    if (!lastAttemptResult.isIdleTimeout || attempt >= MAX_IDLE_RETRIES) {
+      if (traceContext) {
+        emitTraceEvent(
+          {
+            evalRunId: traceContext.evalRunId,
+            scenarioId: traceContext.scenarioId,
+            scenarioName: traceContext.scenarioName,
+            targetId: traceContext.targetId,
+            targetName: traceContext.targetName,
+            stepNumber: traceStepNumber + 1,
+            type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
+            outputPreview: JSON.stringify({
+              event: "cli-execution-failed",
+              error: lastAttemptResult.error?.message ?? "Unknown error",
+              attempt,
+              isIdleTimeout: lastAttemptResult.isIdleTimeout
+            }).slice(0, 2e3),
+            timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+            isComplete: true
+          },
+          traceContext.tracePushUrl,
+          traceContext.routeHeader,
+          traceContext.authToken
+        );
+      }
+      throw lastAttemptResult.error ?? new Error(
+        `OpenCode CLI execution failed.
+Attempt: ${attempt}, Events: ${accumulatedEvents.length}`
+      );
+    }
+    console.warn(
+      `[OpenCode] Attempt ${attempt} failed due to idle timeout, will retry`
+    );
+  }
+  const endTime = /* @__PURE__ */ new Date();
+  const totalDurationMs = endTime.getTime() - startTime.getTime();
+  let outputText = "";
+  for (const { event: evt } of accumulatedEvents) {
+    if (evt.type === "text") {
+      outputText += evt.part.text;
+    }
+  }
+  if (!outputText) {
+    throw new Error(
+      `Agent produced no text output. Model: ${providerID}/${modelID}, Events: ${accumulatedEvents.length}`
+    );
+  }
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let costUsd = 0;
+  for (const { event: evt } of accumulatedEvents) {
+    if (evt.type === "step_finish") {
+      const sf = evt;
+      inputTokens += sf.part.tokens.input;
+      outputTokens += sf.part.tokens.output;
+      costUsd += sf.part.cost;
+    }
+  }
+  if (traceContext) {
+    emitTraceEvent(
+      {
+        evalRunId: traceContext.evalRunId,
+        scenarioId: traceContext.scenarioId,
+        scenarioName: traceContext.scenarioName,
+        targetId: traceContext.targetId,
+        targetName: traceContext.targetName,
+        stepNumber: traceStepNumber + 1,
+        type: import_evalforge_types8.LiveTraceEventType.COMPLETION,
+        outputPreview: "Scenario execution completed",
+        timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+        isComplete: true
+      },
+      traceContext.tracePushUrl,
+      traceContext.routeHeader,
+      traceContext.authToken
+    );
+  }
+  const modelStr = options.model || `${providerID}/${modelID}`;
+  const llmTrace = buildLLMTrace(
+    accumulatedEvents,
+    totalDurationMs,
+    modelStr,
+    providerID,
+    startTime
+  );
+  const conversation = buildConversation2(accumulatedEvents);
+  return {
+    result: {
+      outputText,
+      durationMs: totalDurationMs,
+      usage: {
+        inputTokens,
+        outputTokens,
+        totalTokens: inputTokens + outputTokens
+      },
+      costUsd
+    },
+    llmTrace,
+    conversation
+  };
+}
 
 // src/run-scenario/agents/opencode/opencode-adapter.ts
 var OpenCodeAdapter = class {
@@ -4465,13 +4569,14 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     infrastructurePaths
   );
   const templateFiles = workDir ? extractTemplateFiles(beforeSnapshot, afterSnapshot, infrastructurePaths) : void 0;
+  const resolvedModelConfig = agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
   return {
     id: (0, import_crypto4.randomUUID)(),
     targetId,
     targetName,
     scenarioId: scenario.id,
     scenarioName: scenario.name,
-    modelConfig: agent?.modelConfig,
+    modelConfig: resolvedModelConfig,
     duration: durationMs,
     outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,