@wix/evalforge-evaluator 0.117.0 → 0.119.0

package/build/index.js

@@ -1200,10 +1200,10 @@ function createTraceEventFromAnyMessage(message, context, stepNumber, isComplete
   };
 }
 async function prepareClaudeCodeEnvironment(cwd, skills, options) {
-  const { mkdir: mkdirAsync, writeFile: writeFile6 } = await import("fs/promises");
+  const { mkdir: mkdirAsync, writeFile: writeFile7 } = await import("fs/promises");
   const claudeDir = `${cwd}/.claude`;
   await mkdirAsync(claudeDir, { recursive: true });
-  await writeFile6(`${claudeDir}/settings.json`, "{}", {
+  await writeFile7(`${claudeDir}/settings.json`, "{}", {
     flag: "wx"
   }).catch(() => {
   });
@@ -2162,9 +2162,18 @@ defaultRegistry.register(claudeCodeAdapter);
 var import_evalforge_types9 = require("@wix/evalforge-types");
 
 // src/run-scenario/agents/opencode/execute.ts
-var import_os3 = require("os");
+var import_child_process = require("child_process");
 var import_evalforge_types8 = require("@wix/evalforge-types");
 
+// src/run-scenario/agents/opencode/types.ts
+function tryParseJson(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
 // src/run-scenario/agents/opencode/write-skills.ts
 var import_promises7 = require("fs/promises");
 var import_path8 = require("path");
@@ -2260,6 +2269,7 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = import_eval
 }
 
 // src/run-scenario/agents/opencode/config.ts
+var import_os3 = require("os");
 var import_evalforge_types6 = require("@wix/evalforge-types");
 var DEFAULT_MODEL2 = `${import_evalforge_types6.ClaudeModel.CLAUDE_4_5_SONNET_1_0}`;
 function parseModel(model) {
@@ -2312,7 +2322,14 @@ function toOpenCodeMcpConfig(servers) {
   }
   return result;
 }
-async function buildOpenCodeConfig(options) {
+function ensureOpenCodeInPath(currentPath) {
+  const opencodeBin = `${(0, import_os3.homedir)()}/.opencode/bin`;
+  if (currentPath.includes(opencodeBin)) {
+    return currentPath;
+  }
+  return `${opencodeBin}:${currentPath}`;
+}
+async function buildOpenCodeEnv(options) {
   const modelStr = options.model || DEFAULT_MODEL2;
   const { providerID, modelID } = parseModel(modelStr);
   const provider = {};
@@ -2324,9 +2341,7 @@ async function buildOpenCodeConfig(options) {
     if (options.aiGatewayHeaders) {
       providerOptions.headers = { ...options.aiGatewayHeaders };
     }
-    provider[providerID] = {
-      options: providerOptions
-    };
+    provider[providerID] = { options: providerOptions };
   }
   let mcp;
   if (options.mcps && options.mcps.length > 0) {
@@ -2367,70 +2382,81 @@ async function buildOpenCodeConfig(options) {
     },
     ...mcp ? { mcp } : {}
   };
-  return { config, providerID, modelID };
+  const env = {
+    ...process.env,
+    PATH: ensureOpenCodeInPath(process.env.PATH || ""),
+    OPENCODE_CONFIG_CONTENT: JSON.stringify(config),
+    OPENCODE_DISABLE_LSP_DOWNLOAD: "true"
+  };
+  return { env, providerID, modelID };
 }
 
 // src/run-scenario/agents/opencode/build-trace.ts
 var import_evalforge_types7 = require("@wix/evalforge-types");
 var import_crypto2 = require("crypto");
-function buildLLMTrace(messages, totalDurationMs, model, provider) {
-  const assistantMessages = messages.filter(
-    (m) => m.info.role === "assistant"
-  );
-  const allSteps = assistantMessages.flatMap((msg, turnIndex) => {
-    const { info, parts } = msg;
-    let text = "";
-    let thinking = "";
-    const toolCalls = [];
-    let stepInputTokens = 0;
-    let stepOutputTokens = 0;
-    let stepCost = 0;
-    let finishReason = "unknown";
-    for (const part of parts) {
-      switch (part.type) {
-        case "text": {
-          const textPart = part;
-          text += textPart.text;
-          break;
-        }
-        case "reasoning": {
-          const reasoningPart = part;
-          thinking += reasoningPart.text;
-          break;
-        }
-        case "tool": {
-          const toolPart = part;
-          toolCalls.push({
-            toolName: toolPart.tool,
-            args: toolPart.state.input
-          });
-          break;
-        }
-        case "step-finish": {
-          const sf = part;
-          stepInputTokens += sf.tokens.input;
-          stepOutputTokens += sf.tokens.output;
-          stepCost += sf.cost;
-          finishReason = sf.reason;
-          break;
-        }
+function buildLLMTrace(timestampedEvents, totalDurationMs, model, provider, executionStartTime) {
+  const turns = [];
+  let current = {
+    textParts: [],
+    reasoningParts: [],
+    toolCalls: []
+  };
+  for (const { event: evt, receivedAt } of timestampedEvents) {
+    switch (evt.type) {
+      case "text":
+        current.textParts.push(evt.part.text);
+        break;
+      case "reasoning":
+        current.reasoningParts.push(evt.part.text);
+        break;
+      case "tool_use": {
+        const tu = evt;
+        current.toolCalls.push({
+          toolName: tu.part.tool,
+          args: tu.part.state.input
+        });
+        break;
+      }
+      case "step_finish": {
+        const sf = evt;
+        current.stepFinish = sf.part;
+        current.receivedAt = receivedAt;
+        turns.push(current);
+        current = {
+          textParts: [],
+          reasoningParts: [],
+          toolCalls: []
+        };
+        break;
       }
     }
-    if (stepInputTokens === 0 && stepOutputTokens === 0) {
-      stepInputTokens = info.tokens.input;
-      stepOutputTokens = info.tokens.output;
-      stepCost = info.cost;
-    }
-    const startedAt = new Date(info.time.created).toISOString();
-    const completedAt = info.time.completed ? info.time.completed : turnIndex + 1 < assistantMessages.length ? assistantMessages[turnIndex + 1].info.time.created : info.time.created + totalDurationMs;
-    const durationMs = Math.max(0, completedAt - info.time.created);
-    const isSuccess = finishReason !== "error";
-    const errorMsg = finishReason === "error" ? "Generation failed" : void 0;
-    const stepModel = info.modelID || model;
-    const stepProvider = info.providerID || provider;
-    const toolCallCount = toolCalls.length;
+  }
+  if (current.textParts.length > 0 || current.reasoningParts.length > 0 || current.toolCalls.length > 0) {
+    if (timestampedEvents.length > 0) {
+      current.receivedAt = timestampedEvents[timestampedEvents.length - 1].receivedAt;
+    }
+    turns.push(current);
+  }
+  const executionStartMs = executionStartTime.getTime();
+  const allSteps = turns.flatMap((turn, turnIndex) => {
+    const sf = turn.stepFinish;
+    const stepInputTokens = sf?.tokens.input ?? 0;
+    const stepOutputTokens = sf?.tokens.output ?? 0;
+    const stepCost = sf?.cost ?? 0;
+    const finishReason = sf?.reason ?? "unknown";
+    const stepModel = sf?.modelID || model;
+    const stepProvider = sf?.providerID || provider;
+    const turnEndMs = turn.receivedAt ?? executionStartMs + totalDurationMs;
+    const prevEndMs = turnIndex > 0 ? turns[turnIndex - 1].receivedAt ?? executionStartMs : executionStartMs;
+    const durationMs = Math.max(0, turnEndMs - prevEndMs);
+    const startedAt = new Date(prevEndMs).toISOString();
+    const text = turn.textParts.join("");
+    const thinking = turn.reasoningParts.join("");
+    const toolCallCount = turn.toolCalls.length;
     const hasThinking = !!thinking;
     const hasText = !!text;
+    const isSuccess = finishReason !== "error";
+    const errorMsg = finishReason === "error" ? "Generation failed" : void 0;
     const subSteps = [];
     const thinkingSubSteps = hasThinking && (hasText || toolCallCount > 0) ? 1 : 0;
     const toolSubSteps = toolCallCount;
@@ -2440,7 +2466,6 @@ function buildLLMTrace(messages, totalDurationMs, model, provider) {
       subSteps.push({
         id: (0, import_crypto2.randomUUID)(),
         stepNumber: 0,
-        // renumbered below
         turnIndex,
         type: import_evalforge_types7.LLMStepType.THINKING,
         model: stepModel,
@@ -2462,7 +2487,7 @@ function buildLLMTrace(messages, totalDurationMs, model, provider) {
     }
     if (toolCallCount > 0) {
       for (let tcIdx = 0; tcIdx < toolCallCount; tcIdx++) {
-        const tc = toolCalls[tcIdx];
+        const tc = turn.toolCalls[tcIdx];
         const isLast = tcIdx === toolCallCount - 1 && textSubSteps === 0;
         const toolBudgetSteps = toolSubSteps + textSubSteps;
         const toolFraction = toolBudgetSteps > 0 ? 1 / toolBudgetSteps : 1;
@@ -2541,11 +2566,21 @@ function buildLLMTrace(messages, totalDurationMs, model, provider) {
     }
     return subSteps;
   }).map((s, i) => ({ ...s, stepNumber: i + 1 }));
-  const totalTokens = buildTotalTokens(assistantMessages);
-  const totalCost = assistantMessages.reduce((sum, m) => {
-    const aMsg = m.info;
-    return sum + aMsg.cost;
-  }, 0);
+  let totalPrompt = 0;
+  let totalCompletion = 0;
+  let totalCost = 0;
+  for (const turn of turns) {
+    if (turn.stepFinish) {
+      totalPrompt += turn.stepFinish.tokens.input;
+      totalCompletion += turn.stepFinish.tokens.output;
+      totalCost += turn.stepFinish.cost;
+    }
+  }
+  const totalTokens = {
+    prompt: totalPrompt,
+    completion: totalCompletion,
+    total: totalPrompt + totalCompletion
+  };
   const stepTypeBreakdown = {};
   for (const step of allSteps) {
     const entry = stepTypeBreakdown[step.type] ?? {
@@ -2563,7 +2598,7 @@ function buildLLMTrace(messages, totalDurationMs, model, provider) {
   const modelUsed = allSteps[0]?.model || model;
   const summary = {
     totalSteps: allSteps.length,
-    totalTurns: assistantMessages.length,
+    totalTurns: turns.length,
     totalDurationMs,
     totalTokens,
     totalCostUsd: totalCost,
@@ -2584,116 +2619,100 @@ function buildLLMTrace(messages, totalDurationMs, model, provider) {
     summary
   };
 }
-function buildTotalTokens(assistantMessages) {
-  let prompt = 0;
-  let completion = 0;
-  for (const { info } of assistantMessages) {
-    prompt += info.tokens.input;
-    completion += info.tokens.output;
-  }
-  return { prompt, completion, total: prompt + completion };
-}
 
 // src/run-scenario/agents/opencode/build-conversation.ts
-function buildConversation2(messages) {
+function buildConversation2(timestampedEvents) {
   const result = [];
-  for (const { info, parts } of messages) {
-    const timestamp = new Date(info.time.created).toISOString();
-    if (info.role === "assistant") {
-      const content = [];
-      for (const part of parts) {
-        switch (part.type) {
-          case "text": {
-            const textPart = part;
-            content.push({ type: "text", text: textPart.text });
-            break;
-          }
-          case "reasoning": {
-            const reasoningPart = part;
-            content.push({ type: "thinking", thinking: reasoningPart.text });
-            break;
-          }
-          case "tool": {
-            const toolPart = part;
-            content.push({
-              type: "tool_use",
-              toolName: toolPart.tool,
-              toolId: toolPart.callID,
-              input: toolPart.state.input
-            });
-            break;
-          }
-        }
+  let assistantContent = [];
+  let userContent = [];
+  let latestReceivedAt = 0;
+  const flushAssistant = () => {
+    if (assistantContent.length > 0) {
+      const timestamp = latestReceivedAt > 0 ? new Date(latestReceivedAt).toISOString() : (/* @__PURE__ */ new Date()).toISOString();
+      result.push({ role: "assistant", content: assistantContent, timestamp });
+      assistantContent = [];
+    }
+  };
+  const flushUser = () => {
+    if (userContent.length > 0) {
+      const timestamp = latestReceivedAt > 0 ? new Date(latestReceivedAt).toISOString() : (/* @__PURE__ */ new Date()).toISOString();
+      result.push({ role: "user", content: userContent, timestamp });
+      userContent = [];
+    }
+  };
+  for (const { event: evt, receivedAt } of timestampedEvents) {
+    latestReceivedAt = receivedAt;
+    switch (evt.type) {
+      case "text": {
+        const te = evt;
+        assistantContent.push({ type: "text", text: te.part.text });
+        break;
       }
-      if (content.length > 0) {
-        result.push({ role: "assistant", content, timestamp });
+      case "reasoning": {
+        const re = evt;
+        assistantContent.push({ type: "thinking", thinking: re.part.text });
+        break;
       }
-    } else if (info.role === "user") {
-      const content = [];
-      for (const part of parts) {
-        if (part.type === "text") {
-          const textPart = part;
-          content.push({ type: "text", text: textPart.text });
-        } else if (part.type === "tool") {
-          const toolPart = part;
-          const state = toolPart.state;
-          if (state.status === "completed") {
-            const completed = state;
-            content.push({
-              type: "tool_result",
-              toolUseId: toolPart.callID,
-              content: completed.output
-            });
-          } else if (state.status === "error") {
-            const errState = state;
-            content.push({
-              type: "tool_result",
-              toolUseId: toolPart.callID,
-              content: errState.error,
-              isError: true
-            });
-          }
+      case "tool_use": {
+        const tu = evt;
+        assistantContent.push({
+          type: "tool_use",
+          toolName: tu.part.tool,
+          toolId: tu.part.callID,
+          input: tu.part.state.input
+        });
+        if (tu.part.state.status === "completed" || tu.part.state.status === "error") {
+          flushAssistant();
+          const isError = tu.part.state.status === "error";
+          const content = isError ? tu.part.state.error || "Tool execution failed" : tu.part.state.output || "";
+          userContent.push({
+            type: "tool_result",
+            toolUseId: tu.part.callID,
+            content,
+            ...isError ? { isError: true } : {}
+          });
+          flushUser();
         }
+        break;
       }
-      if (content.length > 0) {
-        result.push({ role: "user", content, timestamp });
+      case "step_finish": {
+        flushAssistant();
+        flushUser();
+        break;
       }
     }
   }
+  flushAssistant();
+  flushUser();
   return result;
 }
 
 // src/run-scenario/agents/opencode/execute.ts
-var DEFAULT_MODEL3 = `anthropic/${import_evalforge_types8.ClaudeModel.CLAUDE_4_5_SONNET_1_0}`;
-function ensureOpenCodeInPath() {
-  const opencodeBin = `${(0, import_os3.homedir)()}/.opencode/bin`;
-  const currentPath = process.env.PATH || "";
-  if (!currentPath.includes(opencodeBin)) {
-    process.env.PATH = `${opencodeBin}:${currentPath}`;
-  }
-}
+var import_promises9 = require("fs/promises");
+var import_path10 = require("path");
+var KILL_GRACE_PERIOD_MS = 5e3;
+var IDLE_TIMEOUT_MS = 12e4;
+var IDLE_CHECK_INTERVAL_MS = 15e3;
 function extractToolAction(toolName, args) {
   if (!toolName) return "Using tool...";
-  const a = args;
-  if ((toolName === "Task" || toolName === "dispatch_agent") && a?.description) {
-    const desc = String(a.description).slice(0, 55);
-    return `Task: ${desc}${String(a.description).length > 55 ? "..." : ""}`;
-  }
-  if ((toolName === "Bash" || toolName === "bash" || toolName === "execute") && a?.command) {
-    const cmd = String(a.command).slice(0, 50);
-    return `Running: ${cmd}${String(a.command).length > 50 ? "..." : ""}`;
-  }
-  if (a?.file_path || a?.path || a?.target_file) {
-    const filePath = String(a.file_path || a.path || a.target_file).slice(
-      0,
-      50
-    );
+  if ((toolName === "Task" || toolName === "dispatch_agent") && args?.description) {
+    const desc = String(args.description).slice(0, 55);
+    return `Task: ${desc}${String(args.description).length > 55 ? "..." : ""}`;
+  }
+  if ((toolName === "Bash" || toolName === "bash" || toolName === "execute") && args?.command) {
+    const cmd = String(args.command).slice(0, 50);
+    return `Running: ${cmd}${String(args.command).length > 50 ? "..." : ""}`;
+  }
+  if (args?.file_path || args?.path || args?.target_file) {
+    const filePath = String(
+      args.file_path || args.path || args.target_file
+    ).slice(0, 50);
     if (/write|edit/i.test(toolName)) return `Writing: ${filePath}`;
     if (/read|view/i.test(toolName)) return `Reading: ${filePath}`;
   }
   return `Using ${toolName}...`;
 }
-function createTraceEventFromPart(part, context, stepNumber, isComplete) {
+function createTraceEventFromNdjson(evt, context, stepNumber, isComplete) {
   const base = {
     evalRunId: context.evalRunId,
     scenarioId: context.scenarioId,
@@ -2704,42 +2723,41 @@ function createTraceEventFromPart(part, context, stepNumber, isComplete) {
     timestamp: (/* @__PURE__ */ new Date()).toISOString(),
     isComplete
   };
-  switch (part.type) {
+  switch (evt.type) {
     case "text": {
-      const textPart = part;
+      const te = evt;
       return {
         ...base,
         type: import_evalforge_types8.LiveTraceEventType.COMPLETION,
-        outputPreview: textPart.text.slice(0, 500)
+        outputPreview: te.part.text.slice(0, 500)
       };
     }
-    case "reasoning": {
-      const reasoningPart = part;
+    case "reasoning":
       return {
         ...base,
         type: import_evalforge_types8.LiveTraceEventType.THINKING,
-        thinking: reasoningPart.text.slice(0, 500)
+        thinking: evt.part.text.slice(0, 500)
       };
-    }
-    case "tool": {
-      const toolPart = part;
-      const toolName = toolPart.tool;
-      const args = toolPart.state.input;
+    case "tool_use": {
+      const tu = evt;
+      const toolName = tu.part.tool;
+      const args = tu.part.state.input;
       const toolArgs = JSON.stringify(args).slice(0, 500);
       let type = import_evalforge_types8.LiveTraceEventType.TOOL_USE;
       let filePath;
-      const a = args;
-      if (a.file_path || a.path || a.target_file) {
-        filePath = String(a.file_path || a.path || a.target_file);
-        if (/write|edit/i.test(toolName)) {
-          type = import_evalforge_types8.LiveTraceEventType.FILE_WRITE;
-        } else if (/read|view/i.test(toolName)) {
-          type = import_evalforge_types8.LiveTraceEventType.FILE_READ;
+      if (args) {
+        if (args.file_path || args.path || args.target_file) {
+          filePath = String(args.file_path || args.path || args.target_file);
+          if (/write|edit/i.test(toolName)) {
+            type = import_evalforge_types8.LiveTraceEventType.FILE_WRITE;
+          } else if (/read|view/i.test(toolName)) {
+            type = import_evalforge_types8.LiveTraceEventType.FILE_READ;
+          }
         }
       }
       return { ...base, type, toolName, toolArgs, filePath };
     }
-    case "step-finish":
+    case "step_finish":
       return {
         ...base,
         type: import_evalforge_types8.LiveTraceEventType.PROGRESS,
@@ -2769,6 +2787,37 @@ async function prepareOpenCodeEnvironment(cwd, skills, options) {
     );
   }
 }
+async function writeSystemPromptRule(cwd, systemPrompt) {
+  const rulesDir = (0, import_path10.join)(cwd, ".opencode", "rules");
+  await (0, import_promises9.mkdir)(rulesDir, { recursive: true });
+  await (0, import_promises9.writeFile)(
+    (0, import_path10.join)(rulesDir, "evalforge-system-prompt.md"),
+    systemPrompt,
+    "utf-8"
+  );
+}
+function killProcess(child, resolved) {
+  if (!child) return;
+  const killSignal = (signal) => {
+    if (child.pid) {
+      try {
+        process.kill(-child.pid, signal);
+        console.log(
+          `[OpenCode] Sent ${signal} to process group (pid: -${child.pid})`
+        );
+        return;
+      } catch {
+      }
+    }
+    child.kill(signal);
+  };
+  killSignal("SIGTERM");
+  setTimeout(() => {
+    if (child && !resolved) {
+      killSignal("SIGKILL");
+    }
+  }, KILL_GRACE_PERIOD_MS);
+}
 async function executeWithOpenCode(skills, scenario, options) {
   const skillNames = skills.map((s) => s.name).join(", ");
   console.log("[executeWithOpenCode] Starting execution", {
@@ -2783,7 +2832,8 @@ async function executeWithOpenCode(skills, scenario, options) {
   });
   const startTime = /* @__PURE__ */ new Date();
   const maxTurns = options.maxTurns ?? 10;
-  const { config, providerID, modelID } = await buildOpenCodeConfig({
+  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
+  const { env, providerID, modelID } = await buildOpenCodeEnv({
     model: options.model,
     temperature: options.temperature,
     maxTurns,
@@ -2792,12 +2842,6 @@ async function executeWithOpenCode(skills, scenario, options) {
     mcps: options.mcps,
     cwd: options.cwd
   });
-  const { createOpencodeServer, createOpencodeClient } = await import("@opencode-ai/sdk");
-  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
-  const abortController = new AbortController();
-  let timeoutHandle;
-  let heartbeatHandle;
-  let timedOut = false;
   const traceContext = options.traceContext;
   let traceStepNumber = 0;
   let lastAction = "Starting...";
@@ -2814,7 +2858,7 @@ async function executeWithOpenCode(skills, scenario, options) {
         stepNumber: 0,
         type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
         outputPreview: JSON.stringify({
-          event: "pre-sdk-execution",
+          event: "pre-cli-execution",
           model: `${providerID}/${modelID}`,
           maxTurns,
           timestamp: (/* @__PURE__ */ new Date()).toISOString()
@@ -2827,105 +2871,200 @@ async function executeWithOpenCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
-  let server;
-  try {
-    ensureOpenCodeInPath();
-    console.log("[SDK-DEBUG] Starting OpenCode server...");
-    server = await createOpencodeServer({
-      config,
-      signal: abortController.signal,
-      timeout: 3e4
-    });
-    console.log(`[SDK-DEBUG] Server started at ${server.url}`);
-    const client = createOpencodeClient({
-      baseUrl: server.url,
-      directory: options.cwd
-    });
-    const session = await client.session.create({
-      body: { title: `eval-${scenario.name}` }
-    });
-    if (!session.data) {
-      const errorDetail = "error" in session ? JSON.stringify(session.error) : "unknown";
-      throw new Error(
-        `OpenCode session.create() failed: ${errorDetail} (HTTP ${session.response?.status})`
+  let systemPrompt;
+  if (options.systemPrompt === null || options.systemPrompt === "") {
+  } else if (options.systemPrompt != null) {
+    systemPrompt = options.systemPrompt;
+  } else {
+    systemPrompt = import_evalforge_types8.DEFAULT_EVALUATOR_SYSTEM_PROMPT;
+  }
+  if (systemPrompt) {
+    await writeSystemPromptRule(options.cwd, systemPrompt);
+  }
+  const args = [
+    "run",
+    "--format",
+    "json",
+    "--thinking",
+    "--variant",
+    "high",
+    "--model",
+    `${providerID}/${modelID}`,
+    "--dir",
+    options.cwd,
+    // NOTE: Trigger prompt is passed as a positional CLI arg. On Linux a single
+    // arg is capped at 128 KB (MAX_ARG_STRLEN); on macOS the combined args+env
+    // share a ~1 MB limit. Prompts exceeding this would fail with E2BIG.
+    // In practice eval prompts are well under this limit.
+    scenario.triggerPrompt
+  ];
+  console.log("[executeWithOpenCode] Spawning: opencode", args.slice(0, 5));
+  return new Promise((resolve2, reject) => {
+    let resolved = false;
+    let stderr = "";
+    let lineBuffer = "";
+    let lastOutputTime = Date.now();
+    const allEvents = [];
+    const timers = {};
+    const cleanup = () => {
+      if (timers.timeout) clearTimeout(timers.timeout);
+      if (timers.idleCheck) clearInterval(timers.idleCheck);
+      if (timers.heartbeat) clearInterval(timers.heartbeat);
+    };
+    const finalize = (success, error) => {
+      if (resolved) return;
+      resolved = true;
+      cleanup();
+      if (!success) {
+        if (traceContext) {
+          emitTraceEvent(
+            {
+              evalRunId: traceContext.evalRunId,
+              scenarioId: traceContext.scenarioId,
+              scenarioName: traceContext.scenarioName,
+              targetId: traceContext.targetId,
+              targetName: traceContext.targetName,
+              stepNumber: traceStepNumber + 1,
+              type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
+              outputPreview: JSON.stringify({
+                event: "cli-execution-failed",
+                error: error?.message ?? "Unknown error"
+              }).slice(0, 2e3),
+              timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+              isComplete: true
+            },
+            traceContext.tracePushUrl,
+            traceContext.routeHeader,
+            traceContext.authToken
+          );
+        }
+        reject(
+          error ?? new Error(
+            `OpenCode CLI execution failed (exit code unknown).
+Stderr: ${stderr.slice(0, 1e3)}`
+          )
+        );
+        return;
+      }
+      const endTime = /* @__PURE__ */ new Date();
+      const totalDurationMs = endTime.getTime() - startTime.getTime();
+      let outputText = "";
+      for (const { event: evt } of allEvents) {
+        if (evt.type === "text") {
+          outputText += evt.part.text;
+        }
+      }
+      if (!outputText) {
+        reject(
+          new Error(
+            `Agent produced no text output. Model: ${providerID}/${modelID}, Events: ${allEvents.length}`
+          )
+        );
+        return;
+      }
+      let inputTokens = 0;
+      let outputTokens = 0;
+      let costUsd = 0;
+      for (const { event: evt } of allEvents) {
+        if (evt.type === "step_finish") {
+          const sf = evt;
+          inputTokens += sf.part.tokens.input;
+          outputTokens += sf.part.tokens.output;
+          costUsd += sf.part.cost;
+        }
+      }
+      if (traceContext) {
+        emitTraceEvent(
+          {
+            evalRunId: traceContext.evalRunId,
+            scenarioId: traceContext.scenarioId,
+            scenarioName: traceContext.scenarioName,
+            targetId: traceContext.targetId,
+            targetName: traceContext.targetName,
+            stepNumber: traceStepNumber + 1,
+            type: import_evalforge_types8.LiveTraceEventType.COMPLETION,
+            outputPreview: "Scenario execution completed",
+            timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+            isComplete: true
+          },
+          traceContext.tracePushUrl,
+          traceContext.routeHeader,
+          traceContext.authToken
+        );
+      }
+      const modelStr = options.model || `${providerID}/${modelID}`;
+      const llmTrace = buildLLMTrace(
+        allEvents,
+        totalDurationMs,
+        modelStr,
+        providerID,
+        startTime
       );
-    }
-    const sessionId = session.data.id;
-    console.log(`[SDK-DEBUG] Session created: ${sessionId}`);
-    let eventStreamAbort;
+      const conversation = buildConversation2(allEvents);
+      resolve2({
+        result: {
+          outputText,
+          durationMs: totalDurationMs,
+          usage: {
+            inputTokens,
+            outputTokens,
+            totalTokens: inputTokens + outputTokens
+          },
+          costUsd
+        },
+        llmTrace,
+        conversation
+      });
+    };
+    let child;
+    try {
+      child = (0, import_child_process.spawn)("opencode", args, {
+        cwd: options.cwd,
+        env,
+        stdio: ["ignore", "pipe", "pipe"],
+        detached: true
+      });
+    } catch (spawnError) {
+      reject(
+        new Error(
+          `Failed to spawn opencode: ${spawnError instanceof Error ? spawnError.message : String(spawnError)}`
+        )
+      );
+      return;
+    }
+    timers.timeout = setTimeout(() => {
+      if (!resolved) {
+        console.error(`[OpenCode] Process timed out after ${SDK_TIMEOUT_MS}ms`);
+        killProcess(child, resolved);
+        finalize(
+          false,
+          new Error(
+            `OpenCode execution timed out after ${SDK_TIMEOUT_MS}ms. Skills: ${skillNames}, Scenario: ${scenario.name}, MaxTurns: ${maxTurns}`
+          )
+        );
+      }
+    }, SDK_TIMEOUT_MS);
+    timers.idleCheck = setInterval(() => {
+      if (resolved) return;
+      const idleTime = Date.now() - lastOutputTime;
+      if (idleTime >= IDLE_TIMEOUT_MS) {
+        console.warn(
+          `[OpenCode] Process appears stuck - no output for ${Math.round(idleTime / 1e3)}s. Killing process.`
+        );
+        killProcess(child, resolved);
+        finalize(
+          false,
+          new Error(
+            `OpenCode process stuck - no output for ${Math.round(idleTime / 1e3)} seconds (idle timeout). Skills: ${skillNames}, Scenario: ${scenario.name}`
+          )
+        );
+      }
+    }, IDLE_CHECK_INTERVAL_MS);
     if (traceContext) {
-      eventStreamAbort = new AbortController();
       const executionStartTime = Date.now();
-      (async () => {
-        try {
-          const events = await client.event.subscribe();
-          for await (const event of events.stream) {
-            if (eventStreamAbort.signal.aborted) break;
-            const evt = event;
-            if (evt.type === "message.part.updated") {
-              const { part } = evt.properties;
-              traceStepNumber++;
-              const traceEvent = createTraceEventFromPart(
-                part,
-                traceContext,
-                traceStepNumber,
-                false
-              );
-              if (traceEvent) {
-                lastToolName = traceEvent.toolName;
-                lastFilePath = traceEvent.filePath;
-                if (traceEvent.type === import_evalforge_types8.LiveTraceEventType.THINKING) {
-                  lastAction = "Thinking...";
-                } else if (traceEvent.type === import_evalforge_types8.LiveTraceEventType.TOOL_USE) {
-                  lastAction = extractToolAction(
-                    traceEvent.toolName ?? "",
-                    void 0
-                  );
-                } else if (traceEvent.type === import_evalforge_types8.LiveTraceEventType.FILE_WRITE) {
-                  lastAction = `Writing: ${traceEvent.filePath || "file"}`;
-                } else if (traceEvent.type === import_evalforge_types8.LiveTraceEventType.FILE_READ) {
-                  lastAction = `Reading: ${traceEvent.filePath || "file"}`;
-                } else if (traceEvent.type === import_evalforge_types8.LiveTraceEventType.COMPLETION) {
-                  lastAction = "Processing response...";
-                }
-                emitTraceEvent(
-                  traceEvent,
-                  traceContext.tracePushUrl,
-                  traceContext.routeHeader,
-                  traceContext.authToken
-                );
-              }
-            } else if (evt.type === "session.error") {
-              const props = evt.properties;
-              traceStepNumber++;
-              emitTraceEvent(
-                {
-                  evalRunId: traceContext.evalRunId,
-                  scenarioId: traceContext.scenarioId,
-                  scenarioName: traceContext.scenarioName,
-                  targetId: traceContext.targetId,
-                  targetName: traceContext.targetName,
-                  stepNumber: traceStepNumber,
-                  type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
-                  outputPreview: `Session error: ${JSON.stringify(props.error)}`.slice(
-                    0,
-                    500
-                  ),
-                  timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-                  isComplete: false
-                },
-                traceContext.tracePushUrl,
-                traceContext.routeHeader,
-                traceContext.authToken
-              );
-            }
-          }
-        } catch {
-        }
-      })();
       let lastReportedAction = "";
       let sameActionCount = 0;
-      heartbeatHandle = setInterval(() => {
+      timers.heartbeat = setInterval(() => {
         const elapsedMs = Date.now() - executionStartTime;
         let progressMessage = lastAction;
         if (lastAction === lastReportedAction) {
@@ -2966,212 +3105,83 @@ async function executeWithOpenCode(skills, scenario, options) {
         );
       }, 1e4);
     }
-    const promptPromise = (async () => {
-      let systemPrompt;
-      if (options.systemPrompt === null || options.systemPrompt === "") {
-      } else if (options.systemPrompt != null) {
-        systemPrompt = options.systemPrompt;
-      } else {
-        systemPrompt = import_evalforge_types8.DEFAULT_EVALUATOR_SYSTEM_PROMPT;
-      }
-      console.log("[SDK-DEBUG] Sending prompt...");
-      const result = await client.session.prompt({
-        path: { id: sessionId },
-        body: {
-          model: { providerID, modelID },
-          ...systemPrompt ? { system: systemPrompt } : {},
-          parts: [{ type: "text", text: scenario.triggerPrompt }]
-        }
-      });
-      return result;
-    })();
-    const timeoutPromise = new Promise((_, reject) => {
-      timeoutHandle = setTimeout(() => {
-        timedOut = true;
-        client.session.abort({ path: { id: sessionId } }).catch(() => {
-        });
-        reject(
-          new Error(
-            `OpenCode execution timed out after ${SDK_TIMEOUT_MS}ms. Skills: ${skillNames}, Scenario: ${scenario.name}, MaxTurns: ${maxTurns}`
-          )
-        );
-      }, SDK_TIMEOUT_MS);
-    });
-    const promptResult = await Promise.race([promptPromise, timeoutPromise]);
-    if (timeoutHandle) clearTimeout(timeoutHandle);
-    if (heartbeatHandle) clearInterval(heartbeatHandle);
-    if (eventStreamAbort) eventStreamAbort.abort();
-    if ("error" in promptResult && promptResult.error) {
-      const errPayload = promptResult.error;
-      throw new Error(
-        `Agent prompt failed: ${errPayload.name ?? "UnknownError"} - ${JSON.stringify(errPayload.data ?? errPayload)}`
-      );
-    }
-    console.log("[executeWithOpenCode] Prompt completed, fetching messages...");
-    const messagesResponse = await client.session.messages({
-      path: { id: sessionId }
-    });
-    const allMessages = messagesResponse.data ?? [];
-    console.log(
-      `[executeWithOpenCode] Got ${allMessages.length} message(s) from history`
-    );
-    if (traceContext) {
-      emitTraceEvent(
-        {
-          evalRunId: traceContext.evalRunId,
-          scenarioId: traceContext.scenarioId,
-          scenarioName: traceContext.scenarioName,
-          targetId: traceContext.targetId,
-          targetName: traceContext.targetName,
-          stepNumber: traceStepNumber + 1,
-          type: import_evalforge_types8.LiveTraceEventType.COMPLETION,
-          outputPreview: "Scenario execution completed",
-          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-          isComplete: true
-        },
-        traceContext.tracePushUrl,
-        traceContext.routeHeader,
-        traceContext.authToken
-      );
-    }
-    const endTime = /* @__PURE__ */ new Date();
-    const totalDurationMs = endTime.getTime() - startTime.getTime();
-    const resultData = promptResult.data;
-    const lastAssistantInfo = resultData?.info;
-    if (lastAssistantInfo?.error) {
-      const err = lastAssistantInfo.error;
-      throw new Error(
-        `Agent execution failed: ${err.name} - ${JSON.stringify(err.data)}`
-      );
-    }
-    let outputText = "";
-    if (resultData?.parts) {
-      for (const part of resultData.parts) {
-        if (part.type === "text") {
-          outputText += part.text;
-        }
-      }
-    }
-    if (!outputText && allMessages.length > 0) {
-      for (let i = allMessages.length - 1; i >= 0; i--) {
-        const msg = allMessages[i];
-        if (msg.info.role === "assistant") {
-          const assistantInfo = msg.info;
-          if (assistantInfo.error) {
-            throw new Error(
-              `Agent execution failed: ${assistantInfo.error.name} - ${JSON.stringify(assistantInfo.error.data)}`
-            );
-          }
-          for (const part of msg.parts) {
-            if (part.type === "text") {
-              outputText += part.text;
+    child.stdout?.on("data", (data) => {
+      const text = data.toString();
+      lastOutputTime = Date.now();
+      lineBuffer += text;
+      const lines = lineBuffer.split("\n");
+      lineBuffer = lines.pop() || "";
+      for (const line of lines) {
+        if (!line.trim()) continue;
+        const evt = tryParseJson(line);
+        if (!evt || !evt.type) continue;
+        allEvents.push({ event: evt, receivedAt: Date.now() });
+        if (traceContext) {
+          traceStepNumber++;
+          const traceEvt = createTraceEventFromNdjson(
+            evt,
+            traceContext,
+            traceStepNumber,
+            false
+          );
+          if (traceEvt) {
+            lastToolName = traceEvt.toolName;
+            lastFilePath = traceEvt.filePath;
+            if (traceEvt.type === import_evalforge_types8.LiveTraceEventType.THINKING) {
+              lastAction = "Thinking...";
+            } else if (traceEvt.type === import_evalforge_types8.LiveTraceEventType.TOOL_USE) {
+              lastAction = extractToolAction(
+                traceEvt.toolName ?? "",
+                void 0
+              );
+            } else if (traceEvt.type === import_evalforge_types8.LiveTraceEventType.FILE_WRITE) {
+              lastAction = `Writing: ${traceEvt.filePath || "file"}`;
+            } else if (traceEvt.type === import_evalforge_types8.LiveTraceEventType.FILE_READ) {
+              lastAction = `Reading: ${traceEvt.filePath || "file"}`;
+            } else if (traceEvt.type === import_evalforge_types8.LiveTraceEventType.COMPLETION) {
+              lastAction = "Processing response...";
             }
+            emitTraceEvent(
+              traceEvt,
+              traceContext.tracePushUrl,
+              traceContext.routeHeader,
+              traceContext.authToken
+            );
           }
-          if (outputText) break;
         }
       }
-    }
-    if (!outputText) {
-      const hasAssistant = allMessages.some((m) => m.info.role === "assistant");
-      if (!hasAssistant) {
-        throw new Error(
-          `Agent produced no response: no assistant messages in session history. Model: ${providerID}/${modelID}, Messages: ${allMessages.length}`
-        );
-      }
-    }
-    const usage = lastAssistantInfo ? {
-      inputTokens: lastAssistantInfo.tokens.input,
-      outputTokens: lastAssistantInfo.tokens.output,
-      totalTokens: lastAssistantInfo.tokens.input + lastAssistantInfo.tokens.output
-    } : { inputTokens: 0, outputTokens: 0, totalTokens: 0 };
-    const costUsd = lastAssistantInfo?.cost;
-    const modelStr = options.model || DEFAULT_MODEL3;
-    const llmTrace = buildLLMTrace(
-      allMessages,
-      totalDurationMs,
-      modelStr,
-      providerID
-    );
-    const conversation = buildConversation2(allMessages);
-    return {
-      result: {
-        outputText,
-        durationMs: totalDurationMs,
-        usage,
-        costUsd
-      },
-      llmTrace,
-      conversation
-    };
-  } catch (sdkError) {
-    if (timeoutHandle) clearTimeout(timeoutHandle);
-    if (heartbeatHandle) clearInterval(heartbeatHandle);
-    if (timedOut) {
-      console.error("[SDK-TIMEOUT] Execution timed out:", sdkError);
-    }
-    const errorMessage = sdkError instanceof Error ? sdkError.message : String(sdkError);
-    const errorStack = sdkError instanceof Error ? sdkError.stack : void 0;
-    const errorName = sdkError instanceof Error ? sdkError.name : "Unknown";
-    const causeDetails = [];
-    let current = sdkError;
-    while (current instanceof Error && current.cause) {
-      current = current.cause;
-      if (current instanceof Error) {
-        causeDetails.push(`${current.name}: ${current.message}`);
-      } else {
-        causeDetails.push(String(current));
+    });
+    child.stderr?.on("data", (data) => {
+      const text = data.toString();
+      stderr += text;
+      lastOutputTime = Date.now();
+    });
+    child.on("close", (code) => {
+      if (lineBuffer.trim()) {
+        const evt = tryParseJson(lineBuffer);
+        if (evt && evt.type) {
+          allEvents.push({ event: evt, receivedAt: Date.now() });
+        }
       }
-    }
-    const causeChain = causeDetails.length > 0 ? `
-Cause chain: ${causeDetails.join(" -> ")}` : "";
-    console.error("[SDK-ERROR] ====== OPENCODE SDK EXECUTION FAILED ======");
-    console.error("[SDK-ERROR] Error name:", errorName);
-    console.error("[SDK-ERROR] Error message:", errorMessage);
-    if (causeDetails.length > 0) {
-      console.error("[SDK-ERROR] Cause chain:", causeDetails.join(" -> "));
-    }
-    if (errorStack) {
-      console.error("[SDK-ERROR] Stack:", errorStack);
-    }
-    if (traceContext) {
-      emitTraceEvent(
-        {
-          evalRunId: traceContext.evalRunId,
-          scenarioId: traceContext.scenarioId,
-          scenarioName: traceContext.scenarioName,
-          targetId: traceContext.targetId,
-          targetName: traceContext.targetName,
-          stepNumber: traceStepNumber + 1,
-          type: import_evalforge_types8.LiveTraceEventType.DIAGNOSTIC,
-          outputPreview: JSON.stringify({
-            event: "sdk-execution-failed",
-            error: errorMessage,
-            errorName,
-            ...causeDetails.length > 0 && {
-              causeChain: causeDetails.join(" -> ")
-            }
-          }).slice(0, 2e3),
-          timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-          isComplete: true
-        },
-        traceContext.tracePushUrl,
-        traceContext.routeHeader,
-        traceContext.authToken
+      console.log(
+        `[executeWithOpenCode] Process exited with code ${code}, ${allEvents.length} events collected`
       );
-    }
-    throw new Error(
-      `OpenCode SDK execution failed: ${errorMessage}` + causeChain + (errorStack ? `
-Stack: ${errorStack.split("\n").slice(0, 5).join("\n")}` : "")
-    );
-  } finally {
-    if (server) {
-      try {
-        server.close();
-        console.log("[SDK-DEBUG] OpenCode server closed");
-      } catch {
+      if (code === 0) {
+        finalize(true);
+      } else {
+        finalize(
+          false,
+          new Error(
+            `OpenCode CLI exited with code ${code}.
+Stderr: ${stderr.slice(0, 1e3)}`
+          )
+        );
       }
-    }
-  }
+    });
+    child.on("error", (error) => {
+      finalize(false, new Error(`OpenCode CLI spawn error: ${error.message}`));
+    });
+  });
 }
 
 // src/run-scenario/agents/opencode/opencode-adapter.ts
@@ -3757,7 +3767,7 @@ defaultRegistry.register(simpleAgentAdapter);
 
 // src/run-scenario/file-diff.ts
 var import_fs2 = require("fs");
-var import_path10 = require("path");
+var import_path11 = require("path");
 
 // ../../node_modules/diff/lib/index.mjs
 function Diff() {
@@ -3933,7 +3943,7 @@ Diff.prototype = {
   tokenize: function tokenize(value) {
     return Array.from(value);
   },
-  join: function join8(chars) {
+  join: function join9(chars) {
     return chars.join("");
   },
   postProcess: function postProcess(changeObjects) {
@@ -4382,8 +4392,8 @@ function snapshotDirectory(dir, baseDir) {
   }
   const entries = (0, import_fs2.readdirSync)(dir, { withFileTypes: true });
   for (const entry of entries) {
-    const fullPath = (0, import_path10.join)(dir, entry.name);
-    const relativePath = (0, import_path10.relative)(base, fullPath);
+    const fullPath = (0, import_path11.join)(dir, entry.name);
+    const relativePath = (0, import_path11.relative)(base, fullPath);
     if (shouldIgnore(entry.name)) {
       continue;
     }