@wix/evalforge-evaluator 0.147.0 → 0.148.0

package/build/index.mjs

@@ -315,10 +315,10 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
   if (evalRun.agentId) {
     agent = await api.getAgent(projectId2, evalRun.agentId);
   }
-  let skills = [];
-  let mcps = [];
-  let subAgents = [];
-  let rules = [];
+  const skills = [];
+  const mcps = [];
+  const subAgents = [];
+  const rules = [];
   if (evalRun.capabilityIds && evalRun.capabilityIds.length > 0) {
     const fetchResults = await Promise.allSettled(
       evalRun.capabilityIds.map((id) => api.getCapability(projectId2, id))
@@ -605,7 +605,7 @@ function getAdapter(identifier) {
 }
 
 // src/run-scenario/agents/claude-code/claude-code-adapter.ts
-import { AgentRunCommand } from "@wix/evalforge-types";
+import { AgentRunCommand, ClaudeCodeConfigSchema } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/claude-code/execute.ts
 import {
@@ -1187,10 +1187,10 @@ async function executeWithClaudeCode(skills, scenario, options) {
   let lastAction = "Starting...";
   let lastToolName;
   let lastFilePath;
-  const maxTurns = options.maxTurns ?? 10;
+  const maxTurns = options.maxTurns || void 0;
   let messageCount = 0;
   const canUseTool = async (_toolName, input) => ({ behavior: "allow", updatedInput: input });
-  const baseAllowedTools = [
+  const baseAllowedTools = options.allowedTools ?? [
     "Skill",
     "Read",
     "Write",
@@ -1204,13 +1204,17 @@ async function executeWithClaudeCode(skills, scenario, options) {
   ];
   const allowedTools = (options.mcps?.length ?? 0) > 0 ? [...baseAllowedTools, "mcp__*"] : baseAllowedTools;
   const queryOptions = {
+    ...options.extras ?? {},
     env: sdkEnv,
     cwd: options.cwd,
     settingSources: ["project"],
     allowedTools,
+    ...options.disallowedTools?.length ? { disallowedTools: options.disallowedTools } : {},
     model: options.model || DEFAULT_MODEL,
     maxTurns,
     maxThinkingTokens: options.maxThinkingTokens,
+    ...options.effort ? { effort: options.effort } : {},
+    ...options.maxBudgetUsd != null ? { maxBudgetUsd: options.maxBudgetUsd } : {},
     // Use 'default' permission mode with custom canUseTool handler
     // instead of 'bypassPermissions' which fails on root
     permissionMode: "default",
@@ -1285,7 +1289,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
-  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
+  const SDK_TIMEOUT_MS = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
   let timeoutHandle;
   let timedOut = false;
   const HEARTBEAT_INTERVAL_MS = 1e4;
@@ -2049,7 +2053,7 @@ var ClaudeCodeAdapter = class {
       skills,
       scenario,
       cwd,
-      modelConfig,
+      config,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -2058,20 +2062,37 @@ var ClaudeCodeAdapter = class {
       rules,
       systemPrompt
     } = context;
-    const modelForSdk = modelConfig?.model;
+    const typed = config ? ClaudeCodeConfigSchema.passthrough().safeParse(config) : void 0;
+    const cfg = typed?.success ? typed.data : void 0;
+    const schemaKeys = new Set(Object.keys(ClaudeCodeConfigSchema.shape));
+    const extras = {};
+    if (config) {
+      for (const [key, value] of Object.entries(config)) {
+        if (!schemaKeys.has(key)) extras[key] = value;
+      }
+    }
+    const rawMaxTurns = cfg?.maxTurns;
+    const maxTurns = rawMaxTurns === 0 ? void 0 : rawMaxTurns;
     const options = {
       cwd,
-      model: modelForSdk,
-      temperature: modelConfig?.temperature,
-      maxTokens: modelConfig?.maxTokens,
-      maxTurns: modelConfig?.maxTurns,
+      model: cfg?.model,
+      temperature: cfg?.temperature,
+      maxTokens: cfg?.maxTokens,
+      maxTurns,
+      maxThinkingTokens: cfg?.maxThinkingTokens,
+      allowedTools: cfg?.allowedTools,
+      disallowedTools: cfg?.disallowedTools,
+      effort: cfg?.effort,
+      maxBudgetUsd: cfg?.maxBudgetUsd,
+      maxDurationMs: cfg?.maxDurationMs,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
       mcps,
       subAgents,
       rules,
-      systemPrompt
+      systemPrompt,
+      extras
     };
     const { result, llmTrace, conversation } = await executeWithClaudeCode(
       skills,
@@ -2098,7 +2119,7 @@ var claudeCodeAdapter = new ClaudeCodeAdapter();
 defaultRegistry.register(claudeCodeAdapter);
 
 // src/run-scenario/agents/opencode/opencode-adapter.ts
-import { AgentRunCommand as AgentRunCommand2 } from "@wix/evalforge-types";
+import { AgentRunCommand as AgentRunCommand2, OpenCodeConfigSchema as OpenCodeConfigSchema2 } from "@wix/evalforge-types";
 
 // src/run-scenario/agents/opencode/execute.ts
 import { spawn } from "child_process";
@@ -2217,25 +2238,33 @@ import { homedir as homedir2 } from "os";
 import {
   ClaudeModel as ClaudeModel2,
   AVAILABLE_OPENAI_MODEL_IDS,
-  AVAILABLE_GEMINI_MODEL_IDS
+  AVAILABLE_GEMINI_MODEL_IDS,
+  OpenCodeConfigSchema
 } from "@wix/evalforge-types";
 var DEFAULT_MODEL2 = `${ClaudeModel2.CLAUDE_4_5_SONNET_1_0}`;
+var OPENCODE_MODEL_ALIASES = {
+  "claude-sonnet-4": "claude-sonnet-4-0",
+  "claude-opus-4": "claude-opus-4-0"
+};
 function parseModel(model) {
   const slashIndex = model.indexOf("/");
   if (slashIndex > 0) {
+    const providerID = model.slice(0, slashIndex);
+    const rawModelID = model.slice(slashIndex + 1);
     return {
-      providerID: model.slice(0, slashIndex),
-      modelID: model.slice(slashIndex + 1)
+      providerID,
+      modelID: OPENCODE_MODEL_ALIASES[rawModelID] ?? rawModelID
     };
   }
+  const modelID = OPENCODE_MODEL_ALIASES[model] ?? model;
   const isOpenAI = AVAILABLE_OPENAI_MODEL_IDS.includes(
     model
   );
   const isGemini = AVAILABLE_GEMINI_MODEL_IDS.includes(
     model
   );
-  if (isGemini) return { providerID: "google", modelID: model };
-  return { providerID: isOpenAI ? "openai" : "anthropic", modelID: model };
+  if (isGemini) return { providerID: "google", modelID };
+  return { providerID: isOpenAI ? "openai" : "anthropic", modelID };
 }
 function toOpenCodeMcpConfig(servers) {
   const result = {};
@@ -2319,20 +2348,26 @@ async function buildOpenCodeEnv(options) {
   if (options.temperature != null) {
     agentOverrides.temperature = options.temperature;
   }
-  if (options.maxTurns != null) {
+  if (options.topP != null) {
+    agentOverrides.top_p = options.topP;
+  }
+  if (options.maxTurns != null && options.maxTurns > 0) {
     agentOverrides.maxSteps = options.maxTurns;
   }
+  const parsed = options.config ? OpenCodeConfigSchema.passthrough().safeParse(options.config) : void 0;
+  const configPermission = parsed?.success ? parsed.data.permission : void 0;
+  const defaultPermission = {
+    "*": "allow"
+  };
+  const permission = {
+    ...defaultPermission,
+    ...configPermission
+  };
   const config = {
     model: `${providerID}/${modelID}`,
     provider,
     ...Object.keys(agentOverrides).length > 0 ? { agent: { build: agentOverrides } } : {},
-    permission: {
-      edit: "allow",
-      bash: "allow",
-      webfetch: "allow",
-      doom_loop: "allow",
-      external_directory: "allow"
-    },
+    permission,
     ...mcp ? { mcp } : {}
   };
   const env = {
@@ -3066,16 +3101,18 @@ async function executeWithOpenCode(skills, scenario, options) {
     hasAiGatewayHeaders: !!options.aiGatewayHeaders,
     model: options.model
   });
-  const maxTurns = options.maxTurns ?? 10;
-  const sdkTimeoutMs = Math.max(3e5, maxTurns * 6e4);
+  const maxTurns = options.maxTurns || void 0;
+  const sdkTimeoutMs = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
   const { env, providerID, modelID } = await buildOpenCodeEnv({
     model: options.model,
     temperature: options.temperature,
+    topP: options.topP,
     maxTurns,
     aiGatewayUrl: options.aiGatewayUrl,
     aiGatewayHeaders: options.aiGatewayHeaders,
     mcps: options.mcps,
-    cwd: options.cwd
+    cwd: options.cwd,
+    config: options.config
   });
   const startTime = /* @__PURE__ */ new Date();
   const traceContext = options.traceContext;
@@ -3103,13 +3140,13 @@ async function executeWithOpenCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
+  const variant = options.thinkingVariant ?? "high";
+  const thinkingArgs = variant === "none" ? [] : ["--thinking", "--variant", variant];
   const baseArgs = [
     "run",
     "--format",
     "json",
-    "--thinking",
-    "--variant",
-    "high",
+    ...thinkingArgs,
     "--model",
     `${providerID}/${modelID}`,
     "--dir",
@@ -3300,7 +3337,7 @@ var OpenCodeAdapter = class {
       skills,
       scenario,
       cwd,
-      modelConfig,
+      config,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -3309,18 +3346,26 @@ var OpenCodeAdapter = class {
       rules,
       systemPrompt
     } = context;
+    const typed = config ? OpenCodeConfigSchema2.passthrough().safeParse(config) : void 0;
+    const cfg = typed?.success ? typed.data : void 0;
+    const rawMaxTurns = cfg?.maxTurns;
+    const maxTurns = rawMaxTurns === 0 ? void 0 : rawMaxTurns;
     const options = {
       cwd,
-      model: modelConfig?.model,
-      temperature: modelConfig?.temperature,
-      maxTurns: modelConfig?.maxTurns,
+      model: cfg?.model,
+      temperature: cfg?.temperature,
+      topP: cfg?.topP,
+      maxTurns,
+      thinkingVariant: cfg?.thinkingVariant,
+      maxDurationMs: cfg?.maxDurationMs,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
       mcps,
       subAgents,
       rules,
-      systemPrompt
+      systemPrompt,
+      config
     };
     const { result, llmTrace, conversation } = await executeWithOpenCode(
       skills,
@@ -3360,7 +3405,8 @@ import {
   GEMINI_THINKING_MODEL_IDS,
   OPENAI_RESPONSES_MODEL_IDS,
   LLMStepType as LLMStepType3,
-  LiveTraceEventType as LiveTraceEventType3
+  LiveTraceEventType as LiveTraceEventType3,
+  SimpleAgentConfigSchema
 } from "@wix/evalforge-types";
 import { randomUUID as randomUUID3 } from "crypto";
 
@@ -3646,20 +3692,30 @@ async function executeWithAiSdk(context) {
   const {
     scenario,
     cwd,
-    modelConfig,
+    config,
     aiGatewayUrl,
     aiGatewayHeaders = {},
     mcps,
     traceContext
   } = context;
+  const typed = config ? SimpleAgentConfigSchema.passthrough().safeParse(config) : void 0;
+  const cfg = typed?.success ? typed.data : void 0;
+  const schemaKeys = new Set(Object.keys(SimpleAgentConfigSchema.shape));
+  const configExtras = {};
+  if (config) {
+    for (const [key, value] of Object.entries(config)) {
+      if (!schemaKeys.has(key)) configExtras[key] = value;
+    }
+  }
   if (!aiGatewayUrl) {
     throw new Error("Simple Agent requires aiGatewayUrl");
   }
-  if (!modelConfig?.model) {
-    throw new Error("Simple Agent requires a model in modelConfig");
+  if (!cfg?.model) {
+    throw new Error("Simple Agent requires a model in config");
   }
-  const model = createModel(modelConfig.model, aiGatewayUrl, aiGatewayHeaders);
-  const provider = isClaudeModelId(modelConfig.model) ? PROVIDER_ANTHROPIC2 : isGeminiModelId(modelConfig.model) ? PROVIDER_GEMINI2 : PROVIDER_OPENAI;
+  const modelId = cfg.model;
+  const model = createModel(modelId, aiGatewayUrl, aiGatewayHeaders);
+  const provider = isClaudeModelId(modelId) ? PROVIDER_ANTHROPIC2 : isGeminiModelId(modelId) ? PROVIDER_GEMINI2 : PROVIDER_OPENAI;
   const systemPrompt = composeSystemPrompt(context);
   const { tools: mcpTools, clients } = mcps && mcps.length > 0 ? await buildMcpTools(mcps, cwd) : { tools: void 0, clients: [] };
   const startTime = Date.now();
@@ -3669,15 +3725,17 @@ async function executeWithAiSdk(context) {
   try {
     const isAnthropic = provider === PROVIDER_ANTHROPIC2;
     const isResponsesAPI = [...OPENAI_RESPONSES_MODEL_IDS].some(
-      (id) => modelConfig.model === id || modelConfig.model.startsWith(id)
+      (id) => modelId === id || modelId.startsWith(id)
     );
     const isGemini = provider === PROVIDER_GEMINI2;
-    const isGeminiThinking = isGemini && GEMINI_THINKING_MODEL_IDS.has(modelConfig.model);
+    const isGeminiThinking = isGemini && GEMINI_THINKING_MODEL_IDS.has(modelId);
     const supportsThinking = isAnthropic || isResponsesAPI || isGeminiThinking;
-    const providerOpts = {
+    const thinkingBudgetTokens = cfg.thinkingBudgetTokens ?? 1e4;
+    const reasoningEffort = cfg.reasoningEffort ?? "high";
+    const computedProviderOpts = {
       ...isAnthropic && {
         anthropic: {
-          thinking: { type: "enabled", budgetTokens: 1e4 }
+          thinking: { type: "enabled", budgetTokens: thinkingBudgetTokens }
         }
       },
       ...isResponsesAPI && {
@@ -3685,7 +3743,7 @@ async function executeWithAiSdk(context) {
           // Prevent the SDK from sending item_reference inputs that the proxy can't forward
           store: false,
           forceReasoning: true,
-          reasoningEffort: "high",
+          reasoningEffort,
           reasoningSummary: "detailed"
         }
       },
@@ -3698,10 +3756,17 @@ async function executeWithAiSdk(context) {
         }
       }
     };
+    const { providerOptions: extraProviderOptions, ...topLevelExtras } = configExtras;
+    const mergedProviderOptions = {
+      ...extraProviderOptions && typeof extraProviderOptions === "object" ? extraProviderOptions : {},
+      ...computedProviderOpts
+    };
     const stepTimestamps = [];
+    const effectiveMaxTurns = cfg.maxTurns === 0 ? void 0 : cfg.maxTurns ?? DEFAULT_MAX_TOOL_STEPS;
     const { triggerPromptImages } = context;
     const hasImages = triggerPromptImages && triggerPromptImages.length > 0;
-    const generateTextParams = {
+    const result = await generateText({
+      ...topLevelExtras,
       model,
       system: systemPrompt,
       ...hasImages ? {
@@ -3719,11 +3784,16 @@ async function executeWithAiSdk(context) {
           }
         ]
       } : { prompt: scenario.triggerPrompt },
-      temperature: supportsThinking ? void 0 : modelConfig.temperature,
-      maxOutputTokens: modelConfig.maxTokens,
+      temperature: supportsThinking ? void 0 : cfg.temperature,
+      topP: supportsThinking ? void 0 : cfg.topP,
+      frequencyPenalty: cfg.frequencyPenalty,
+      presencePenalty: cfg.presencePenalty,
+      seed: cfg.seed,
+      stopSequences: cfg.stopSequences,
+      maxOutputTokens: cfg.maxTokens,
       tools: mcpTools,
-      stopWhen: mcpTools ? stepCountIs(modelConfig.maxTurns ?? DEFAULT_MAX_TOOL_STEPS) : stepCountIs(1),
-      providerOptions: providerOpts,
+      ...mcpTools && effectiveMaxTurns != null ? { stopWhen: stepCountIs(effectiveMaxTurns) } : !mcpTools ? { stopWhen: stepCountIs(1) } : {},
+      providerOptions: mergedProviderOptions,
       onStepFinish: (step) => {
         stepTimestamps.push(Date.now());
         if (traceContext) {
@@ -3751,8 +3821,7 @@ async function executeWithAiSdk(context) {
           );
         }
       }
-    };
-    const result = await generateText(generateTextParams);
+    });
     const durationMs = Date.now() - startTime;
     const usage = {
       inputTokens: result.usage.inputTokens ?? 0,
@@ -3763,7 +3832,7 @@ async function executeWithAiSdk(context) {
       result.steps,
       durationMs,
       usage,
-      modelConfig.model,
+      modelId,
       provider,
       startTime,
       stepTimestamps
@@ -4676,11 +4745,12 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
   const startedAt = (/* @__PURE__ */ new Date()).toISOString();
   const targetId = evalData.evalRun.presetId ?? agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName || agent?.name || "";
+  const agentConfig = agent?.config;
   const executionContext = {
     skills: evalData.skills,
     scenario,
     cwd: workDir || process.cwd(),
-    modelConfig: agent?.modelConfig,
+    config: agentConfig,
     aiGatewayUrl: config.aiGatewayUrl,
     aiGatewayHeaders: config.aiGatewayHeaders,
     traceContext: {
@@ -4715,7 +4785,18 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     infrastructurePaths
   );
   const templateFiles = workDir ? extractTemplateFiles(beforeSnapshot, afterSnapshot, infrastructurePaths) : void 0;
-  const resolvedModelConfig = agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
+  const snapshotModelConfig = agentConfig?.model ? {
+    model: agentConfig.model,
+    ...agentConfig.temperature != null && {
+      temperature: agentConfig.temperature
+    },
+    ...agentConfig.maxTokens != null && {
+      maxTokens: agentConfig.maxTokens
+    },
+    ...agentConfig.maxTurns != null && {
+      maxTurns: agentConfig.maxTurns
+    }
+  } : agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
   return {
     id: randomUUID4(),
     targetId,
@@ -4723,7 +4804,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     scenarioId: scenario.id,
     scenarioName: scenario.name,
     triggerPrompt: scenario.triggerPrompt,
-    modelConfig: resolvedModelConfig,
+    modelConfig: snapshotModelConfig,
     duration: durationMs,
     outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,