@wix/evalforge-evaluator 0.147.0 → 0.149.0

package/build/index.js

@@ -330,10 +330,10 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
   if (evalRun.agentId) {
     agent = await api.getAgent(projectId2, evalRun.agentId);
   }
-  let skills = [];
-  let mcps = [];
-  let subAgents = [];
-  let rules = [];
+  const skills = [];
+  const mcps = [];
+  const subAgents = [];
+  const rules = [];
   if (evalRun.capabilityIds && evalRun.capabilityIds.length > 0) {
     const fetchResults = await Promise.allSettled(
       evalRun.capabilityIds.map((id) => api.getCapability(projectId2, id))
@@ -1190,10 +1190,10 @@ async function executeWithClaudeCode(skills, scenario, options) {
   let lastAction = "Starting...";
   let lastToolName;
   let lastFilePath;
-  const maxTurns = options.maxTurns ?? 10;
+  const maxTurns = options.maxTurns || void 0;
   let messageCount = 0;
   const canUseTool = async (_toolName, input) => ({ behavior: "allow", updatedInput: input });
-  const baseAllowedTools = [
+  const baseAllowedTools = options.allowedTools ?? [
     "Skill",
     "Read",
     "Write",
@@ -1207,13 +1207,17 @@ async function executeWithClaudeCode(skills, scenario, options) {
   ];
   const allowedTools = (options.mcps?.length ?? 0) > 0 ? [...baseAllowedTools, "mcp__*"] : baseAllowedTools;
   const queryOptions = {
+    ...options.extras ?? {},
     env: sdkEnv,
     cwd: options.cwd,
     settingSources: ["project"],
     allowedTools,
+    ...options.disallowedTools?.length ? { disallowedTools: options.disallowedTools } : {},
     model: options.model || DEFAULT_MODEL,
     maxTurns,
     maxThinkingTokens: options.maxThinkingTokens,
+    ...options.effort ? { effort: options.effort } : {},
+    ...options.maxBudgetUsd != null ? { maxBudgetUsd: options.maxBudgetUsd } : {},
     // Use 'default' permission mode with custom canUseTool handler
     // instead of 'bypassPermissions' which fails on root
     permissionMode: "default",
@@ -1288,7 +1292,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
-  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
+  const SDK_TIMEOUT_MS = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
   let timeoutHandle;
   let timedOut = false;
   const HEARTBEAT_INTERVAL_MS = 1e4;
@@ -2052,7 +2056,7 @@ var ClaudeCodeAdapter = class {
       skills,
       scenario,
       cwd,
-      modelConfig,
+      config,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -2061,20 +2065,37 @@ var ClaudeCodeAdapter = class {
       rules,
       systemPrompt
     } = context;
-    const modelForSdk = modelConfig?.model;
+    const typed = config ? import_evalforge_types5.ClaudeCodeConfigSchema.passthrough().safeParse(config) : void 0;
+    const cfg = typed?.success ? typed.data : void 0;
+    const schemaKeys = new Set(Object.keys(import_evalforge_types5.ClaudeCodeConfigSchema.shape));
+    const extras = {};
+    if (config) {
+      for (const [key, value] of Object.entries(config)) {
+        if (!schemaKeys.has(key)) extras[key] = value;
+      }
+    }
+    const rawMaxTurns = cfg?.maxTurns;
+    const maxTurns = rawMaxTurns === 0 ? void 0 : rawMaxTurns;
     const options = {
       cwd,
-      model: modelForSdk,
-      temperature: modelConfig?.temperature,
-      maxTokens: modelConfig?.maxTokens,
-      maxTurns: modelConfig?.maxTurns,
+      model: cfg?.model,
+      temperature: cfg?.temperature,
+      maxTokens: cfg?.maxTokens,
+      maxTurns,
+      maxThinkingTokens: cfg?.maxThinkingTokens,
+      allowedTools: cfg?.allowedTools,
+      disallowedTools: cfg?.disallowedTools,
+      effort: cfg?.effort,
+      maxBudgetUsd: cfg?.maxBudgetUsd,
+      maxDurationMs: cfg?.maxDurationMs,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
       mcps,
       subAgents,
       rules,
-      systemPrompt
+      systemPrompt,
+      extras
     };
     const { result, llmTrace, conversation } = await executeWithClaudeCode(
       skills,
@@ -2214,22 +2235,29 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = import_eval
 var import_os3 = require("os");
 var import_evalforge_types6 = require("@wix/evalforge-types");
 var DEFAULT_MODEL2 = `${import_evalforge_types6.ClaudeModel.CLAUDE_4_5_SONNET_1_0}`;
+var OPENCODE_MODEL_ALIASES = {
+  "claude-sonnet-4": "claude-sonnet-4-0",
+  "claude-opus-4": "claude-opus-4-0"
+};
 function parseModel(model) {
   const slashIndex = model.indexOf("/");
   if (slashIndex > 0) {
+    const providerID = model.slice(0, slashIndex);
+    const rawModelID = model.slice(slashIndex + 1);
     return {
-      providerID: model.slice(0, slashIndex),
-      modelID: model.slice(slashIndex + 1)
+      providerID,
+      modelID: OPENCODE_MODEL_ALIASES[rawModelID] ?? rawModelID
     };
   }
+  const modelID = OPENCODE_MODEL_ALIASES[model] ?? model;
   const isOpenAI = import_evalforge_types6.AVAILABLE_OPENAI_MODEL_IDS.includes(
     model
   );
   const isGemini = import_evalforge_types6.AVAILABLE_GEMINI_MODEL_IDS.includes(
     model
   );
-  if (isGemini) return { providerID: "google", modelID: model };
-  return { providerID: isOpenAI ? "openai" : "anthropic", modelID: model };
+  if (isGemini) return { providerID: "google", modelID };
+  return { providerID: isOpenAI ? "openai" : "anthropic", modelID };
 }
 function toOpenCodeMcpConfig(servers) {
   const result = {};
@@ -2313,20 +2341,26 @@ async function buildOpenCodeEnv(options) {
   if (options.temperature != null) {
     agentOverrides.temperature = options.temperature;
   }
-  if (options.maxTurns != null) {
+  if (options.topP != null) {
+    agentOverrides.top_p = options.topP;
+  }
+  if (options.maxTurns != null && options.maxTurns > 0) {
     agentOverrides.maxSteps = options.maxTurns;
   }
+  const parsed = options.config ? import_evalforge_types6.OpenCodeConfigSchema.passthrough().safeParse(options.config) : void 0;
+  const configPermission = parsed?.success ? parsed.data.permission : void 0;
+  const defaultPermission = {
+    "*": "allow"
+  };
+  const permission = {
+    ...defaultPermission,
+    ...configPermission
+  };
   const config = {
     model: `${providerID}/${modelID}`,
     provider,
     ...Object.keys(agentOverrides).length > 0 ? { agent: { build: agentOverrides } } : {},
-    permission: {
-      edit: "allow",
-      bash: "allow",
-      webfetch: "allow",
-      doom_loop: "allow",
-      external_directory: "allow"
-    },
+    permission,
     ...mcp ? { mcp } : {}
   };
   const env = {
@@ -3060,16 +3094,18 @@ async function executeWithOpenCode(skills, scenario, options) {
     hasAiGatewayHeaders: !!options.aiGatewayHeaders,
     model: options.model
   });
-  const maxTurns = options.maxTurns ?? 10;
-  const sdkTimeoutMs = Math.max(3e5, maxTurns * 6e4);
+  const maxTurns = options.maxTurns || void 0;
+  const sdkTimeoutMs = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
   const { env, providerID, modelID } = await buildOpenCodeEnv({
     model: options.model,
     temperature: options.temperature,
+    topP: options.topP,
     maxTurns,
     aiGatewayUrl: options.aiGatewayUrl,
     aiGatewayHeaders: options.aiGatewayHeaders,
     mcps: options.mcps,
-    cwd: options.cwd
+    cwd: options.cwd,
+    config: options.config
   });
   const startTime = /* @__PURE__ */ new Date();
   const traceContext = options.traceContext;
@@ -3097,13 +3133,13 @@ async function executeWithOpenCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
+  const variant = options.thinkingVariant ?? "high";
+  const thinkingArgs = variant === "none" ? [] : ["--thinking", "--variant", variant];
   const baseArgs = [
     "run",
     "--format",
     "json",
-    "--thinking",
-    "--variant",
-    "high",
+    ...thinkingArgs,
     "--model",
     `${providerID}/${modelID}`,
     "--dir",
@@ -3294,7 +3330,7 @@ var OpenCodeAdapter = class {
       skills,
       scenario,
       cwd,
-      modelConfig,
+      config,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -3303,18 +3339,26 @@ var OpenCodeAdapter = class {
       rules,
       systemPrompt
     } = context;
+    const typed = config ? import_evalforge_types9.OpenCodeConfigSchema.passthrough().safeParse(config) : void 0;
+    const cfg = typed?.success ? typed.data : void 0;
+    const rawMaxTurns = cfg?.maxTurns;
+    const maxTurns = rawMaxTurns === 0 ? void 0 : rawMaxTurns;
     const options = {
       cwd,
-      model: modelConfig?.model,
-      temperature: modelConfig?.temperature,
-      maxTurns: modelConfig?.maxTurns,
+      model: cfg?.model,
+      temperature: cfg?.temperature,
+      topP: cfg?.topP,
+      maxTurns,
+      thinkingVariant: cfg?.thinkingVariant,
+      maxDurationMs: cfg?.maxDurationMs,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
       mcps,
       subAgents,
       rules,
-      systemPrompt
+      systemPrompt,
+      config
     };
     const { result, llmTrace, conversation } = await executeWithOpenCode(
       skills,
@@ -3630,20 +3674,30 @@ async function executeWithAiSdk(context) {
   const {
     scenario,
     cwd,
-    modelConfig,
+    config,
     aiGatewayUrl,
     aiGatewayHeaders = {},
     mcps,
     traceContext
   } = context;
+  const typed = config ? import_evalforge_types11.SimpleAgentConfigSchema.passthrough().safeParse(config) : void 0;
+  const cfg = typed?.success ? typed.data : void 0;
+  const schemaKeys = new Set(Object.keys(import_evalforge_types11.SimpleAgentConfigSchema.shape));
+  const configExtras = {};
+  if (config) {
+    for (const [key, value] of Object.entries(config)) {
+      if (!schemaKeys.has(key)) configExtras[key] = value;
+    }
+  }
   if (!aiGatewayUrl) {
     throw new Error("Simple Agent requires aiGatewayUrl");
   }
-  if (!modelConfig?.model) {
-    throw new Error("Simple Agent requires a model in modelConfig");
+  if (!cfg?.model) {
+    throw new Error("Simple Agent requires a model in config");
   }
-  const model = createModel(modelConfig.model, aiGatewayUrl, aiGatewayHeaders);
-  const provider = isClaudeModelId(modelConfig.model) ? PROVIDER_ANTHROPIC2 : isGeminiModelId(modelConfig.model) ? PROVIDER_GEMINI2 : PROVIDER_OPENAI;
+  const modelId = cfg.model;
+  const model = createModel(modelId, aiGatewayUrl, aiGatewayHeaders);
+  const provider = isClaudeModelId(modelId) ? PROVIDER_ANTHROPIC2 : isGeminiModelId(modelId) ? PROVIDER_GEMINI2 : PROVIDER_OPENAI;
   const systemPrompt = composeSystemPrompt(context);
   const { tools: mcpTools, clients } = mcps && mcps.length > 0 ? await buildMcpTools(mcps, cwd) : { tools: void 0, clients: [] };
   const startTime = Date.now();
@@ -3653,15 +3707,17 @@ async function executeWithAiSdk(context) {
   try {
     const isAnthropic = provider === PROVIDER_ANTHROPIC2;
     const isResponsesAPI = [...import_evalforge_types11.OPENAI_RESPONSES_MODEL_IDS].some(
-      (id) => modelConfig.model === id || modelConfig.model.startsWith(id)
+      (id) => modelId === id || modelId.startsWith(id)
     );
     const isGemini = provider === PROVIDER_GEMINI2;
-    const isGeminiThinking = isGemini && import_evalforge_types11.GEMINI_THINKING_MODEL_IDS.has(modelConfig.model);
+    const isGeminiThinking = isGemini && import_evalforge_types11.GEMINI_THINKING_MODEL_IDS.has(modelId);
     const supportsThinking = isAnthropic || isResponsesAPI || isGeminiThinking;
-    const providerOpts = {
+    const thinkingBudgetTokens = cfg.thinkingBudgetTokens ?? 1e4;
+    const reasoningEffort = cfg.reasoningEffort ?? "high";
+    const computedProviderOpts = {
       ...isAnthropic && {
         anthropic: {
-          thinking: { type: "enabled", budgetTokens: 1e4 }
+          thinking: { type: "enabled", budgetTokens: thinkingBudgetTokens }
         }
       },
       ...isResponsesAPI && {
@@ -3669,7 +3725,7 @@ async function executeWithAiSdk(context) {
           // Prevent the SDK from sending item_reference inputs that the proxy can't forward
           store: false,
           forceReasoning: true,
-          reasoningEffort: "high",
+          reasoningEffort,
           reasoningSummary: "detailed"
         }
       },
@@ -3682,10 +3738,17 @@ async function executeWithAiSdk(context) {
         }
       }
     };
+    const { providerOptions: extraProviderOptions, ...topLevelExtras } = configExtras;
+    const mergedProviderOptions = {
+      ...extraProviderOptions && typeof extraProviderOptions === "object" ? extraProviderOptions : {},
+      ...computedProviderOpts
+    };
     const stepTimestamps = [];
+    const effectiveMaxTurns = cfg.maxTurns === 0 ? void 0 : cfg.maxTurns ?? DEFAULT_MAX_TOOL_STEPS;
     const { triggerPromptImages } = context;
     const hasImages = triggerPromptImages && triggerPromptImages.length > 0;
-    const generateTextParams = {
+    const result = await (0, import_ai.generateText)({
+      ...topLevelExtras,
       model,
       system: systemPrompt,
       ...hasImages ? {
@@ -3703,11 +3766,16 @@ async function executeWithAiSdk(context) {
           }
         ]
       } : { prompt: scenario.triggerPrompt },
-      temperature: supportsThinking ? void 0 : modelConfig.temperature,
-      maxOutputTokens: modelConfig.maxTokens,
+      temperature: supportsThinking ? void 0 : cfg.temperature,
+      topP: supportsThinking ? void 0 : cfg.topP,
+      frequencyPenalty: cfg.frequencyPenalty,
+      presencePenalty: cfg.presencePenalty,
+      seed: cfg.seed,
+      stopSequences: cfg.stopSequences,
+      maxOutputTokens: cfg.maxTokens,
       tools: mcpTools,
-      stopWhen: mcpTools ? (0, import_ai.stepCountIs)(modelConfig.maxTurns ?? DEFAULT_MAX_TOOL_STEPS) : (0, import_ai.stepCountIs)(1),
-      providerOptions: providerOpts,
+      ...mcpTools && effectiveMaxTurns != null ? { stopWhen: (0, import_ai.stepCountIs)(effectiveMaxTurns) } : !mcpTools ? { stopWhen: (0, import_ai.stepCountIs)(1) } : {},
+      providerOptions: mergedProviderOptions,
       onStepFinish: (step) => {
         stepTimestamps.push(Date.now());
         if (traceContext) {
@@ -3735,8 +3803,7 @@ async function executeWithAiSdk(context) {
           );
         }
       }
-    };
-    const result = await (0, import_ai.generateText)(generateTextParams);
+    });
     const durationMs = Date.now() - startTime;
     const usage = {
       inputTokens: result.usage.inputTokens ?? 0,
@@ -3747,7 +3814,7 @@ async function executeWithAiSdk(context) {
       result.steps,
       durationMs,
       usage,
-      modelConfig.model,
+      modelId,
       provider,
       startTime,
       stepTimestamps
@@ -4660,11 +4727,12 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
   const startedAt = (/* @__PURE__ */ new Date()).toISOString();
   const targetId = evalData.evalRun.presetId ?? agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName || agent?.name || "";
+  const agentConfig = agent?.config;
   const executionContext = {
     skills: evalData.skills,
     scenario,
     cwd: workDir || process.cwd(),
-    modelConfig: agent?.modelConfig,
+    config: agentConfig,
     aiGatewayUrl: config.aiGatewayUrl,
     aiGatewayHeaders: config.aiGatewayHeaders,
     traceContext: {
@@ -4699,7 +4767,18 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     infrastructurePaths
   );
   const templateFiles = workDir ? extractTemplateFiles(beforeSnapshot, afterSnapshot, infrastructurePaths) : void 0;
-  const resolvedModelConfig = agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
+  const snapshotModelConfig = agentConfig?.model ? {
+    model: agentConfig.model,
+    ...agentConfig.temperature != null && {
+      temperature: agentConfig.temperature
+    },
+    ...agentConfig.maxTokens != null && {
+      maxTokens: agentConfig.maxTokens
+    },
+    ...agentConfig.maxTurns != null && {
+      maxTurns: agentConfig.maxTurns
+    }
+  } : agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
   return {
     id: (0, import_crypto4.randomUUID)(),
     targetId,
@@ -4707,7 +4786,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     scenarioId: scenario.id,
     scenarioName: scenario.name,
     triggerPrompt: scenario.triggerPrompt,
-    modelConfig: resolvedModelConfig,
+    modelConfig: snapshotModelConfig,
     duration: durationMs,
     outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,