@wix/evalforge-evaluator 0.146.0 → 0.148.0

package/build/index.js

@@ -330,10 +330,10 @@ async function fetchEvaluationData(api, projectId2, evalRunId2) {
   if (evalRun.agentId) {
     agent = await api.getAgent(projectId2, evalRun.agentId);
   }
-  let skills = [];
-  let mcps = [];
-  let subAgents = [];
-  let rules = [];
+  const skills = [];
+  const mcps = [];
+  const subAgents = [];
+  const rules = [];
   if (evalRun.capabilityIds && evalRun.capabilityIds.length > 0) {
     const fetchResults = await Promise.allSettled(
       evalRun.capabilityIds.map((id) => api.getCapability(projectId2, id))
@@ -1190,10 +1190,10 @@ async function executeWithClaudeCode(skills, scenario, options) {
   let lastAction = "Starting...";
   let lastToolName;
   let lastFilePath;
-  const maxTurns = options.maxTurns ?? 10;
+  const maxTurns = options.maxTurns || void 0;
   let messageCount = 0;
   const canUseTool = async (_toolName, input) => ({ behavior: "allow", updatedInput: input });
-  const baseAllowedTools = [
+  const baseAllowedTools = options.allowedTools ?? [
     "Skill",
     "Read",
     "Write",
@@ -1207,13 +1207,17 @@ async function executeWithClaudeCode(skills, scenario, options) {
   ];
   const allowedTools = (options.mcps?.length ?? 0) > 0 ? [...baseAllowedTools, "mcp__*"] : baseAllowedTools;
   const queryOptions = {
+    ...options.extras ?? {},
     env: sdkEnv,
     cwd: options.cwd,
     settingSources: ["project"],
     allowedTools,
+    ...options.disallowedTools?.length ? { disallowedTools: options.disallowedTools } : {},
     model: options.model || DEFAULT_MODEL,
     maxTurns,
     maxThinkingTokens: options.maxThinkingTokens,
+    ...options.effort ? { effort: options.effort } : {},
+    ...options.maxBudgetUsd != null ? { maxBudgetUsd: options.maxBudgetUsd } : {},
     // Use 'default' permission mode with custom canUseTool handler
     // instead of 'bypassPermissions' which fails on root
     permissionMode: "default",
@@ -1288,7 +1292,7 @@ async function executeWithClaudeCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
-  const SDK_TIMEOUT_MS = Math.max(3e5, maxTurns * 6e4);
+  const SDK_TIMEOUT_MS = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
   let timeoutHandle;
   let timedOut = false;
   const HEARTBEAT_INTERVAL_MS = 1e4;
@@ -2052,7 +2056,7 @@ var ClaudeCodeAdapter = class {
       skills,
       scenario,
       cwd,
-      modelConfig,
+      config,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -2061,20 +2065,37 @@ var ClaudeCodeAdapter = class {
       rules,
       systemPrompt
     } = context;
-    const modelForSdk = modelConfig?.model;
+    const typed = config ? import_evalforge_types5.ClaudeCodeConfigSchema.passthrough().safeParse(config) : void 0;
+    const cfg = typed?.success ? typed.data : void 0;
+    const schemaKeys = new Set(Object.keys(import_evalforge_types5.ClaudeCodeConfigSchema.shape));
+    const extras = {};
+    if (config) {
+      for (const [key, value] of Object.entries(config)) {
+        if (!schemaKeys.has(key)) extras[key] = value;
+      }
+    }
+    const rawMaxTurns = cfg?.maxTurns;
+    const maxTurns = rawMaxTurns === 0 ? void 0 : rawMaxTurns;
     const options = {
       cwd,
-      model: modelForSdk,
-      temperature: modelConfig?.temperature,
-      maxTokens: modelConfig?.maxTokens,
-      maxTurns: modelConfig?.maxTurns,
+      model: cfg?.model,
+      temperature: cfg?.temperature,
+      maxTokens: cfg?.maxTokens,
+      maxTurns,
+      maxThinkingTokens: cfg?.maxThinkingTokens,
+      allowedTools: cfg?.allowedTools,
+      disallowedTools: cfg?.disallowedTools,
+      effort: cfg?.effort,
+      maxBudgetUsd: cfg?.maxBudgetUsd,
+      maxDurationMs: cfg?.maxDurationMs,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
       mcps,
       subAgents,
       rules,
-      systemPrompt
+      systemPrompt,
+      extras
     };
     const { result, llmTrace, conversation } = await executeWithClaudeCode(
       skills,
@@ -2214,18 +2235,29 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = import_eval
 var import_os3 = require("os");
 var import_evalforge_types6 = require("@wix/evalforge-types");
 var DEFAULT_MODEL2 = `${import_evalforge_types6.ClaudeModel.CLAUDE_4_5_SONNET_1_0}`;
+var OPENCODE_MODEL_ALIASES = {
+  "claude-sonnet-4": "claude-sonnet-4-0",
+  "claude-opus-4": "claude-opus-4-0"
+};
 function parseModel(model) {
   const slashIndex = model.indexOf("/");
   if (slashIndex > 0) {
+    const providerID = model.slice(0, slashIndex);
+    const rawModelID = model.slice(slashIndex + 1);
     return {
-      providerID: model.slice(0, slashIndex),
-      modelID: model.slice(slashIndex + 1)
+      providerID,
+      modelID: OPENCODE_MODEL_ALIASES[rawModelID] ?? rawModelID
     };
   }
+  const modelID = OPENCODE_MODEL_ALIASES[model] ?? model;
   const isOpenAI = import_evalforge_types6.AVAILABLE_OPENAI_MODEL_IDS.includes(
     model
   );
-  return { providerID: isOpenAI ? "openai" : "anthropic", modelID: model };
+  const isGemini = import_evalforge_types6.AVAILABLE_GEMINI_MODEL_IDS.includes(
+    model
+  );
+  if (isGemini) return { providerID: "google", modelID };
+  return { providerID: isOpenAI ? "openai" : "anthropic", modelID };
 }
 function toOpenCodeMcpConfig(servers) {
   const result = {};
@@ -2276,8 +2308,9 @@ async function buildOpenCodeEnv(options) {
   const { providerID, modelID } = parseModel(modelStr);
   const provider = {};
   if (options.aiGatewayUrl) {
+    const proxyPath = providerID === "google" ? "gemini" : providerID;
     const providerOptions = {
-      baseURL: `${options.aiGatewayUrl}/proxy/${providerID}`,
+      baseURL: `${options.aiGatewayUrl}/proxy/${proxyPath}`,
       apiKey: "sk-placeholder-auth-handled-by-gateway"
     };
     if (options.aiGatewayHeaders) {
@@ -2308,20 +2341,26 @@ async function buildOpenCodeEnv(options) {
   if (options.temperature != null) {
     agentOverrides.temperature = options.temperature;
   }
-  if (options.maxTurns != null) {
+  if (options.topP != null) {
+    agentOverrides.top_p = options.topP;
+  }
+  if (options.maxTurns != null && options.maxTurns > 0) {
     agentOverrides.maxSteps = options.maxTurns;
   }
+  const parsed = options.config ? import_evalforge_types6.OpenCodeConfigSchema.passthrough().safeParse(options.config) : void 0;
+  const configPermission = parsed?.success ? parsed.data.permission : void 0;
+  const defaultPermission = {
+    "*": "allow"
+  };
+  const permission = {
+    ...defaultPermission,
+    ...configPermission
+  };
   const config = {
     model: `${providerID}/${modelID}`,
     provider,
     ...Object.keys(agentOverrides).length > 0 ? { agent: { build: agentOverrides } } : {},
-    permission: {
-      edit: "allow",
-      bash: "allow",
-      webfetch: "allow",
-      doom_loop: "allow",
-      external_directory: "allow"
-    },
+    permission,
     ...mcp ? { mcp } : {}
   };
   const env = {
@@ -3055,16 +3094,18 @@ async function executeWithOpenCode(skills, scenario, options) {
     hasAiGatewayHeaders: !!options.aiGatewayHeaders,
     model: options.model
   });
-  const maxTurns = options.maxTurns ?? 10;
-  const sdkTimeoutMs = Math.max(3e5, maxTurns * 6e4);
+  const maxTurns = options.maxTurns || void 0;
+  const sdkTimeoutMs = options.maxDurationMs ?? Math.max(3e5, (maxTurns ?? 10) * 6e4);
   const { env, providerID, modelID } = await buildOpenCodeEnv({
     model: options.model,
     temperature: options.temperature,
+    topP: options.topP,
     maxTurns,
     aiGatewayUrl: options.aiGatewayUrl,
     aiGatewayHeaders: options.aiGatewayHeaders,
     mcps: options.mcps,
-    cwd: options.cwd
+    cwd: options.cwd,
+    config: options.config
   });
   const startTime = /* @__PURE__ */ new Date();
   const traceContext = options.traceContext;
@@ -3092,13 +3133,13 @@ async function executeWithOpenCode(skills, scenario, options) {
       traceContext.authToken
     );
   }
+  const variant = options.thinkingVariant ?? "high";
+  const thinkingArgs = variant === "none" ? [] : ["--thinking", "--variant", variant];
   const baseArgs = [
     "run",
     "--format",
     "json",
-    "--thinking",
-    "--variant",
-    "high",
+    ...thinkingArgs,
     "--model",
     `${providerID}/${modelID}`,
     "--dir",
@@ -3289,7 +3330,7 @@ var OpenCodeAdapter = class {
       skills,
       scenario,
       cwd,
-      modelConfig,
+      config,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
@@ -3298,18 +3339,26 @@ var OpenCodeAdapter = class {
       rules,
       systemPrompt
     } = context;
+    const typed = config ? import_evalforge_types9.OpenCodeConfigSchema.passthrough().safeParse(config) : void 0;
+    const cfg = typed?.success ? typed.data : void 0;
+    const rawMaxTurns = cfg?.maxTurns;
+    const maxTurns = rawMaxTurns === 0 ? void 0 : rawMaxTurns;
     const options = {
       cwd,
-      model: modelConfig?.model,
-      temperature: modelConfig?.temperature,
-      maxTurns: modelConfig?.maxTurns,
+      model: cfg?.model,
+      temperature: cfg?.temperature,
+      topP: cfg?.topP,
+      maxTurns,
+      thinkingVariant: cfg?.thinkingVariant,
+      maxDurationMs: cfg?.maxDurationMs,
       aiGatewayUrl,
       aiGatewayHeaders,
       traceContext,
       mcps,
       subAgents,
       rules,
-      systemPrompt
+      systemPrompt,
+      config
     };
     const { result, llmTrace, conversation } = await executeWithOpenCode(
       skills,
@@ -3338,6 +3387,7 @@ defaultRegistry.register(openCodeAdapter);
 // src/run-scenario/agents/simple-agent/execute.ts
 var import_ai = require("ai");
 var import_anthropic = require("@ai-sdk/anthropic");
+var import_google = require("@ai-sdk/google");
 var import_openai = require("@ai-sdk/openai");
 var import_evalforge_types11 = require("@wix/evalforge-types");
 var import_crypto3 = require("crypto");
@@ -3360,8 +3410,9 @@ async function buildMcpTools(mcps, cwd) {
         const client = await (0, import_mcp.createMCPClient)({ transport });
         clients.push(client);
         const tools = await client.tools();
+        const safePrefix = serverName.replace(/[^a-zA-Z0-9]/g, "_");
         for (const [toolName, tool] of Object.entries(tools)) {
-          allTools[`${serverName}__${toolName}`] = tool;
+          allTools[`${safePrefix}_${toolName}`] = tool;
         }
       }
     }
@@ -3437,6 +3488,7 @@ function extractErrorText(content) {
 // src/run-scenario/agents/simple-agent/cost-calculation.ts
 var import_evalforge_types10 = require("@wix/evalforge-types");
 var PROVIDER_ANTHROPIC = "anthropic";
+var PROVIDER_GEMINI = "gemini";
 var MODEL_PRICING = {
   // Anthropic — Claude 4.6
   "claude-sonnet-4-6": { input: 3, output: 15 },
@@ -3463,7 +3515,18 @@ var MODEL_PRICING = {
   o3: { input: 2, output: 8 },
   "o4-mini": { input: 1.1, output: 4.4 },
   "o3-mini": { input: 1.1, output: 4.4 },
-  o1: { input: 15, output: 60 }
+  o1: { input: 15, output: 60 },
+  // Google Gemini 2.0
+  "gemini-2.0-flash": { input: 0.1, output: 0.4 },
+  "gemini-2.0-flash-lite": { input: 0.075, output: 0.3 },
+  // Google Gemini 2.5
+  "gemini-2.5-pro": { input: 1.25, output: 10 },
+  "gemini-2.5-flash": { input: 0.15, output: 0.6 },
+  "gemini-2.5-flash-lite": { input: 0.075, output: 0.3 },
+  // Google Gemini 3.x — standard pricing up to 200K context tokens
+  "gemini-3-pro-preview": { input: 2, output: 12 },
+  "gemini-3-flash-preview": { input: 0.5, output: 3 },
+  "gemini-3.1-pro-preview": { input: 2, output: 12 }
 };
 function extractGatewayCost(step, provider) {
   try {
@@ -3474,6 +3537,15 @@ function extractGatewayCost(step, provider) {
       const cost2 = usage?.total_cost_usd;
       return typeof cost2 === "number" && cost2 > 0 ? cost2 : void 0;
     }
+    if (provider === PROVIDER_GEMINI) {
+      const meta = step.providerMetadata;
+      const google = meta?.google;
+      const cost2 = google?.total_cost_usd;
+      if (typeof cost2 === "number" && cost2 > 0) return cost2;
+      const body2 = step.response?.body;
+      const bodyCost = body2?.total_cost_usd;
+      return typeof bodyCost === "number" && bodyCost > 0 ? bodyCost : void 0;
+    }
     const body = step.response?.body;
     const cost = body?.total_cost_usd;
     return typeof cost === "number" && cost > 0 ? cost : void 0;
@@ -3551,10 +3623,10 @@ function buildConversation3(triggerPrompt, steps, executionStartMs, stepTimestam
 // src/run-scenario/agents/simple-agent/execute.ts
 var PROVIDER_ANTHROPIC2 = "anthropic";
 var PROVIDER_OPENAI = "openai";
+var PROVIDER_GEMINI2 = "gemini";
 var DEFAULT_MAX_TOOL_STEPS = 25;
 function createModel(modelId, baseUrl, headers) {
-  const isClaudeModel = isClaudeModelId(modelId);
-  if (isClaudeModel) {
+  if (isClaudeModelId(modelId)) {
     const anthropic = (0, import_anthropic.createAnthropic)({
       baseURL: `${baseUrl}/proxy/anthropic`,
       apiKey: "proxy-auth",
@@ -3562,6 +3634,14 @@ function createModel(modelId, baseUrl, headers) {
     });
     return anthropic(modelId);
   }
+  if (isGeminiModelId(modelId)) {
+    const google = (0, import_google.createGoogleGenerativeAI)({
+      baseURL: `${baseUrl}/proxy/gemini`,
+      apiKey: "proxy-auth",
+      headers
+    });
+    return google(modelId);
+  }
   const openai = (0, import_openai.createOpenAI)({
     baseURL: `${baseUrl}/proxy/openai`,
     apiKey: "proxy-auth",
@@ -3579,6 +3659,11 @@ function isClaudeModelId(modelId) {
     (id) => modelId === id || modelId.startsWith(id)
   );
 }
+function isGeminiModelId(modelId) {
+  return import_evalforge_types11.AVAILABLE_GEMINI_MODEL_IDS.some(
+    (id) => modelId === id || modelId.startsWith(id)
+  );
+}
 function extractSkillContent(files) {
   if (!files || files.length === 0) return void 0;
   const skillMd = files.find((f) => f.path === "SKILL.md");
@@ -3589,20 +3674,30 @@ async function executeWithAiSdk(context) {
   const {
     scenario,
     cwd,
-    modelConfig,
+    config,
     aiGatewayUrl,
     aiGatewayHeaders = {},
     mcps,
     traceContext
   } = context;
+  const typed = config ? import_evalforge_types11.SimpleAgentConfigSchema.passthrough().safeParse(config) : void 0;
+  const cfg = typed?.success ? typed.data : void 0;
+  const schemaKeys = new Set(Object.keys(import_evalforge_types11.SimpleAgentConfigSchema.shape));
+  const configExtras = {};
+  if (config) {
+    for (const [key, value] of Object.entries(config)) {
+      if (!schemaKeys.has(key)) configExtras[key] = value;
+    }
+  }
   if (!aiGatewayUrl) {
     throw new Error("Simple Agent requires aiGatewayUrl");
   }
-  if (!modelConfig?.model) {
-    throw new Error("Simple Agent requires a model in modelConfig");
+  if (!cfg?.model) {
+    throw new Error("Simple Agent requires a model in config");
   }
-  const model = createModel(modelConfig.model, aiGatewayUrl, aiGatewayHeaders);
-  const provider = isClaudeModelId(modelConfig.model) ? PROVIDER_ANTHROPIC2 : PROVIDER_OPENAI;
+  const modelId = cfg.model;
+  const model = createModel(modelId, aiGatewayUrl, aiGatewayHeaders);
+  const provider = isClaudeModelId(modelId) ? PROVIDER_ANTHROPIC2 : isGeminiModelId(modelId) ? PROVIDER_GEMINI2 : PROVIDER_OPENAI;
   const systemPrompt = composeSystemPrompt(context);
   const { tools: mcpTools, clients } = mcps && mcps.length > 0 ? await buildMcpTools(mcps, cwd) : { tools: void 0, clients: [] };
   const startTime = Date.now();
@@ -3612,13 +3707,17 @@ async function executeWithAiSdk(context) {
   try {
     const isAnthropic = provider === PROVIDER_ANTHROPIC2;
     const isResponsesAPI = [...import_evalforge_types11.OPENAI_RESPONSES_MODEL_IDS].some(
-      (id) => modelConfig.model === id || modelConfig.model.startsWith(id)
+      (id) => modelId === id || modelId.startsWith(id)
     );
-    const supportsThinking = isAnthropic || isResponsesAPI;
-    const providerOpts = {
+    const isGemini = provider === PROVIDER_GEMINI2;
+    const isGeminiThinking = isGemini && import_evalforge_types11.GEMINI_THINKING_MODEL_IDS.has(modelId);
+    const supportsThinking = isAnthropic || isResponsesAPI || isGeminiThinking;
+    const thinkingBudgetTokens = cfg.thinkingBudgetTokens ?? 1e4;
+    const reasoningEffort = cfg.reasoningEffort ?? "high";
+    const computedProviderOpts = {
       ...isAnthropic && {
         anthropic: {
-          thinking: { type: "enabled", budgetTokens: 1e4 }
+          thinking: { type: "enabled", budgetTokens: thinkingBudgetTokens }
         }
       },
       ...isResponsesAPI && {
@@ -3626,15 +3725,30 @@ async function executeWithAiSdk(context) {
           // Prevent the SDK from sending item_reference inputs that the proxy can't forward
           store: false,
           forceReasoning: true,
-          reasoningEffort: "high",
+          reasoningEffort,
           reasoningSummary: "detailed"
         }
+      },
+      ...isGeminiThinking && {
+        google: {
+          thinkingConfig: {
+            includeThoughts: true,
+            thinkingBudget: 1e4
+          }
+        }
       }
     };
+    const { providerOptions: extraProviderOptions, ...topLevelExtras } = configExtras;
+    const mergedProviderOptions = {
+      ...extraProviderOptions && typeof extraProviderOptions === "object" ? extraProviderOptions : {},
+      ...computedProviderOpts
+    };
     const stepTimestamps = [];
+    const effectiveMaxTurns = cfg.maxTurns === 0 ? void 0 : cfg.maxTurns ?? DEFAULT_MAX_TOOL_STEPS;
     const { triggerPromptImages } = context;
     const hasImages = triggerPromptImages && triggerPromptImages.length > 0;
-    const generateTextParams = {
+    const result = await (0, import_ai.generateText)({
+      ...topLevelExtras,
       model,
       system: systemPrompt,
       ...hasImages ? {
@@ -3652,11 +3766,16 @@ async function executeWithAiSdk(context) {
           }
         ]
       } : { prompt: scenario.triggerPrompt },
-      temperature: supportsThinking ? void 0 : modelConfig.temperature,
-      maxOutputTokens: modelConfig.maxTokens,
+      temperature: supportsThinking ? void 0 : cfg.temperature,
+      topP: supportsThinking ? void 0 : cfg.topP,
+      frequencyPenalty: cfg.frequencyPenalty,
+      presencePenalty: cfg.presencePenalty,
+      seed: cfg.seed,
+      stopSequences: cfg.stopSequences,
+      maxOutputTokens: cfg.maxTokens,
       tools: mcpTools,
-      stopWhen: mcpTools ? (0, import_ai.stepCountIs)(modelConfig.maxTurns ?? DEFAULT_MAX_TOOL_STEPS) : (0, import_ai.stepCountIs)(1),
-      providerOptions: providerOpts,
+      ...mcpTools && effectiveMaxTurns != null ? { stopWhen: (0, import_ai.stepCountIs)(effectiveMaxTurns) } : !mcpTools ? { stopWhen: (0, import_ai.stepCountIs)(1) } : {},
+      providerOptions: mergedProviderOptions,
       onStepFinish: (step) => {
         stepTimestamps.push(Date.now());
         if (traceContext) {
@@ -3684,8 +3803,7 @@ async function executeWithAiSdk(context) {
           );
         }
       }
-    };
-    const result = await (0, import_ai.generateText)(generateTextParams);
+    });
     const durationMs = Date.now() - startTime;
     const usage = {
       inputTokens: result.usage.inputTokens ?? 0,
@@ -3696,7 +3814,7 @@ async function executeWithAiSdk(context) {
       result.steps,
       durationMs,
       usage,
-      modelConfig.model,
+      modelId,
       provider,
       startTime,
       stepTimestamps
@@ -4609,11 +4727,12 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
   const startedAt = (/* @__PURE__ */ new Date()).toISOString();
   const targetId = evalData.evalRun.presetId ?? agent?.id ?? evalData.evalRun.id;
   const targetName = evalData.presetName || agent?.name || "";
+  const agentConfig = agent?.config;
   const executionContext = {
     skills: evalData.skills,
     scenario,
     cwd: workDir || process.cwd(),
-    modelConfig: agent?.modelConfig,
+    config: agentConfig,
     aiGatewayUrl: config.aiGatewayUrl,
     aiGatewayHeaders: config.aiGatewayHeaders,
     traceContext: {
@@ -4648,7 +4767,18 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     infrastructurePaths
   );
   const templateFiles = workDir ? extractTemplateFiles(beforeSnapshot, afterSnapshot, infrastructurePaths) : void 0;
-  const resolvedModelConfig = agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
+  const snapshotModelConfig = agentConfig?.model ? {
+    model: agentConfig.model,
+    ...agentConfig.temperature != null && {
+      temperature: agentConfig.temperature
+    },
+    ...agentConfig.maxTokens != null && {
+      maxTokens: agentConfig.maxTokens
+    },
+    ...agentConfig.maxTurns != null && {
+      maxTurns: agentConfig.maxTurns
+    }
+  } : agent?.modelConfig ?? (llmTrace?.summary.modelsUsed?.[0] ? { model: llmTrace.summary.modelsUsed[0] } : void 0);
   return {
     id: (0, import_crypto4.randomUUID)(),
     targetId,
@@ -4656,7 +4786,7 @@ async function runAgentWithContext(config, evalRunId2, scenario, evalData, workD
     scenarioId: scenario.id,
     scenarioName: scenario.name,
     triggerPrompt: scenario.triggerPrompt,
-    modelConfig: resolvedModelConfig,
+    modelConfig: snapshotModelConfig,
     duration: durationMs,
     outputText,
     fileDiffs: fileDiffs.length > 0 ? fileDiffs : void 0,