@wix/evalforge-evaluator 0.146.0 → 0.147.0

package/build/index.mjs

@@ -2216,7 +2216,8 @@ async function writeSubAgentsToFilesystem2(cwd, subAgents, fetchFn = fetchGitHub
 import { homedir as homedir2 } from "os";
 import {
   ClaudeModel as ClaudeModel2,
-  AVAILABLE_OPENAI_MODEL_IDS
+  AVAILABLE_OPENAI_MODEL_IDS,
+  AVAILABLE_GEMINI_MODEL_IDS
 } from "@wix/evalforge-types";
 var DEFAULT_MODEL2 = `${ClaudeModel2.CLAUDE_4_5_SONNET_1_0}`;
 function parseModel(model) {
@@ -2230,6 +2231,10 @@ function parseModel(model) {
   const isOpenAI = AVAILABLE_OPENAI_MODEL_IDS.includes(
     model
   );
+  const isGemini = AVAILABLE_GEMINI_MODEL_IDS.includes(
+    model
+  );
+  if (isGemini) return { providerID: "google", modelID: model };
   return { providerID: isOpenAI ? "openai" : "anthropic", modelID: model };
 }
 function toOpenCodeMcpConfig(servers) {
@@ -2281,8 +2286,9 @@ async function buildOpenCodeEnv(options) {
   const { providerID, modelID } = parseModel(modelStr);
   const provider = {};
   if (options.aiGatewayUrl) {
+    const proxyPath = providerID === "google" ? "gemini" : providerID;
     const providerOptions = {
-      baseURL: `${options.aiGatewayUrl}/proxy/${providerID}`,
+      baseURL: `${options.aiGatewayUrl}/proxy/${proxyPath}`,
       apiKey: "sk-placeholder-auth-handled-by-gateway"
     };
     if (options.aiGatewayHeaders) {
@@ -3346,9 +3352,12 @@ import {
   stepCountIs
 } from "ai";
 import { createAnthropic } from "@ai-sdk/anthropic";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { createOpenAI } from "@ai-sdk/openai";
 import {
   AVAILABLE_CLAUDE_MODEL_IDS,
+  AVAILABLE_GEMINI_MODEL_IDS as AVAILABLE_GEMINI_MODEL_IDS2,
+  GEMINI_THINKING_MODEL_IDS,
   OPENAI_RESPONSES_MODEL_IDS,
   LLMStepType as LLMStepType3,
   LiveTraceEventType as LiveTraceEventType3
@@ -3373,8 +3382,9 @@ async function buildMcpTools(mcps, cwd) {
         const client = await createMCPClient({ transport });
         clients.push(client);
         const tools = await client.tools();
+        const safePrefix = serverName.replace(/[^a-zA-Z0-9]/g, "_");
         for (const [toolName, tool] of Object.entries(tools)) {
-          allTools[`${serverName}__${toolName}`] = tool;
+          allTools[`${safePrefix}_${toolName}`] = tool;
         }
       }
     }
@@ -3450,6 +3460,7 @@ function extractErrorText(content) {
 // src/run-scenario/agents/simple-agent/cost-calculation.ts
 import { normalizeModelId } from "@wix/evalforge-types";
 var PROVIDER_ANTHROPIC = "anthropic";
+var PROVIDER_GEMINI = "gemini";
 var MODEL_PRICING = {
   // Anthropic — Claude 4.6
   "claude-sonnet-4-6": { input: 3, output: 15 },
@@ -3476,7 +3487,18 @@ var MODEL_PRICING = {
   o3: { input: 2, output: 8 },
   "o4-mini": { input: 1.1, output: 4.4 },
   "o3-mini": { input: 1.1, output: 4.4 },
-  o1: { input: 15, output: 60 }
+  o1: { input: 15, output: 60 },
+  // Google Gemini 2.0
+  "gemini-2.0-flash": { input: 0.1, output: 0.4 },
+  "gemini-2.0-flash-lite": { input: 0.075, output: 0.3 },
+  // Google Gemini 2.5
+  "gemini-2.5-pro": { input: 1.25, output: 10 },
+  "gemini-2.5-flash": { input: 0.15, output: 0.6 },
+  "gemini-2.5-flash-lite": { input: 0.075, output: 0.3 },
+  // Google Gemini 3.x — standard pricing up to 200K context tokens
+  "gemini-3-pro-preview": { input: 2, output: 12 },
+  "gemini-3-flash-preview": { input: 0.5, output: 3 },
+  "gemini-3.1-pro-preview": { input: 2, output: 12 }
 };
 function extractGatewayCost(step, provider) {
   try {
@@ -3487,6 +3509,15 @@ function extractGatewayCost(step, provider) {
       const cost2 = usage?.total_cost_usd;
       return typeof cost2 === "number" && cost2 > 0 ? cost2 : void 0;
     }
+    if (provider === PROVIDER_GEMINI) {
+      const meta = step.providerMetadata;
+      const google = meta?.google;
+      const cost2 = google?.total_cost_usd;
+      if (typeof cost2 === "number" && cost2 > 0) return cost2;
+      const body2 = step.response?.body;
+      const bodyCost = body2?.total_cost_usd;
+      return typeof bodyCost === "number" && bodyCost > 0 ? bodyCost : void 0;
+    }
     const body = step.response?.body;
     const cost = body?.total_cost_usd;
     return typeof cost === "number" && cost > 0 ? cost : void 0;
@@ -3564,10 +3595,10 @@ function buildConversation3(triggerPrompt, steps, executionStartMs, stepTimestam
 // src/run-scenario/agents/simple-agent/execute.ts
 var PROVIDER_ANTHROPIC2 = "anthropic";
 var PROVIDER_OPENAI = "openai";
+var PROVIDER_GEMINI2 = "gemini";
 var DEFAULT_MAX_TOOL_STEPS = 25;
 function createModel(modelId, baseUrl, headers) {
-  const isClaudeModel = isClaudeModelId(modelId);
-  if (isClaudeModel) {
+  if (isClaudeModelId(modelId)) {
     const anthropic = createAnthropic({
       baseURL: `${baseUrl}/proxy/anthropic`,
       apiKey: "proxy-auth",
@@ -3575,6 +3606,14 @@ function createModel(modelId, baseUrl, headers) {
     });
     return anthropic(modelId);
   }
+  if (isGeminiModelId(modelId)) {
+    const google = createGoogleGenerativeAI({
+      baseURL: `${baseUrl}/proxy/gemini`,
+      apiKey: "proxy-auth",
+      headers
+    });
+    return google(modelId);
+  }
   const openai = createOpenAI({
     baseURL: `${baseUrl}/proxy/openai`,
     apiKey: "proxy-auth",
@@ -3592,6 +3631,11 @@ function isClaudeModelId(modelId) {
     (id) => modelId === id || modelId.startsWith(id)
   );
 }
+function isGeminiModelId(modelId) {
+  return AVAILABLE_GEMINI_MODEL_IDS2.some(
+    (id) => modelId === id || modelId.startsWith(id)
+  );
+}
 function extractSkillContent(files) {
   if (!files || files.length === 0) return void 0;
   const skillMd = files.find((f) => f.path === "SKILL.md");
@@ -3615,7 +3659,7 @@ async function executeWithAiSdk(context) {
     throw new Error("Simple Agent requires a model in modelConfig");
   }
   const model = createModel(modelConfig.model, aiGatewayUrl, aiGatewayHeaders);
-  const provider = isClaudeModelId(modelConfig.model) ? PROVIDER_ANTHROPIC2 : PROVIDER_OPENAI;
+  const provider = isClaudeModelId(modelConfig.model) ? PROVIDER_ANTHROPIC2 : isGeminiModelId(modelConfig.model) ? PROVIDER_GEMINI2 : PROVIDER_OPENAI;
   const systemPrompt = composeSystemPrompt(context);
   const { tools: mcpTools, clients } = mcps && mcps.length > 0 ? await buildMcpTools(mcps, cwd) : { tools: void 0, clients: [] };
   const startTime = Date.now();
@@ -3627,7 +3671,9 @@ async function executeWithAiSdk(context) {
     const isResponsesAPI = [...OPENAI_RESPONSES_MODEL_IDS].some(
       (id) => modelConfig.model === id || modelConfig.model.startsWith(id)
     );
-    const supportsThinking = isAnthropic || isResponsesAPI;
+    const isGemini = provider === PROVIDER_GEMINI2;
+    const isGeminiThinking = isGemini && GEMINI_THINKING_MODEL_IDS.has(modelConfig.model);
+    const supportsThinking = isAnthropic || isResponsesAPI || isGeminiThinking;
     const providerOpts = {
       ...isAnthropic && {
         anthropic: {
@@ -3642,6 +3688,14 @@ async function executeWithAiSdk(context) {
           reasoningEffort: "high",
           reasoningSummary: "detailed"
         }
+      },
+      ...isGeminiThinking && {
+        google: {
+          thinkingConfig: {
+            includeThoughts: true,
+            thinkingBudget: 1e4
+          }
+        }
       }
     };
     const stepTimestamps = [];