@llumiverse/drivers 1.3.0 → 1.4.0

package/lib/{esm/vertexai → vertexai}/models/gemini.js

@@ -1,62 +1,92 @@
-import { FinishReason, FunctionCallingConfigMode, HarmBlockThreshold, HarmCategory, Modality, ProminentPeople, ThinkingLevel } from "@google/genai";
-import { getConversationMeta, getGeminiModelVersion, incrementConversationTurn, isGeminiModelVersionGte, LlumiverseError, ModelType, PromptRole, readStreamAsBase64, stripBase64ImagesFromConversation, stripHeartbeatsFromConversation, truncateLargeTextInConversation, unwrapConversationArray } from "@llumiverse/core";
-import { asyncMap } from "@llumiverse/core/async";
+import { FinishReason, FunctionCallingConfigMode, HarmBlockThreshold, HarmCategory, Modality, ProminentPeople, ThinkingLevel, } from '@google/genai';
+import { getConversationMeta, getGeminiModelVersion, incrementConversationTurn, isGeminiModelVersionGte, LlumiverseError, ModelType, PromptRole, readStreamAsBase64, stripBase64ImagesFromConversation, stripHeartbeatsFromConversation, truncateLargeTextInConversation, unwrapConversationArray, } from '@llumiverse/core';
+import { asyncMap } from '@llumiverse/core/async';
+import { truncateBinaryForDebug } from '../../shared/debug-prompt.js';
 function supportsStructuredOutput(options) {
     // Gemini 1.0 Ultra does not support JSON output, 1.0 Pro does.
-    return !!options.result_schema && !options.model.includes("ultra");
+    return !!options.result_schema && !options.model.includes('ultra');
 }
 const geminiSafetySettings = [
     {
         category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
-        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH
+        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
     },
     {
         category: HarmCategory.HARM_CATEGORY_HARASSMENT,
-        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH
+        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
     },
     {
         category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
-        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH
+        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
     },
     {
         category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
-        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH
+        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
     },
     {
         category: HarmCategory.HARM_CATEGORY_UNSPECIFIED,
-        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH
+        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
     },
     {
         category: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY,
-        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH
-    }
+        threshold: HarmBlockThreshold.BLOCK_ONLY_HIGH,
+    },
 ];
+function formatGeminiContentForDebug(content) {
+    return {
+        ...content,
+        parts: content.parts?.map((part) => {
+            if (!part.inlineData?.data) {
+                return part;
+            }
+            return {
+                ...part,
+                inlineData: {
+                    ...part.inlineData,
+                    data: truncateBinaryForDebug(part.inlineData.data),
+                },
+            };
+        }),
+    };
+}
+export function formatGeminiDebugPrompt(prompt) {
+    return {
+        ...prompt,
+        contents: prompt.contents.map(formatGeminiContentForDebug),
+        system: prompt.system ? formatGeminiContentForDebug(prompt.system) : undefined,
+    };
+}
 // We do the mapping here rather than in common to avoid bringing the SDK into the common package.
 function getProminentPeopleOption(prominentPeople) {
     switch (prominentPeople) {
-        case "ALLOW_PROMINENT_PEOPLE":
+        case 'ALLOW_PROMINENT_PEOPLE':
             return ProminentPeople.ALLOW_PROMINENT_PEOPLE;
-        case "BLOCK_PROMINENT_PEOPLE":
+        case 'BLOCK_PROMINENT_PEOPLE':
             return ProminentPeople.BLOCK_PROMINENT_PEOPLE;
-        case "PROMINENT_PEOPLE_UNSPECIFIED":
+        case 'PROMINENT_PEOPLE_UNSPECIFIED':
             return ProminentPeople.PROMINENT_PEOPLE_UNSPECIFIED;
         default:
             return undefined;
     }
 }
-function getGeminiPayload(options, prompt) {
+export function getGeminiPayload(options, prompt) {
     const model_options = options.model_options;
     const tools = getToolDefinitions(options.tools);
     // When no tools are provided but conversation contains functionCall/functionResponse parts
     // (e.g. checkpoint summary calls), convert them to text to avoid API errors.
     // Use a local variable to avoid mutating the caller's conversation object.
-    let payloadContents = prompt.contents;
+    let payloadContents = mergeConsecutiveRole(prompt.contents);
     if (!tools && payloadContents) {
-        const hasToolParts = payloadContents.some(c => c.parts?.some(p => p.functionCall || p.functionResponse));
+        const hasToolParts = payloadContents.some((c) => c.parts?.some((p) => p.functionCall || p.functionResponse));
         if (hasToolParts) {
             payloadContents = convertGeminiFunctionPartsToText(payloadContents);
         }
     }
+    // Drop functionResponse parts whose functionCall was lost (e.g. compaction),
+    // which would otherwise trip Gemini's functionCall/functionResponse pairing.
+    if (payloadContents) {
+        payloadContents = fixOrphanedToolResults(payloadContents);
+    }
     const useStructuredOutput = supportsStructuredOutput(options) && !tools;
     const configNanoBanana = {
         systemInstruction: prompt.system,
@@ -77,20 +107,22 @@ function getGeminiPayload(options, prompt) {
             prominentPeople: getProminentPeopleOption(model_options?.prominent_people),
             outputMimeType: model_options?.output_mime_type,
             outputCompressionQuality: model_options?.output_compression_quality,
-        }
+        },
     };
     const config = {
         systemInstruction: prompt.system,
         safetySettings: geminiSafetySettings,
         tools: tools ? [tools] : undefined,
-        toolConfig: tools ? {
-            functionCallingConfig: {
-                mode: FunctionCallingConfigMode.AUTO,
+        toolConfig: tools
+            ? {
+                functionCallingConfig: {
+                    mode: FunctionCallingConfigMode.AUTO,
+                },
             }
-        } : undefined,
+            : undefined,
         candidateCount: 1,
         //JSON/Structured output
-        responseMimeType: useStructuredOutput ? "application/json" : undefined,
+        responseMimeType: useStructuredOutput ? 'application/json' : undefined,
         responseJsonSchema: useStructuredOutput ? options.result_schema : undefined,
         //Model options
         temperature: model_options?.temperature,
@@ -107,7 +139,7 @@ function getGeminiPayload(options, prompt) {
     return {
         model: options.model,
         contents: payloadContents,
-        config: options.model.toLowerCase().includes("image") ? configNanoBanana : config,
+        config: options.model.toLowerCase().includes('image') ? configNanoBanana : config,
     };
 }
 /**
@@ -121,17 +153,17 @@ function extractCompletionResults(content) {
         for (const part of parts) {
             if (part.text) {
                 results.push({
-                    type: "text",
-                    value: part.text
+                    type: 'text',
+                    value: part.text,
                 });
             }
             else if (part.inlineData) {
-                const base64ImageBytes = part.inlineData.data ?? "";
-                const mimeType = part.inlineData.mimeType ?? "image/png";
+                const base64ImageBytes = part.inlineData.data ?? '';
+                const mimeType = part.inlineData.mimeType ?? 'image/png';
                 const imageUrl = `data:${mimeType};base64,${base64ImageBytes}`;
                 results.push({
-                    type: "image",
-                    value: imageUrl
+                    type: 'image',
+                    value: imageUrl,
                 });
             }
         }
@@ -182,6 +214,49 @@ export function mergeConsecutiveRole(contents) {
     result.push(currentContent);
     return result;
 }
+/**
+ * Drop functionResponse parts whose name has no matching functionCall in the
+ * immediately-preceding `model` content. Gemini pairs a functionResponse to its
+ * functionCall by name; a response left dangling after its call was dropped
+ * (e.g. by conversation compaction/trimming, or an unmergeable parallel batch)
+ * causes the API to reject the request. Mirrors the same guard added to the
+ * Claude, Bedrock, and OpenAI drivers.
+ *
+ * Run after mergeConsecutiveRole so parallel responses that were split across
+ * user contents are first recombined under the model turn that issued the calls,
+ * and are therefore not mistaken for orphans.
+ */
+export function fixOrphanedToolResults(contents) {
+    if (contents.length === 0)
+        return contents;
+    const result = [];
+    for (let i = 0; i < contents.length; i++) {
+        const content = contents[i];
+        if (content.role !== 'user' || !content.parts) {
+            result.push(content);
+            continue;
+        }
+        const hasFunctionResponse = content.parts.some((part) => part.functionResponse);
+        if (!hasFunctionResponse) {
+            result.push(content);
+            continue;
+        }
+        const prev = contents[i - 1];
+        const allowedNames = new Set();
+        if (prev && prev.role === 'model' && prev.parts) {
+            for (const part of prev.parts) {
+                if (part.functionCall?.name)
+                    allowedNames.add(part.functionCall.name);
+            }
+        }
+        const filtered = content.parts.filter((part) => part.functionResponse ? allowedNames.has(part.functionResponse.name ?? '') : true);
+        // Drop the content if every part was an orphaned functionResponse.
+        if (filtered.length === 0)
+            continue;
+        result.push(filtered.length === content.parts.length ? content : { ...content, parts: filtered });
+    }
+    return result;
+}
 const supportedFinishReasons = [
     FinishReason.MAX_TOKENS,
     FinishReason.STOP,
@@ -205,7 +280,7 @@ function geminiThinkingBudget(option) {
     // Set minimum thinking level by default.
     // Docs: https://ai.google.dev/gemini-api/docs/thinking#set-budget
     if (getGeminiModelVersion(option.model) === '2.5') {
-        if (option.model.includes("pro")) {
+        if (option.model.includes('pro')) {
             return 128;
         }
         return 0;
@@ -213,28 +288,28 @@ function geminiThinkingBudget(option) {
     return undefined;
 }
 function geminiThinkingLevelForEffort(model, effort) {
-    if (model.includes("gemini-3-pro-image")) {
+    if (model.includes('gemini-3-pro-image')) {
         return ThinkingLevel.HIGH;
     }
-    if (model.includes("gemini-3.1-flash-image")) {
-        return effort === "low" ? ThinkingLevel.MINIMAL : ThinkingLevel.HIGH;
+    if (model.includes('gemini-3.1-flash-image')) {
+        return effort === 'low' ? ThinkingLevel.MINIMAL : ThinkingLevel.HIGH;
     }
     switch (effort) {
-        case "low":
+        case 'low':
             return ThinkingLevel.LOW;
-        case "medium":
+        case 'medium':
             return ThinkingLevel.MEDIUM;
-        case "high":
+        case 'high':
             return ThinkingLevel.HIGH;
         default:
             return undefined;
     }
 }
 function geminiBudgetForEffort(model, effort) {
-    const isFlashLite = model.includes("flash-lite");
-    const isFlash = model.includes("flash") && !isFlashLite;
-    const isPro = model.includes("pro");
-    if (effort === "low") {
+    const isFlashLite = model.includes('flash-lite');
+    const isFlash = model.includes('flash') && !isFlashLite;
+    const isPro = model.includes('pro');
+    if (effort === 'low') {
         if (isPro)
             return 128;
         if (isFlashLite)
@@ -243,7 +318,7 @@ function geminiBudgetForEffort(model, effort) {
             return 1;
         return 1024;
     }
-    if (effort === "medium") {
+    if (effort === 'medium') {
         return 8192;
     }
     if (isPro)
@@ -281,14 +356,14 @@ function geminiThinkingConfig(option) {
     if (isGeminiModelVersionGte(option.model, '3.0')) {
         return {
             includeThoughts: include_thoughts,
-            thinkingLevel: option.model.includes("gemini-3-pro-image") ? ThinkingLevel.HIGH : ThinkingLevel.LOW
+            thinkingLevel: option.model.includes('gemini-3-pro-image') ? ThinkingLevel.HIGH : ThinkingLevel.LOW,
         };
     }
     if (isGeminiModelVersionGte(option.model, '2.5')) {
         const thinking_budget_tokens = geminiThinkingBudget(option) ?? 0;
         return {
             includeThoughts: include_thoughts,
-            thinkingBudget: thinking_budget_tokens
+            thinkingBudget: thinking_budget_tokens,
         };
     }
 }
@@ -300,33 +375,33 @@ export class GeminiModelDefinition {
             name: modelId,
             provider: 'vertexai',
             type: ModelType.Text,
-            can_stream: true
+            can_stream: true,
         };
     }
     async createPrompt(_driver, segments, options) {
-        const splits = options.model.split("/");
+        const splits = options.model.split('/');
         const modelName = splits[splits.length - 1];
         options = { ...options, model: modelName };
         const schema = options.result_schema;
         let contents = [];
-        let system = { role: "user", parts: [] }; // Single content block for system messages
+        let system = { role: 'user', parts: [] }; // Single content block for system messages
         const safety = [];
         for (const msg of segments) {
             // Role specific handling
             if (msg.role === PromptRole.system) {
                 // Text only for system messages
                 if (msg.files && msg.files.length > 0) {
-                    throw new Error("Gemini does not support files/images etc. in system messages. Only text content is allowed.");
+                    throw new Error('Gemini does not support files/images etc. in system messages. Only text content is allowed.');
                 }
                 if (msg.content) {
                     system.parts?.push({
-                        text: msg.content
+                        text: msg.content,
                     });
                 }
             }
             else if (msg.role === PromptRole.tool) {
                 if (!msg.tool_use_id) {
-                    throw new Error("Tool response missing tool_use_id");
+                    throw new Error('Tool response missing tool_use_id');
                 }
                 // Build functionResponse part with optional thought_signature for Gemini thinking models
                 const functionResponsePart = {
@@ -339,10 +414,11 @@ export class GeminiModelDefinition {
                 };
                 contents.push({
                     role: 'user',
-                    parts: [functionResponsePart]
+                    parts: [functionResponsePart],
                 });
             }
-            else { // PromptRole.user, PromptRole.assistant, PromptRole.safety
+            else {
+                // PromptRole.user, PromptRole.assistant, PromptRole.safety
                 const parts = [];
                 // Text content handling
                 if (msg.content) {
@@ -353,14 +429,14 @@ export class GeminiModelDefinition {
                 // File content handling
                 if (msg.files) {
                     for (const f of msg.files) {
-                        const fileUrl = await f.getURL();
-                        const isGsUrl = fileUrl.startsWith('gs://') || fileUrl.startsWith('https://storage.googleapis.com/');
-                        if (isGsUrl) {
+                        const fileUri = await f.getURI();
+                        const isGsUri = fileUri.startsWith('gs://') || fileUri.startsWith('https://storage.googleapis.com/');
+                        if (isGsUri) {
                             parts.push({
                                 fileData: {
-                                    fileUri: fileUrl,
-                                    mimeType: f.mime_type
-                                }
+                                    fileUri,
+                                    mimeType: f.mime_type,
+                                },
                             });
                         }
                         else {
@@ -370,8 +446,8 @@ export class GeminiModelDefinition {
                             parts.push({
                                 inlineData: {
                                     data,
-                                    mimeType: f.mime_type
-                                }
+                                    mimeType: f.mime_type,
+                                },
                             });
                         }
                     }
@@ -397,17 +473,19 @@ export class GeminiModelDefinition {
             if (supportsStructuredOutput(options) && !options.tools) {
                 // Gemini structured output is unnecessarily sparse. Adding encouragement to fill the fields.
                 // Putting JSON in prompt is not recommended by Google, when using structured output.
-                system.parts?.push({ text: "Fill all appropriate fields in the JSON output." });
+                system.parts?.push({ text: 'Fill all appropriate fields in the JSON output.' });
             }
             else {
                 // Fallback to putting the schema in the system instructions, if not using structured output.
                 if (options.tools) {
                     system.parts?.push({
-                        text: "When not calling tools, the output must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema)
+                        text: `When not calling tools, the output must be a JSON object using the following JSON Schema:\n${JSON.stringify(schema)}`,
                     });
                 }
                 else {
-                    system.parts?.push({ text: "The output must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema) });
+                    system.parts?.push({
+                        text: `The output must be a JSON object using the following JSON Schema:\n${JSON.stringify(schema)}`,
+                    });
                 }
             }
         }
@@ -424,7 +502,7 @@ export class GeminiModelDefinition {
         return { contents, system };
     }
     usageMetadataToTokenUsage(driver, usageMetadata) {
-        if (!usageMetadata || !usageMetadata.totalTokenCount) {
+        if (!usageMetadata?.totalTokenCount) {
             return {};
         }
         const tokenUsage = {
@@ -434,15 +512,16 @@ export class GeminiModelDefinition {
             prompt_new: (usageMetadata.promptTokenCount ?? 0) - (usageMetadata.cachedContentTokenCount ?? 0),
         };
         //Output/Response side
-        tokenUsage.result = (usageMetadata.candidatesTokenCount ?? 0)
-            + (usageMetadata.thoughtsTokenCount ?? 0)
-            + (usageMetadata.toolUsePromptTokenCount ?? 0);
+        tokenUsage.result =
+            (usageMetadata.candidatesTokenCount ?? 0) +
+                (usageMetadata.thoughtsTokenCount ?? 0) +
+                (usageMetadata.toolUsePromptTokenCount ?? 0);
         if ((tokenUsage.total ?? 0) !== (tokenUsage.prompt ?? 0) + tokenUsage.result) {
             // Token-accounting mismatch: warn-level diagnostic (the call still
             // returns the best-effort tokenUsage). Use the driver's structured
             // logger so we don't promote stderr writes to ERROR in serverless
             // log aggregators — see the recoverable-tool-call sites below.
-            driver.logger.warn({ total: tokenUsage.total, prompt: tokenUsage.prompt, result: tokenUsage.result }, "[VertexAI] Gemini token usage mismatch: total does not equal prompt + result");
+            driver.logger.warn({ total: tokenUsage.total, prompt: tokenUsage.prompt, result: tokenUsage.result }, '[VertexAI] Gemini token usage mismatch: total does not equal prompt + result');
         }
         if (!tokenUsage.result) {
             tokenUsage.result = undefined; // If no result, mark as undefined
@@ -450,9 +529,9 @@ export class GeminiModelDefinition {
         return tokenUsage;
     }
     async requestTextCompletion(driver, prompt, options) {
-        const splits = options.model.split("/");
-        let region = undefined;
-        if (splits[0] === "locations" && splits.length >= 2) {
+        const splits = options.model.split('/');
+        let region;
+        if (splits[0] === 'locations' && splits.length >= 2) {
             region = splits[1];
         }
         const modelName = splits[splits.length - 1];
@@ -468,34 +547,37 @@ export class GeminiModelDefinition {
         let conversation = updateConversation(options.conversation, prompt.contents);
         prompt.contents = conversation;
         // TODO: Remove hack, use global endpoint manually if needed.
-        if (options.model.includes("gemini-2.5-flash-image")) {
-            region = "global"; // Gemini Flash Image only available in global region, this is for nano-banana model
+        if (options.model.includes('gemini-2.5-flash-image')) {
+            region = 'global'; // Gemini Flash Image only available in global region, this is for nano-banana model
         }
         const model_options = options.model_options;
-        const client = driver.getGoogleGenAIClient(region, model_options?.flex ?? false);
+        const client = driver.getGoogleGenAIClient(region, model_options?.flex ?? false, options.httpTimeout);
         const payload = getGeminiPayload(options, prompt);
         const response = await client.models.generateContent(payload);
         const token_usage = this.usageMetadataToTokenUsage(driver, response.usageMetadata);
         let tool_use;
         let finish_reason, result;
-        const candidate = response.candidates && response.candidates[0];
+        const candidate = response.candidates?.[0];
         if (candidate) {
             switch (candidate.finishReason) {
                 case FinishReason.MAX_TOKENS:
-                    finish_reason = "length";
+                    finish_reason = 'length';
                     break;
                 case FinishReason.STOP:
-                    finish_reason = "stop";
+                    finish_reason = 'stop';
                     break;
-                default: finish_reason = candidate.finishReason;
+                default:
+                    finish_reason = candidate.finishReason;
             }
             const content = candidate.content;
             // Check for unsupported finish reasons, but allow recoverable tool call issues
             const isRecoverableToolCall = recoverableToolCallReasons.includes(candidate.finishReason);
-            if (candidate.finishReason && !supportedFinishReasons.includes(candidate.finishReason) && !isRecoverableToolCall) {
-                throw new Error(`Unsupported finish reason: ${candidate.finishReason}, `
-                    + `finish message: ${candidate.finishMessage}, `
-                    + `content: ${JSON.stringify(content, null, 2)}, safety: ${JSON.stringify(candidate.safetyRatings, null, 2)}`);
+            if (candidate.finishReason &&
+                !supportedFinishReasons.includes(candidate.finishReason) &&
+                !isRecoverableToolCall) {
+                throw new Error(`Unsupported finish reason: ${candidate.finishReason}, ` +
+                    `finish message: ${candidate.finishMessage}, ` +
+                    `content: ${JSON.stringify(content, null, 2)}, safety: ${JSON.stringify(candidate.safetyRatings, null, 2)}`);
             }
             if (content) {
                 tool_use = collectToolUseParts(content);
@@ -506,14 +588,14 @@ export class GeminiModelDefinition {
                 // to ERROR severity for what is, by definition, a recoverable event.
                 if (isRecoverableToolCall && tool_use && tool_use.length > 0) {
                     driver.logger.warn(`[Gemini] Recoverable tool call issue (${candidate.finishReason}): ` +
-                        `Model tried to call undeclared tool(s): ${tool_use.map(t => t.tool_name).join(', ')}`);
+                        `Model tried to call undeclared tool(s): ${tool_use.map((t) => t.tool_name).join(', ')}`);
                 }
                 result = extractCompletionResults(content);
                 conversation = updateConversation(conversation, [content]);
             }
         }
         if (tool_use) {
-            finish_reason = "tool_use";
+            finish_reason = 'tool_use';
         }
         // Increment turn counter for deferred stripping
         conversation = incrementConversationTurn(conversation);
@@ -522,7 +604,7 @@ export class GeminiModelDefinition {
         const stripOptions = {
             keepForTurns: options.stripImagesAfterTurns ?? Infinity,
             currentTurn,
-            textMaxTokens: options.stripTextMaxTokens
+            textMaxTokens: options.stripTextMaxTokens,
         };
         let processedConversation = stripBase64ImagesFromConversation(conversation, stripOptions);
         // Truncate large text content if configured
@@ -535,18 +617,18 @@ export class GeminiModelDefinition {
         // Preserve system instruction in conversation for multi-turn support
         const finalConversation = storeSystemInConversation(processedConversation, prompt.system);
         return {
-            result: result && result.length > 0 ? result : [{ type: "text", value: '' }],
+            result: result && result.length > 0 ? result : [{ type: 'text', value: '' }],
             token_usage: token_usage,
             finish_reason: finish_reason,
             original_response: options.include_original_response ? response : undefined,
             conversation: finalConversation,
-            tool_use
+            tool_use,
         };
     }
     async requestTextCompletionStream(driver, prompt, options) {
-        const splits = options.model.split("/");
-        let region = undefined;
-        if (splits[0] === "locations" && splits.length >= 2) {
+        const splits = options.model.split('/');
+        let region;
+        if (splits[0] === 'locations' && splits.length >= 2) {
             region = splits[1];
         }
         const modelName = splits[splits.length - 1];
@@ -562,11 +644,11 @@ export class GeminiModelDefinition {
         // Include conversation history in prompt contents (same as non-streaming)
         const conversation = updateConversation(options.conversation, prompt.contents);
         prompt.contents = conversation;
-        if (options.model.includes("gemini-2.5-flash-image")) {
-            region = "global"; // Gemini Flash Image only available in global region, this is for nano-banana model
+        if (options.model.includes('gemini-2.5-flash-image')) {
+            region = 'global'; // Gemini Flash Image only available in global region, this is for nano-banana model
         }
         const model_options = options.model_options;
-        const client = driver.getGoogleGenAIClient(region, model_options?.flex ?? false);
+        const client = driver.getGoogleGenAIClient(region, model_options?.flex ?? false, options.httpTimeout);
         const payload = getGeminiPayload(options, prompt);
         const response = await client.models.generateContentStream(payload);
         const stream = asyncMap(response, async (item) => {
@@ -577,33 +659,36 @@ export class GeminiModelDefinition {
                     let finish_reason;
                     switch (candidate.finishReason) {
                         case FinishReason.MAX_TOKENS:
-                            finish_reason = "length";
+                            finish_reason = 'length';
                             break;
                         case FinishReason.STOP:
-                            finish_reason = "stop";
+                            finish_reason = 'stop';
                             break;
-                        default: finish_reason = candidate.finishReason;
+                        default:
+                            finish_reason = candidate.finishReason;
                     }
                     // Check for unsupported finish reasons, but allow recoverable tool call issues
                     const isRecoverableToolCall = recoverableToolCallReasons.includes(candidate.finishReason);
-                    if (candidate.finishReason && !supportedFinishReasons.includes(candidate.finishReason) && !isRecoverableToolCall) {
-                        throw new Error(`Unsupported finish reason: ${candidate.finishReason}, `
-                            + `finish message: ${candidate.finishMessage}, `
-                            + `content: ${JSON.stringify(candidate.content, null, 2)}, safety: ${JSON.stringify(candidate.safetyRatings, null, 2)}`);
+                    if (candidate.finishReason &&
+                        !supportedFinishReasons.includes(candidate.finishReason) &&
+                        !isRecoverableToolCall) {
+                        throw new Error(`Unsupported finish reason: ${candidate.finishReason}, ` +
+                            `finish message: ${candidate.finishMessage}, ` +
+                            `content: ${JSON.stringify(candidate.content, null, 2)}, safety: ${JSON.stringify(candidate.safetyRatings, null, 2)}`);
                     }
                     if (candidate.content?.role === 'model') {
                         // Collect all parts in order (text and images)
                         const combinedResults = extractCompletionResults(candidate.content);
                         tool_use = collectToolUseParts(candidate.content);
                         if (tool_use) {
-                            finish_reason = "tool_use";
+                            finish_reason = 'tool_use';
                             // Log warning for recoverable tool call issues — see the
                             // matching site in `requestTextCompletion` above for why
                             // we route through the driver's logger instead of
                             // `console.warn`.
                             if (isRecoverableToolCall) {
                                 driver.logger.warn(`[Gemini] Recoverable tool call issue (${candidate.finishReason}): ` +
-                                    `Model tried to call undeclared tool(s): ${tool_use.map(t => t.tool_name).join(', ')}`);
+                                    `Model tried to call undeclared tool(s): ${tool_use.map((t) => t.tool_name).join(', ')}`);
                             }
                         }
                         return {
@@ -617,8 +702,10 @@ export class GeminiModelDefinition {
             }
             //No normal output, returning block reason if it exists.
             return {
-                result: item.promptFeedback?.blockReasonMessage ? [{ type: "text", value: item.promptFeedback.blockReasonMessage }] : [],
-                finish_reason: item.promptFeedback?.blockReason ?? "",
+                result: item.promptFeedback?.blockReasonMessage
+                    ? [{ type: 'text', value: item.promptFeedback.blockReasonMessage }]
+                    : [],
+                finish_reason: item.promptFeedback?.blockReason ?? '',
                 token_usage: token_usage,
             };
         });
@@ -655,7 +742,7 @@ export class GeminiModelDefinition {
         const apiError = error;
         const httpStatusCode = apiError.status;
         // Extract error message
-        const message = apiError.message || String(error);
+        const message = apiError.message;
         // Build user-facing message with status code
         let userMessage = message;
         // Include status code in message (for end-user visibility)
@@ -663,7 +750,7 @@ export class GeminiModelDefinition {
             userMessage = `[${httpStatusCode}] ${userMessage}`;
         }
         // Determine retryability based on Google error codes
-        const retryable = this.isGeminiErrorRetryable(httpStatusCode);
+        const retryable = this.isGeminiErrorRetryable(httpStatusCode, message);
         // Extract error name/type from message if present
         const errorName = this.extractErrorName(message);
         return new LlumiverseError(`[${context.provider}] ${userMessage}`, retryable, context, error, httpStatusCode, errorName);
@@ -676,13 +763,15 @@ export class GeminiModelDefinition {
             typeof error === 'object' &&
             'status' in error &&
             typeof error.status === 'number' &&
-            'message' in error);
+            'message' in error &&
+            typeof error.message === 'string');
     }
     /**
      * Determine if a Google API error is retryable based on HTTP status code.
      *
      * Retryable errors (per Google AIP-194):
      * - 408 (REQUEST_TIMEOUT): Request timeout
+     * - 499 (CANCELLED / Client Closed Request): Transport cancellation
      * - 429 (RESOURCE_EXHAUSTED): Rate limit exceeded, quota exhausted
      * - 500 (INTERNAL): Internal server error
      * - 502 (BAD_GATEWAY): Bad gateway
@@ -697,15 +786,26 @@ export class GeminiModelDefinition {
      * - 409 (CONFLICT): Resource conflict
      * - Other 4xx client errors
      *
+     * Exception: certain 400s from Vertex AI's inline URL fetcher (used when
+     * passing a file by URL to multimodal models) surface as INVALID_ARGUMENT
+     * but are actually transient throttling/rate-limit signals on the
+     * fetcher, not a bad request. Detect those by message substring and
+     * treat them as retryable.
+     *
      * @param httpStatusCode - The HTTP status code from the API error
+     * @param message - The error message (used to detect transient 400 sub-cases)
      * @returns True if retryable, false if not retryable, undefined if unknown
      */
-    isGeminiErrorRetryable(httpStatusCode) {
+    isGeminiErrorRetryable(httpStatusCode, message) {
+        if (message && this.isNonRetryableAuthError(message))
+            return false;
         // Retryable status codes
         if (httpStatusCode === 408)
             return true; // Request timeout
         if (httpStatusCode === 429)
             return true; // Rate limit/quota
+        if (httpStatusCode === 499)
+            return true; // Client closed / operation cancelled
         if (httpStatusCode === 502)
             return true; // Bad gateway
         if (httpStatusCode === 503)
@@ -714,12 +814,32 @@ export class GeminiModelDefinition {
             return true; // Gateway timeout
         if (httpStatusCode >= 500 && httpStatusCode < 600)
             return true; // Other 5xx server errors
+        // Vertex AI URL fetcher transient throttling, surfaced as 400 INVALID_ARGUMENT
+        // but really a Google-side rate limit on the inline-content fetcher.
+        if (httpStatusCode === 400 && message) {
+            if (message.includes('URL_REJECTED-REJECTED_CLIENT_THROTTLED') ||
+                message.includes('URL_REJECTED-REJECTED_RATE_LIMITED')) {
+                return true;
+            }
+        }
+        // A transport-level abort/cancel (request-timeout / dropped connection, sometimes reported
+        // as 499 client-closed) or a deadline-exceeded is transient and should be retried,
+        // even though it carries a 4xx status. Honor it before the 4xx -> non-retryable rule.
+        if (message) {
+            const lower = message.toLowerCase();
+            if (lower.includes('aborted') || lower.includes('cancelled') || lower.includes('deadline'))
+                return true;
+        }
         // Non-retryable 4xx client errors
         if (httpStatusCode >= 400 && httpStatusCode < 500)
             return false;
         // Unknown status codes - let consumer decide retry strategy
         return undefined;
     }
+    isNonRetryableAuthError(message) {
+        const lowerMessage = message.toLowerCase();
+        return lowerMessage.includes('invalid_grant') || lowerMessage.includes("credential's issuer");
+    }
     /**
      * Extract error type name from error message.
      * Google errors often include the error type in the message.
@@ -728,6 +848,7 @@ export class GeminiModelDefinition {
     extractErrorName(message) {
         // Common Google error patterns
         const patterns = [
+            /^Error code ([a-zA-Z0-9_-]+):/, // "Error code invalid_grant: message"
             /^([A-Z_]+):/, // "ERROR_NAME: message"
             /\[([A-Z_]+)\]/, // "[ERROR_NAME] message"
             /^(\w+Error):/, // "ErrorTypeError: message"
@@ -747,22 +868,23 @@ export class GeminiModelDefinition {
  * tools/toolConfig to be defined in the API request.
  */
 export function convertGeminiFunctionPartsToText(contents) {
-    return contents.map(content => {
+    return contents.map((content) => {
         if (!content.parts)
             return content;
-        const hasFunctionParts = content.parts.some(p => p.functionCall || p.functionResponse);
+        const hasFunctionParts = content.parts.some((p) => p.functionCall || p.functionResponse);
         if (!hasFunctionParts)
             return content;
-        const newParts = content.parts.map(part => {
+        const newParts = content.parts.map((part) => {
             if (part.functionCall) {
                 const argsStr = part.functionCall.args ? JSON.stringify(part.functionCall.args) : '';
-                const truncated = argsStr.length > 500 ? argsStr.substring(0, 500) + '...' : argsStr;
+                const truncated = argsStr.length > 500 ? `${argsStr.substring(0, 500)}...` : argsStr;
                 return { text: `[Tool call: ${part.functionCall.name}(${truncated})]` };
             }
             if (part.functionResponse) {
                 const respStr = part.functionResponse.response
-                    ? JSON.stringify(part.functionResponse.response) : 'No response';
-                const truncated = respStr.length > 500 ? respStr.substring(0, 500) + '...' : respStr;
+                    ? JSON.stringify(part.functionResponse.response)
+                    : 'No response';
+                const truncated = respStr.length > 500 ? `${respStr.substring(0, 500)}...` : respStr;
                 return { text: `[Tool result for ${part.functionResponse.name}: ${truncated}]` };
             }
             return part;
@@ -843,11 +965,11 @@ function storeSystemInConversation(conversation, system) {
  */
 function formatFunctionResponse(response) {
     response = response.trim();
-    if (response.startsWith("{") && response.endsWith("}")) {
+    if (response.startsWith('{') && response.endsWith('}')) {
         try {
             return JSON.parse(response);
         }
-        catch (e) {
+        catch {
             return { output: response };
         }
     }