@juspay/neurolink 9.63.1 → 9.65.0

package/dist/providers/googleNativeGemini3.js

@@ -9,12 +9,69 @@
  * providers so they can share a single implementation.
  */
 import { randomUUID } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { extname } from "node:path";
 import { jsonSchema as aiJsonSchema, tool as createAISDKTool, } from "ai";
 import { DEFAULT_MAX_STEPS, DEFAULT_TOOL_MAX_RETRIES, } from "../core/constants.js";
 import { logger } from "../utils/logger.js";
-import { convertZodToJsonSchema, inlineJsonSchema, isZodSchema, normalizeJsonSchemaObject, } from "../utils/schemaConversion.js";
+import { convertZodToJsonSchema, ensureNestedSchemaTypes, inlineJsonSchema, isZodSchema, normalizeJsonSchemaObject, } from "../utils/schemaConversion.js";
 import { createNativeThinkingConfig } from "../utils/thinkingConfig.js";
 // ── Functions ──
+/**
+ * Google's `function_declarations[].name` validator regex.
+ *
+ * Empirically (and per the Vertex/AI Studio API error message), the server
+ * enforces `[A-Za-z_][A-Za-z0-9_.:-]{0,127}`. Tool names that don't match
+ * fail with HTTP 400 "Invalid function name", which surfaces as a misleading
+ * tool-calling failure for the whole request.
+ *
+ * MCP-imported or user-registered tools may legally contain characters
+ * outside this set (e.g. `/`, spaces, unicode), so we sanitize defensively
+ * before sending to Google. The sanitized name is also used as the
+ * `executeMap` key so the round-trip from Google's function-call response
+ * back to our executor still works.
+ */
+const GOOGLE_FN_NAME_REGEX = /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/;
+const GOOGLE_FN_NAME_MAX_LENGTH = 128;
+export function sanitizeForGoogleFunctionName(name) {
+    if (GOOGLE_FN_NAME_REGEX.test(name)) {
+        return name;
+    }
+    let sanitized = name.replace(/[^A-Za-z0-9_.:-]/g, "_");
+    if (!/^[A-Za-z_]/.test(sanitized)) {
+        sanitized = `_${sanitized}`;
+    }
+    if (sanitized.length > GOOGLE_FN_NAME_MAX_LENGTH) {
+        sanitized = sanitized.slice(0, GOOGLE_FN_NAME_MAX_LENGTH);
+    }
+    return sanitized;
+}
+/**
+ * Resolve a sanitized Gemini tool name to one that is both unique within
+ * the current request and at most 128 characters. When the candidate
+ * collides with an already-used name we append `_2`, `_3`, … — but
+ * reserve room for the suffix by truncating the base first so the
+ * resolved name never exceeds Google's `function_declarations[].name`
+ * limit.
+ *
+ * @param base       The already-sanitized candidate name.
+ * @param isTaken    Predicate that returns true if `name` is already used.
+ */
+export function resolveUniqueGoogleFunctionName(base, isTaken) {
+    if (!isTaken(base)) {
+        return base;
+    }
+    let suffix = 2;
+    while (true) {
+        const suffixStr = `_${suffix}`;
+        const trimmedBase = base.slice(0, GOOGLE_FN_NAME_MAX_LENGTH - suffixStr.length);
+        const candidate = `${trimmedBase}${suffixStr}`;
+        if (!isTaken(candidate)) {
+            return candidate;
+        }
+        suffix++;
+    }
+}
 /**
  * Sanitize a JSON Schema for Gemini's proto-based API.
  *
@@ -100,6 +157,40 @@ export function sanitizeSchemaForGemini(schema) {
             result[branch] = sanitizeSchemaForGemini(result[branch]);
         }
     }
+    // JSON Schema Draft-4 `exclusiveMinimum: true` / `exclusiveMaximum: true`
+    // (boolean form) is rejected by Gemini's OpenAPI 3.0 validator, which
+    // expects a numeric bound. zod-to-json-schema's openApi3 target still
+    // emits the Draft-4 form for `z.number().positive()` etc. Translate to
+    // the numeric form when paired with `minimum`/`maximum`, or drop.
+    if (typeof result.exclusiveMinimum === "boolean") {
+        if (result.exclusiveMinimum === true &&
+            typeof result.minimum === "number") {
+            result.exclusiveMinimum = result.minimum;
+            delete result.minimum;
+        }
+        else {
+            delete result.exclusiveMinimum;
+        }
+    }
+    if (typeof result.exclusiveMaximum === "boolean") {
+        if (result.exclusiveMaximum === true &&
+            typeof result.maximum === "number") {
+            result.exclusiveMaximum = result.maximum;
+            delete result.maximum;
+        }
+        else {
+            delete result.exclusiveMaximum;
+        }
+    }
+    // Clamp `maximum`/`minimum` past int32 — Gemini's protobuf serializer
+    // treats `type: "integer"` as int32 and rejects bounds beyond ~2.1e9.
+    const INT32_MAX = 2147483647;
+    if (typeof result.maximum === "number" && result.maximum > INT32_MAX) {
+        delete result.maximum;
+    }
+    if (typeof result.minimum === "number" && result.minimum < -INT32_MAX) {
+        delete result.minimum;
+    }
     return result;
 }
 /**
@@ -115,8 +206,27 @@ export function sanitizeSchemaForGemini(schema) {
 export function sanitizeToolsForGemini(tools) {
     const sanitized = {};
     const dropped = [];
+    const renamed = [];
+    const originalNameMap = new Map();
     for (const [name, tool] of Object.entries(tools)) {
         try {
+            // Sanitize the tool name to fit Google's function_declarations regex.
+            // Without this, MCP-imported or user-registered tools whose names contain
+            // characters outside [A-Za-z_][A-Za-z0-9_.:-]{0,127} cause the entire
+            // request to 400 with "Invalid function name", surfacing as a misleading
+            // tool-calling failure. Distinct originals that collapse onto the same
+            // sanitized name (e.g. "my/tool" and "my-tool" → "my_tool") are
+            // disambiguated with a numeric suffix that preserves Google's 128-char
+            // ceiling.
+            const candidate = sanitizeForGoogleFunctionName(name);
+            const safeName = resolveUniqueGoogleFunctionName(candidate, (n) => n in sanitized);
+            // Always record the mapping so downstream code can translate every
+            // safeName back to the original — including the no-rename identity
+            // mapping, which simplifies the lookup path.
+            originalNameMap.set(safeName, name);
+            if (safeName !== name) {
+                renamed.push({ from: name, to: safeName });
+            }
             // Access the legacy `parameters` field that may exist on older AI SDK tools.
             // AI SDK v6 uses `inputSchema`, but v3/v4 tools and third-party wrappers use `parameters`.
             const legacyTool = tool;
@@ -125,15 +235,15 @@ export function sanitizeToolsForGemini(tools) {
                 typeof params === "object" &&
                 "_def" in params &&
                 typeof params.parse === "function") {
-                const rawJsonSchema = convertZodToJsonSchema(params);
+                const rawJsonSchema = convertZodToJsonSchema(params, "openApi3");
                 const inlined = inlineJsonSchema(rawJsonSchema);
                 // Gemini sanitization strips Zod-only features not supported by the Gemini API:
                 // union types (anyOf/oneOf) are collapsed to string, default values and
                 // additionalProperties are removed. The resulting schema is Gemini-compatible
                 // but loses some type constraints from the original Zod schema.
                 const sanitizedSchema = sanitizeSchemaForGemini(inlined);
-                sanitized[name] = createAISDKTool({
-                    description: tool.description || `Tool: ${name}`,
+                sanitized[safeName] = createAISDKTool({
+                    description: tool.description || `Tool: ${safeName}`,
                     inputSchema: aiJsonSchema(sanitizedSchema),
                     execute: tool.execute,
                 });
@@ -145,14 +255,14 @@ export function sanitizeToolsForGemini(tools) {
                 const rawSchema = params
                     .jsonSchema;
                 const sanitizedSchema = sanitizeSchemaForGemini(inlineJsonSchema(rawSchema));
-                sanitized[name] = createAISDKTool({
-                    description: tool.description || `Tool: ${name}`,
+                sanitized[safeName] = createAISDKTool({
+                    description: tool.description || `Tool: ${safeName}`,
                     inputSchema: aiJsonSchema(sanitizedSchema),
                     execute: tool.execute,
                 });
             }
             else {
-                sanitized[name] = tool;
+                sanitized[safeName] = tool;
             }
         }
         catch (error) {
@@ -161,7 +271,12 @@ export function sanitizeToolsForGemini(tools) {
             dropped.push(name);
         }
     }
-    return { tools: sanitized, dropped };
+    if (renamed.length > 0) {
+        logger.warn(`[Gemini] ${renamed.length} tool name(s) sanitized for Google's function-name regex: ${renamed
+            .map((r) => `"${r.from}" -> "${r.to}"`)
+            .join(", ")}`);
+    }
+    return { tools: sanitized, dropped, originalNameMap };
 }
 export function normalizeToolsForJsonSchemaProvider(tools) {
     const normalizedTools = {};
@@ -171,7 +286,7 @@ export function normalizeToolsForJsonSchemaProvider(tools) {
         const toolParams = legacyTool.parameters || tool.inputSchema;
         let rawSchema;
         if (isZodSchema(toolParams)) {
-            rawSchema = convertZodToJsonSchema(toolParams);
+            rawSchema = convertZodToJsonSchema(toolParams, "openApi3");
         }
         else if (toolParams && typeof toolParams === "object") {
             rawSchema = toolParams;
@@ -210,11 +325,29 @@ export function buildNativeToolDeclarations(tools) {
     const functionDeclarations = [];
     const executeMap = new Map();
     const skippedTools = [];
+    const renamedTools = [];
+    // Disambiguate distinct originals that collapse onto the same sanitized
+    // name (e.g. "my/tool" and "my-tool" both → "my_tool") via
+    // resolveUniqueGoogleFunctionName, which appends `_N` while keeping the
+    // final string within Google's 128-char limit. Track all assigned names
+    // regardless of whether the tool has an `execute` function (tools without
+    // execute are still pushed to functionDeclarations). The originalNameMap
+    // lets the calling stream loop translate Google-returned function-call
+    // names back to the consumer-facing identifier so the sanitization is
+    // transport-only.
+    const usedNames = new Set();
+    const originalNameMap = new Map();
     for (const [name, tool] of Object.entries(tools)) {
         try {
+            const candidate = sanitizeForGoogleFunctionName(name);
+            const safeName = resolveUniqueGoogleFunctionName(candidate, (n) => usedNames.has(n));
+            originalNameMap.set(safeName, name);
+            if (safeName !== name) {
+                renamedTools.push({ from: name, to: safeName });
+            }
             const decl = {
-                name,
-                description: tool.description || `Tool: ${name}`,
+                name: safeName,
+                description: tool.description || `Tool: ${safeName}`,
             };
             // Access legacy `parameters` (AI SDK v3/v4) or current `inputSchema` (v6)
             const legacyTool = tool;
@@ -222,7 +355,7 @@ export function buildNativeToolDeclarations(tools) {
                 let rawSchema;
                 const toolParams = legacyTool.parameters || tool.inputSchema;
                 if (isZodSchema(toolParams)) {
-                    rawSchema = convertZodToJsonSchema(toolParams);
+                    rawSchema = convertZodToJsonSchema(toolParams, "openApi3");
                 }
                 else if (typeof toolParams === "object") {
                     rawSchema = toolParams;
@@ -239,8 +372,9 @@ export function buildNativeToolDeclarations(tools) {
                 decl.parametersJsonSchema = sanitizeSchemaForGemini(inlineJsonSchema(rawSchema));
             }
             functionDeclarations.push(decl);
+            usedNames.add(safeName);
             if (tool.execute) {
-                executeMap.set(name, tool.execute);
+                executeMap.set(decl.name, tool.execute);
             }
         }
         catch (err) {
@@ -251,10 +385,27 @@ export function buildNativeToolDeclarations(tools) {
     if (skippedTools.length > 0) {
         logger.warn(`[buildNativeToolDeclarations] ${skippedTools.length} tool(s) skipped due to schema errors: ${skippedTools.join(", ")}`);
     }
-    return { toolsConfig: [{ functionDeclarations }], executeMap };
+    if (renamedTools.length > 0) {
+        logger.warn(`[buildNativeToolDeclarations] ${renamedTools.length} tool name(s) sanitized for Google's function-name regex: ${renamedTools
+            .map((r) => `"${r.from}" -> "${r.to}"`)
+            .join(", ")}`);
+    }
+    return {
+        toolsConfig: [{ functionDeclarations }],
+        executeMap,
+        originalNameMap,
+    };
 }
 /**
  * Build the native @google/genai config object shared by stream and generate.
+ *
+ * Caller is responsible for the tools-vs-JSON conflict resolution: Gemini's
+ * function calling cannot be combined with `responseMimeType:
+ * "application/json"`, and `responseSchema` requires that mime type. So
+ * when tools are active, callers must NOT pass `wantsJsonOutput`/
+ * `responseSchema` here; when JSON/schema output is requested, callers
+ * must omit `toolsConfig`. The AI Studio path enforces this by forcing
+ * `disableTools: true` whenever JSON/schema output is requested.
  */
 export function buildNativeConfig(options, toolsConfig) {
     const config = {
@@ -272,6 +423,16 @@ export function buildNativeConfig(options, toolsConfig) {
     if (nativeThinkingConfig) {
         config.thinkingConfig = nativeThinkingConfig;
     }
+    // Native JSON / schema enforcement. Only set when tools are NOT being sent
+    // (Gemini rejects the combination). responseSchema implies JSON mime type.
+    if (!toolsConfig) {
+        if (options.responseSchema || options.wantsJsonOutput) {
+            config.responseMimeType = "application/json";
+        }
+        if (options.responseSchema) {
+            config.responseSchema = options.responseSchema;
+        }
+    }
     return config;
 }
 /**
@@ -473,27 +634,38 @@ export function extractTextFromParts(rawResponseParts) {
  * @param executeMap - Map of tool name to execute function
  * @param failedTools - Mutable map tracking per-tool failure counts
  * @param allToolCalls - Mutable array accumulating all tool call records
- * @param options - Optional settings for execution tracking and cancellation
+ * @param options - Optional settings for execution tracking and cancellation,
+ *                  plus an `originalNameMap` (Google-safe → consumer-supplied
+ *                  identifier) so the sanitization stays transport-only and
+ *                  consumers see the names they registered.
  * @returns Array of function responses for conversation history
  */
 export async function executeNativeToolCalls(logLabel, stepFunctionCalls, executeMap, failedTools, allToolCalls, options) {
     const functionResponses = [];
+    // Translate a Google-safe sanitized name back to the consumer-facing
+    // original name. Falls back to the safe name if the map is missing or
+    // doesn't contain the call (e.g. tool added mid-conversation).
+    const externalName = (safeName) => options?.originalNameMap?.get(safeName) ?? safeName;
     for (const call of stepFunctionCalls) {
-        allToolCalls.push({ toolName: call.name, args: call.args });
+        const exposedName = externalName(call.name);
+        allToolCalls.push({ toolName: exposedName, args: call.args });
         // Check if this tool has already exceeded retry limit
         const failedInfo = failedTools.get(call.name);
         if (failedInfo && failedInfo.count >= DEFAULT_TOOL_MAX_RETRIES) {
-            logger.warn(`${logLabel} Tool "${call.name}" has exceeded retry limit (${DEFAULT_TOOL_MAX_RETRIES}), skipping execution`);
+            logger.warn(`${logLabel} Tool "${exposedName}" has exceeded retry limit (${DEFAULT_TOOL_MAX_RETRIES}), skipping execution`);
             const errorOutput = {
-                error: `TOOL_PERMANENTLY_FAILED: The tool "${call.name}" has failed ${failedInfo.count} times and will not be retried. Last error: ${failedInfo.lastError}. Please proceed without using this tool or inform the user that this functionality is unavailable.`,
+                error: `TOOL_PERMANENTLY_FAILED: The tool "${exposedName}" has failed ${failedInfo.count} times and will not be retried. Last error: ${failedInfo.lastError}. Please proceed without using this tool or inform the user that this functionality is unavailable.`,
                 status: "permanently_failed",
                 do_not_retry: true,
             };
+            // Wire transport-side `name: call.name` (Google needs the sanitized
+            // form to match the function declaration) while exposing the
+            // consumer-facing name in toolExecutions metadata.
             functionResponses.push({
                 functionResponse: { name: call.name, response: errorOutput },
             });
             options?.toolExecutions?.push({
-                name: call.name,
+                name: exposedName,
                 input: call.args,
                 output: errorOutput,
             });
@@ -514,7 +686,7 @@ export async function executeNativeToolCalls(logLabel, stepFunctionCalls, execut
                     functionResponse: { name: call.name, response: { result } },
                 });
                 options?.toolExecutions?.push({
-                    name: call.name,
+                    name: exposedName,
                     input: call.args,
                     output: result,
                 });
@@ -529,12 +701,12 @@ export async function executeNativeToolCalls(logLabel, stepFunctionCalls, execut
                 currentFailInfo.count++;
                 currentFailInfo.lastError = errorMessage;
                 failedTools.set(call.name, currentFailInfo);
-                logger.warn(`${logLabel} Tool "${call.name}" failed (attempt ${currentFailInfo.count}/${DEFAULT_TOOL_MAX_RETRIES}): ${errorMessage}`);
+                logger.warn(`${logLabel} Tool "${exposedName}" failed (attempt ${currentFailInfo.count}/${DEFAULT_TOOL_MAX_RETRIES}): ${errorMessage}`);
                 // Determine if this is a permanent failure
                 const isPermanentFailure = currentFailInfo.count >= DEFAULT_TOOL_MAX_RETRIES;
                 const errorOutput = {
                     error: isPermanentFailure
-                        ? `TOOL_PERMANENTLY_FAILED: The tool "${call.name}" has failed ${currentFailInfo.count} times with error: ${errorMessage}. This tool will not be retried. Please proceed without using this tool or inform the user that this functionality is unavailable.`
+                        ? `TOOL_PERMANENTLY_FAILED: The tool "${exposedName}" has failed ${currentFailInfo.count} times with error: ${errorMessage}. This tool will not be retried. Please proceed without using this tool or inform the user that this functionality is unavailable.`
                         : `TOOL_EXECUTION_ERROR: ${errorMessage}. Retry attempt ${currentFailInfo.count}/${DEFAULT_TOOL_MAX_RETRIES}.`,
                     status: isPermanentFailure ? "permanently_failed" : "failed",
                     do_not_retry: isPermanentFailure,
@@ -545,7 +717,7 @@ export async function executeNativeToolCalls(logLabel, stepFunctionCalls, execut
                     functionResponse: { name: call.name, response: errorOutput },
                 });
                 options?.toolExecutions?.push({
-                    name: call.name,
+                    name: exposedName,
                     input: call.args,
                     output: errorOutput,
                 });
@@ -554,7 +726,7 @@ export async function executeNativeToolCalls(logLabel, stepFunctionCalls, execut
         else {
             // Tool not found is a permanent error
             const errorOutput = {
-                error: `TOOL_NOT_FOUND: The tool "${call.name}" does not exist. Do not attempt to call this tool again.`,
+                error: `TOOL_NOT_FOUND: The tool "${exposedName}" does not exist. Do not attempt to call this tool again.`,
                 status: "permanently_failed",
                 do_not_retry: true,
             };
@@ -562,7 +734,7 @@ export async function executeNativeToolCalls(logLabel, stepFunctionCalls, execut
                 functionResponse: { name: call.name, response: errorOutput },
             });
             options?.toolExecutions?.push({
-                name: call.name,
+                name: exposedName,
                 input: call.args,
                 output: errorOutput,
             });
@@ -597,3 +769,187 @@ export function pushModelResponseToHistory(currentContents, rawResponseParts, st
             : stepFunctionCalls.map((fc) => ({ functionCall: fc })),
     });
 }
+/**
+ * Convert a Zod schema (or AI SDK `jsonSchema()` wrapper) into the shape
+ * `@google/genai` accepts as `responseSchema`. Mirrors the inline pipeline
+ * the Vertex Gemini paths already use:
+ *
+ *   convertZodToJsonSchema → inlineJsonSchema → strip `$schema` → ensure
+ *   every nested schema has a `type` (Vertex/Gemini reject schemas missing
+ *   that field, even on nested objects).
+ *
+ * Lives here so the AI Studio and Vertex paths can share the same
+ * sanitization without duplicating the schema-conversion churn.
+ */
+export function buildGeminiResponseSchema(schema) {
+    const raw = convertZodToJsonSchema(schema, "openApi3");
+    const inlined = inlineJsonSchema(raw);
+    if (inlined.$schema) {
+        delete inlined.$schema;
+    }
+    return ensureNestedSchemaTypes(inlined);
+}
+/**
+ * Map NeuroLink ChatMessage[] history into the @google/genai content format
+ * and push the entries onto a contents array.
+ *
+ * Used by the native Vertex Gemini and Google AI Studio paths to honor
+ * `options.conversationMessages` so multi-turn conversations (memory, loop
+ * REPL, agent flows) actually carry prior turns into the request.
+ *
+ * Behavior notes:
+ *  - Only `user` and `assistant` roles are forwarded; system messages are
+ *    expected to be wired via `systemInstruction`, and tool-call /
+ *    tool-result roles only appear inside intra-call tool loops which build
+ *    their own model/function entries.
+ *  - String content is wrapped as a single `{ text }` part. Empty strings
+ *    are skipped to avoid sending empty parts that some Gemini regions
+ *    reject.
+ *  - The current user input should be appended AFTER calling this helper
+ *    so the prior turns appear first in chronological order.
+ */
+export function prependConversationMessages(contents, conversationMessages) {
+    if (!conversationMessages || conversationMessages.length === 0) {
+        return;
+    }
+    for (const msg of conversationMessages) {
+        if (msg.role !== "user" && msg.role !== "assistant") {
+            continue;
+        }
+        const text = typeof msg.content === "string" ? msg.content : "";
+        if (text.length === 0) {
+            continue;
+        }
+        contents.push({
+            role: msg.role === "assistant" ? "model" : "user",
+            parts: [{ text }],
+        });
+    }
+}
+/**
+ * Build the `parts` array for the current user turn of a Gemini native
+ * `generateContent` request, including inline image + PDF blobs.
+ *
+ * Both providers that hit the native `@google/genai` SDK — `GoogleVertex`
+ * and `GoogleAIStudio` — need this. The previous AI Studio code only
+ * pushed a single `{ text }` part, which silently dropped `input.images`
+ * and `input.pdfFiles` on the floor: the model received text only and
+ * legitimately reported "no image attached". Extracting this from the
+ * Vertex copy keeps both providers on one definition.
+ *
+ * Accepted shapes per element (mirroring the runtime behaviour the Vertex
+ * code already supported):
+ *   - `Buffer` → used as-is
+ *   - local file path → read via `readFileSync`, MIME guessed from extension
+ *   - `data:<mime>;base64,...` URL → mime parsed, data base64-decoded
+ *   - `http(s)://...` URL → fetched, mime from `content-type`
+ *   - any other string → assumed to be a base64-encoded payload
+ *
+ * Image MIME guessing is conservative — only known extensions override the
+ * default `image/jpeg`. Fetch failures are logged and the offending entry
+ * is skipped rather than aborting the entire request, matching prior
+ * Vertex behaviour.
+ */
+export async function buildUserPartsWithMultimodal(input, textOverride, logPrefix = "[GeminiNative]") {
+    const text = typeof textOverride === "string" ? textOverride : (input?.text ?? "");
+    const parts = [{ text }];
+    if (input?.pdfFiles && input.pdfFiles.length > 0) {
+        logger.debug(`${logPrefix} Processing ${input.pdfFiles.length} PDF(s)`);
+        for (const pdfFile of input.pdfFiles) {
+            let pdfBuffer;
+            if (typeof pdfFile === "string") {
+                if (existsSync(pdfFile)) {
+                    pdfBuffer = readFileSync(pdfFile);
+                }
+                else {
+                    // Treat as already-base64-encoded payload
+                    pdfBuffer = Buffer.from(pdfFile, "base64");
+                }
+            }
+            else {
+                pdfBuffer = pdfFile;
+            }
+            parts.push({
+                inlineData: {
+                    mimeType: "application/pdf",
+                    data: pdfBuffer.toString("base64"),
+                },
+            });
+        }
+    }
+    if (input?.images && input.images.length > 0) {
+        logger.debug(`${logPrefix} Processing ${input.images.length} image(s)`);
+        for (const rawImage of input.images) {
+            // `images` may carry plain Buffer/string values or `{ data, altText? }`
+            // objects. Normalise to the inner payload before format detection.
+            const image = rawImage && typeof rawImage === "object" && !Buffer.isBuffer(rawImage)
+                ? rawImage.data
+                : rawImage;
+            let imageBuffer;
+            let mimeType = "image/jpeg";
+            if (typeof image === "string") {
+                if (existsSync(image)) {
+                    imageBuffer = readFileSync(image);
+                    const ext = extname(image).toLowerCase();
+                    if (ext === ".png") {
+                        mimeType = "image/png";
+                    }
+                    else if (ext === ".gif") {
+                        mimeType = "image/gif";
+                    }
+                    else if (ext === ".webp") {
+                        mimeType = "image/webp";
+                    }
+                }
+                else if (image.startsWith("data:")) {
+                    const matches = image.match(/^data:([^;]+);base64,(.+)$/);
+                    if (matches) {
+                        mimeType = matches[1];
+                        imageBuffer = Buffer.from(matches[2], "base64");
+                    }
+                    else {
+                        continue;
+                    }
+                }
+                else if (image.startsWith("http://") ||
+                    image.startsWith("https://")) {
+                    try {
+                        const response = await fetch(image);
+                        if (!response.ok) {
+                            logger.warn(`${logPrefix} Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: image });
+                            continue;
+                        }
+                        const arrayBuffer = await response.arrayBuffer();
+                        imageBuffer = Buffer.from(arrayBuffer);
+                        const headerMime = response.headers.get("content-type");
+                        if (headerMime && headerMime.startsWith("image/")) {
+                            mimeType = headerMime.split(";")[0];
+                        }
+                    }
+                    catch (fetchError) {
+                        logger.warn(`${logPrefix} Image URL fetch threw, skipping: ${fetchError instanceof Error
+                            ? fetchError.message
+                            : String(fetchError)}`, { url: image });
+                        continue;
+                    }
+                }
+                else {
+                    imageBuffer = Buffer.from(image, "base64");
+                }
+            }
+            else {
+                imageBuffer = image;
+            }
+            if (!imageBuffer) {
+                continue;
+            }
+            parts.push({
+                inlineData: {
+                    mimeType,
+                    data: imageBuffer.toString("base64"),
+                },
+            });
+        }
+    }
+    return parts;
+}