@juspay/neurolink 7.46.0 → 7.47.0

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [7.47.0](https://github.com/juspay/neurolink/compare/v7.46.0...v7.47.0) (2025-09-25)
+
+### Features
+
+- **(chat):** Implement multimodal UI and extend SDK support ([12a2f59](https://github.com/juspay/neurolink/commit/12a2f59c4826e82ab1feb1347d08980682748ad2))
+
 ## [7.46.0](https://github.com/juspay/neurolink/compare/v7.45.0...v7.46.0) (2025-09-24)
 
 ### Features

package/dist/adapters/providerImageAdapter.js

@@ -34,6 +34,14 @@ const VISION_CAPABILITIES = {
         "claude-3-sonnet",
         "claude-3-haiku",
     ],
+    azure: [
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4-turbo",
+        "gpt-4-vision-preview",
+        "gpt-4.1",
+        "gpt-4",
+    ],
     vertex: [
         // Gemini models on Vertex AI
         "gemini-2.5-pro",
@@ -78,6 +86,10 @@ export class ProviderImageAdapter {
                 case "openai":
                     adaptedPayload = this.formatForOpenAI(text, images);
                     break;
+                case "azure":
+                case "azure-openai":
+                    adaptedPayload = this.formatForOpenAI(text, images);
+                    break;
                 case "google-ai":
                 case "google":
                     adaptedPayload = this.formatForGoogleAI(text, images);

package/dist/core/constants.js

@@ -51,7 +51,7 @@ export const PROVIDER_MAX_TOKENS = {
         default: 64000,
     },
     azure: {
-        default: 64000,
+        default: 32000,
     },
     ollama: {
         default: 64000,

package/dist/lib/adapters/providerImageAdapter.js

@@ -34,6 +34,14 @@ const VISION_CAPABILITIES = {
         "claude-3-sonnet",
         "claude-3-haiku",
     ],
+    azure: [
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4-turbo",
+        "gpt-4-vision-preview",
+        "gpt-4.1",
+        "gpt-4",
+    ],
     vertex: [
         // Gemini models on Vertex AI
         "gemini-2.5-pro",
@@ -78,6 +86,10 @@ export class ProviderImageAdapter {
                 case "openai":
                     adaptedPayload = this.formatForOpenAI(text, images);
                     break;
+                case "azure":
+                case "azure-openai":
+                    adaptedPayload = this.formatForOpenAI(text, images);
+                    break;
                 case "google-ai":
                 case "google":
                     adaptedPayload = this.formatForGoogleAI(text, images);

package/dist/lib/core/constants.js

@@ -51,7 +51,7 @@ export const PROVIDER_MAX_TOKENS = {
         default: 64000,
     },
     azure: {
-        default: 64000,
+        default: 32000,
     },
     ollama: {
         default: 64000,

package/dist/lib/providers/azureOpenai.js

@@ -4,7 +4,7 @@ import { BaseProvider } from "../core/baseProvider.js";
 import { APIVersions } from "../types/providers.js";
 import { validateApiKey, createAzureAPIKeyConfig, createAzureEndpointConfig, } from "../utils/providerConfig.js";
 import { logger } from "../utils/logger.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 export class AzureOpenAIProvider extends BaseProvider {
@@ -109,8 +109,41 @@ export class AzureOpenAIProvider extends BaseProvider {
                     })),
                 });
             }
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Azure OpenAI: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Azure OpenAI: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options);
             const stream = await streamText({
                 model,

package/dist/lib/providers/googleAiStudio.js

@@ -7,8 +7,9 @@ import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/errors.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 // Google AI Live API types now imported from ../types/providerSpecific.js
+// Import proper types for multimodal message handling
 // Create Google GenAI client
 async function createGoogleGenAIClient(apiKey) {
     const mod = await import("@google/genai");
@@ -90,8 +91,41 @@ export class GoogleAIStudioProvider extends BaseProvider {
             // Get tools consistently with generate method
             const shouldUseTools = !options.disableTools && this.supportsTools();
             const tools = shouldUseTools ? await this.getAllTools() : {};
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Google AI Studio: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Google AI Studio: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const result = await streamText({
                 model,
                 messages: messages,

package/dist/lib/providers/googleVertex.js

@@ -11,8 +11,9 @@ import fs from "fs";
 import path from "path";
 import os from "os";
 import dns from "dns";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
+// Import proper types for multimodal message handling
 // Enhanced Anthropic support with direct imports
 // Using the dual provider architecture from Vercel AI SDK
 const hasAnthropicSupport = () => {
@@ -594,8 +595,41 @@ export class GoogleVertexProvider extends BaseProvider {
         try {
             // Validate stream options
             this.validateStreamOptionsOnly(options);
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`${functionTag}: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`${functionTag}: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options); // This is where network connection happens!
             // Get all available tools (direct + MCP + external) for streaming
             const shouldUseTools = !options.disableTools && this.supportsTools();

package/dist/lib/utils/imageProcessor.d.ts

@@ -81,4 +81,48 @@ export declare const imageUtils: {
      * Convert file size to human readable format
      */
     formatFileSize: (bytes: number) => string;
+    /**
+     * Convert Buffer to base64 string
+     */
+    bufferToBase64: (buffer: Buffer) => string;
+    /**
+     * Convert base64 string to Buffer
+     */
+    base64ToBuffer: (base64: string) => Buffer;
+    /**
+     * Convert file path to base64 data URI
+     */
+    fileToBase64DataUri: (filePath: string, maxBytes?: number) => Promise<string>;
+    /**
+     * Convert URL to base64 data URI by downloading the image
+     */
+    urlToBase64DataUri: (url: string, { timeoutMs, maxBytes }?: {
+        timeoutMs?: number | undefined;
+        maxBytes?: number | undefined;
+    }) => Promise<string>;
+    /**
+     * Extract base64 data from data URI
+     */
+    extractBase64FromDataUri: (dataUri: string) => string;
+    /**
+     * Extract MIME type from data URI
+     */
+    extractMimeTypeFromDataUri: (dataUri: string) => string;
+    /**
+     * Create data URI from base64 and MIME type
+     */
+    createDataUri: (base64: string, mimeType?: string) => string;
+    /**
+     * Validate base64 string format
+     */
+    isValidBase64: (str: string) => boolean;
+    /**
+     * Get base64 string size in bytes
+     */
+    getBase64Size: (base64: string) => number;
+    /**
+     * Compress base64 image by reducing quality (basic implementation)
+     * Note: This is a placeholder - for production use, consider using sharp or similar
+     */
+    compressBase64: (base64: string, _quality?: number) => string;
 };

package/dist/lib/utils/imageProcessor.js

@@ -151,6 +151,8 @@ export class ImageProcessor {
                     bmp: "image/bmp",
                     tiff: "image/tiff",
                     tif: "image/tiff",
+                    svg: "image/svg+xml",
+                    avif: "image/avif",
                 };
                 return imageTypes[extension || ""] || "image/jpeg";
             }
@@ -183,6 +185,21 @@ export class ImageProcessor {
                         return "image/webp";
                     }
                 }
+                // SVG: check for "<svg" or "<?xml" at start (text-based)
+                if (input.length >= 4) {
+                    const start = input.subarray(0, 4).toString();
+                    if (start === "<svg" || start === "<?xm") {
+                        return "image/svg+xml";
+                    }
+                }
+                // AVIF: check for "ftypavif" signature at bytes 4-11
+                if (input.length >= 12) {
+                    const ftyp = input.subarray(4, 8).toString();
+                    const brand = input.subarray(8, 12).toString();
+                    if (ftyp === "ftyp" && brand === "avif") {
+                        return "image/avif";
+                    }
+                }
             }
             return "image/jpeg"; // Default fallback
         }
@@ -217,6 +234,8 @@ export class ImageProcessor {
             "image/webp",
             "image/bmp",
             "image/tiff",
+            "image/svg+xml",
+            "image/avif",
         ];
         return supportedFormats.includes(mediaType.toLowerCase());
     }
@@ -332,14 +351,7 @@ export const imageUtils = {
     /**
      * Check if a string is base64 encoded
      */
-    isBase64: (str) => {
-        try {
-            return btoa(atob(str)) === str;
-        }
-        catch {
-            return false;
-        }
-    },
+    isBase64: (str) => imageUtils.isValidBase64(str),
     /**
      * Extract file extension from filename or URL
      */
@@ -359,4 +371,143 @@ export const imageUtils = {
         const i = Math.floor(Math.log(bytes) / Math.log(k));
         return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
     },
+    /**
+     * Convert Buffer to base64 string
+     */
+    bufferToBase64: (buffer) => {
+        return buffer.toString("base64");
+    },
+    /**
+     * Convert base64 string to Buffer
+     */
+    base64ToBuffer: (base64) => {
+        // Remove data URI prefix if present
+        const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
+        return Buffer.from(cleanBase64, "base64");
+    },
+    /**
+     * Convert file path to base64 data URI
+     */
+    fileToBase64DataUri: async (filePath, maxBytes = 10 * 1024 * 1024) => {
+        try {
+            const fs = await import("fs/promises");
+            // File existence and type validation
+            const stat = await fs.stat(filePath);
+            if (!stat.isFile()) {
+                throw new Error("Not a file");
+            }
+            // Size check before reading - prevent memory exhaustion
+            if (stat.size > maxBytes) {
+                throw new Error(`File too large: ${stat.size} bytes (max: ${maxBytes} bytes)`);
+            }
+            const buffer = await fs.readFile(filePath);
+            // Enhanced MIME detection: try buffer content first, fallback to filename
+            const mimeType = ImageProcessor.detectImageType(buffer) ||
+                ImageProcessor.detectImageType(filePath);
+            const base64 = buffer.toString("base64");
+            return `data:${mimeType};base64,${base64}`;
+        }
+        catch (error) {
+            throw new Error(`Failed to convert file to base64: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    },
+    /**
+     * Convert URL to base64 data URI by downloading the image
+     */
+    urlToBase64DataUri: async (url, { timeoutMs = 15000, maxBytes = 10 * 1024 * 1024 } = {}) => {
+        try {
+            // Basic protocol whitelist
+            if (!/^https?:\/\//i.test(url)) {
+                throw new Error("Unsupported protocol");
+            }
+            const controller = new AbortController();
+            const t = setTimeout(() => controller.abort(), timeoutMs);
+            try {
+                const response = await fetch(url, { signal: controller.signal });
+                if (!response.ok) {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                }
+                const contentType = response.headers.get("content-type") || "";
+                if (!/^image\//i.test(contentType)) {
+                    throw new Error(`Unsupported content-type: ${contentType || "unknown"}`);
+                }
+                const len = Number(response.headers.get("content-length") || 0);
+                if (len && len > maxBytes) {
+                    throw new Error(`Content too large: ${len} bytes`);
+                }
+                const buffer = await response.arrayBuffer();
+                if (buffer.byteLength > maxBytes) {
+                    throw new Error(`Downloaded content too large: ${buffer.byteLength} bytes`);
+                }
+                const base64 = Buffer.from(buffer).toString("base64");
+                return `data:${contentType || "image/jpeg"};base64,${base64}`;
+            }
+            finally {
+                clearTimeout(t);
+            }
+        }
+        catch (error) {
+            throw new Error(`Failed to download and convert URL to base64: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    },
+    /**
+     * Extract base64 data from data URI
+     */
+    extractBase64FromDataUri: (dataUri) => {
+        if (!dataUri.includes(",")) {
+            return dataUri; // Already just base64
+        }
+        return dataUri.split(",")[1];
+    },
+    /**
+     * Extract MIME type from data URI
+     */
+    extractMimeTypeFromDataUri: (dataUri) => {
+        const match = dataUri.match(/^data:([^;]+);base64,/);
+        return match ? match[1] : "image/jpeg";
+    },
+    /**
+     * Create data URI from base64 and MIME type
+     */
+    createDataUri: (base64, mimeType = "image/jpeg") => {
+        // Remove data URI prefix if already present
+        const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
+        return `data:${mimeType};base64,${cleanBase64}`;
+    },
+    /**
+     * Validate base64 string format
+     */
+    isValidBase64: (str) => {
+        try {
+            // Remove data URI prefix if present
+            const cleanBase64 = str.includes(",") ? str.split(",")[1] : str;
+            // Check if it's valid base64
+            const decoded = Buffer.from(cleanBase64, "base64");
+            const reencoded = decoded.toString("base64");
+            // Remove padding for comparison (base64 can have different padding)
+            const normalizeBase64 = (b64) => b64.replace(/=+$/, "");
+            return normalizeBase64(cleanBase64) === normalizeBase64(reencoded);
+        }
+        catch {
+            return false;
+        }
+    },
+    /**
+     * Get base64 string size in bytes
+     */
+    getBase64Size: (base64) => {
+        // Remove data URI prefix if present
+        const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
+        return Buffer.byteLength(cleanBase64, "base64");
+    },
+    /**
+     * Compress base64 image by reducing quality (basic implementation)
+     * Note: This is a placeholder - for production use, consider using sharp or similar
+     */
+    compressBase64: (base64, _quality = 0.8) => {
+        // This is a basic implementation that just returns the original
+        // In a real implementation, you'd use an image processing library
+        logger.warn("Base64 compression not implemented - returning original");
+        return base64;
+    },
 };

package/dist/lib/utils/messageBuilder.d.ts

@@ -7,13 +7,12 @@ import type { MultimodalChatMessage } from "../types/conversation.js";
 import type { TextGenerationOptions } from "../types/index.js";
 import type { StreamOptions } from "../types/streamTypes.js";
 import type { GenerateOptions } from "../types/generateTypes.js";
+import type { CoreMessage } from "ai";
 /**
- * Core message type compatible with AI SDK
+ * Type-safe conversion from MultimodalChatMessage[] to CoreMessage[]
+ * Filters out invalid content and ensures strict CoreMessage contract compliance
  */
-type CoreMessage = {
-    role: "user" | "assistant" | "system";
-    content: string;
-};
+export declare function convertToCoreMessages(messages: MultimodalChatMessage[]): CoreMessage[];
 /**
  * Build a properly formatted message array for AI providers
  * Combines system prompt, conversation history, and current user prompt
@@ -25,4 +24,3 @@ export declare function buildMessagesArray(options: TextGenerationOptions | Stre
  * Detects when images are present and routes through provider adapter
  */
 export declare function buildMultimodalMessagesArray(options: GenerateOptions, provider: string, model: string): Promise<MultimodalChatMessage[]>;
-export {};

package/dist/lib/utils/messageBuilder.js

@@ -8,6 +8,147 @@ import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerIma
 import { logger } from "./logger.js";
 import { request } from "undici";
 import { readFileSync, existsSync } from "fs";
+/**
+ * Type guard for validating message roles
+ */
+function isValidRole(role) {
+    return (typeof role === "string" &&
+        (role === "user" || role === "assistant" || role === "system"));
+}
+/**
+ * Type guard for validating content items
+ */
+function isValidContentItem(item) {
+    if (!item || typeof item !== "object") {
+        return false;
+    }
+    const contentItem = item;
+    if (contentItem.type === "text") {
+        return typeof contentItem.text === "string";
+    }
+    if (contentItem.type === "image") {
+        return (typeof contentItem.image === "string" &&
+            (contentItem.mimeType === undefined ||
+                typeof contentItem.mimeType === "string"));
+    }
+    return false;
+}
+/**
+ * Safely convert content item to AI SDK content format
+ */
+function convertContentItem(item) {
+    if (!isValidContentItem(item)) {
+        return null;
+    }
+    const contentItem = item;
+    if (contentItem.type === "text" && typeof contentItem.text === "string") {
+        return { type: "text", text: contentItem.text };
+    }
+    if (contentItem.type === "image" && typeof contentItem.image === "string") {
+        return {
+            type: "image",
+            image: contentItem.image,
+            ...(contentItem.mimeType && { mimeType: contentItem.mimeType }),
+        };
+    }
+    return null;
+}
+/**
+ * Type-safe conversion from MultimodalChatMessage[] to CoreMessage[]
+ * Filters out invalid content and ensures strict CoreMessage contract compliance
+ */
+export function convertToCoreMessages(messages) {
+    return messages
+        .map((msg) => {
+        // Validate role
+        if (!isValidRole(msg.role)) {
+            logger.warn("Invalid message role found, skipping", { role: msg.role });
+            return null;
+        }
+        // Handle string content
+        if (typeof msg.content === "string") {
+            // Create properly typed discriminated union messages
+            if (msg.role === "system") {
+                return {
+                    role: "system",
+                    content: msg.content,
+                };
+            }
+            else if (msg.role === "user") {
+                return {
+                    role: "user",
+                    content: msg.content,
+                };
+            }
+            else if (msg.role === "assistant") {
+                return {
+                    role: "assistant",
+                    content: msg.content,
+                };
+            }
+        }
+        // Handle array content (multimodal) - only user messages support full multimodal content
+        if (Array.isArray(msg.content)) {
+            const validContent = msg.content
+                .map(convertContentItem)
+                .filter((item) => item !== null);
+            // If no valid content items, skip the message
+            if (validContent.length === 0) {
+                logger.warn("No valid content items found in multimodal message, skipping");
+                return null;
+            }
+            if (msg.role === "user") {
+                // User messages support both text and image content
+                return {
+                    role: "user",
+                    content: validContent,
+                };
+            }
+            else if (msg.role === "assistant") {
+                // Assistant messages only support text content, filter out images
+                const textOnlyContent = validContent.filter((item) => item.type === "text");
+                if (textOnlyContent.length === 0) {
+                    // If no text content, convert to empty string
+                    return {
+                        role: "assistant",
+                        content: "",
+                    };
+                }
+                else if (textOnlyContent.length === 1) {
+                    // Single text item, use string content
+                    return {
+                        role: "assistant",
+                        content: textOnlyContent[0].text,
+                    };
+                }
+                else {
+                    // Multiple text items, concatenate them
+                    const combinedText = textOnlyContent
+                        .map((item) => item.text)
+                        .join(" ");
+                    return {
+                        role: "assistant",
+                        content: combinedText,
+                    };
+                }
+            }
+            else {
+                // System messages cannot have multimodal content, convert to text
+                const textContent = validContent.find((item) => item.type === "text")?.text || "";
+                return {
+                    role: "system",
+                    content: textContent,
+                };
+            }
+        }
+        // Invalid content type
+        logger.warn("Invalid message content type found, skipping", {
+            contentType: typeof msg.content,
+        });
+        return null;
+    })
+        .filter((msg) => msg !== null);
+}
 /**
  * Convert ChatMessage to CoreMessage for AI SDK compatibility
  */
@@ -84,7 +225,10 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     // If no images, use standard message building and convert to MultimodalChatMessage[]
     if (!hasImages) {
         const standardMessages = buildMessagesArray(options);
-        return standardMessages.map((msg) => ({ ...msg, content: msg.content }));
+        return standardMessages.map((msg) => ({
+            role: msg.role,
+            content: typeof msg.content === "string" ? msg.content : msg.content,
+        }));
     }
     // Validate provider supports vision
     if (!ProviderImageAdapter.supportsVision(provider, model)) {

package/dist/providers/azureOpenai.js

@@ -4,7 +4,7 @@ import { BaseProvider } from "../core/baseProvider.js";
 import { APIVersions } from "../types/providers.js";
 import { validateApiKey, createAzureAPIKeyConfig, createAzureEndpointConfig, } from "../utils/providerConfig.js";
 import { logger } from "../utils/logger.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 export class AzureOpenAIProvider extends BaseProvider {
@@ -109,8 +109,41 @@ export class AzureOpenAIProvider extends BaseProvider {
                     })),
                 });
             }
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Azure OpenAI: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Azure OpenAI: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options);
             const stream = await streamText({
                 model,

package/dist/providers/googleAiStudio.js

@@ -7,8 +7,9 @@ import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/errors.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 // Google AI Live API types now imported from ../types/providerSpecific.js
+// Import proper types for multimodal message handling
 // Create Google GenAI client
 async function createGoogleGenAIClient(apiKey) {
     const mod = await import("@google/genai");
@@ -90,8 +91,41 @@ export class GoogleAIStudioProvider extends BaseProvider {
             // Get tools consistently with generate method
             const shouldUseTools = !options.disableTools && this.supportsTools();
             const tools = shouldUseTools ? await this.getAllTools() : {};
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Google AI Studio: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Google AI Studio: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const result = await streamText({
                 model,
                 messages: messages,

package/dist/providers/googleVertex.js

@@ -11,8 +11,9 @@ import fs from "fs";
 import path from "path";
 import os from "os";
 import dns from "dns";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
+// Import proper types for multimodal message handling
 // Enhanced Anthropic support with direct imports
 // Using the dual provider architecture from Vercel AI SDK
 const hasAnthropicSupport = () => {
@@ -594,8 +595,41 @@ export class GoogleVertexProvider extends BaseProvider {
         try {
             // Validate stream options
             this.validateStreamOptionsOnly(options);
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`${functionTag}: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`${functionTag}: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options); // This is where network connection happens!
             // Get all available tools (direct + MCP + external) for streaming
             const shouldUseTools = !options.disableTools && this.supportsTools();

package/dist/utils/imageProcessor.d.ts

@@ -81,4 +81,48 @@ export declare const imageUtils: {
      * Convert file size to human readable format
      */
     formatFileSize: (bytes: number) => string;
+    /**
+     * Convert Buffer to base64 string
+     */
+    bufferToBase64: (buffer: Buffer) => string;
+    /**
+     * Convert base64 string to Buffer
+     */
+    base64ToBuffer: (base64: string) => Buffer;
+    /**
+     * Convert file path to base64 data URI
+     */
+    fileToBase64DataUri: (filePath: string, maxBytes?: number) => Promise<string>;
+    /**
+     * Convert URL to base64 data URI by downloading the image
+     */
+    urlToBase64DataUri: (url: string, { timeoutMs, maxBytes }?: {
+        timeoutMs?: number | undefined;
+        maxBytes?: number | undefined;
+    }) => Promise<string>;
+    /**
+     * Extract base64 data from data URI
+     */
+    extractBase64FromDataUri: (dataUri: string) => string;
+    /**
+     * Extract MIME type from data URI
+     */
+    extractMimeTypeFromDataUri: (dataUri: string) => string;
+    /**
+     * Create data URI from base64 and MIME type
+     */
+    createDataUri: (base64: string, mimeType?: string) => string;
+    /**
+     * Validate base64 string format
+     */
+    isValidBase64: (str: string) => boolean;
+    /**
+     * Get base64 string size in bytes
+     */
+    getBase64Size: (base64: string) => number;
+    /**
+     * Compress base64 image by reducing quality (basic implementation)
+     * Note: This is a placeholder - for production use, consider using sharp or similar
+     */
+    compressBase64: (base64: string, _quality?: number) => string;
 };

package/dist/utils/imageProcessor.js

@@ -151,6 +151,8 @@ export class ImageProcessor {
                     bmp: "image/bmp",
                     tiff: "image/tiff",
                     tif: "image/tiff",
+                    svg: "image/svg+xml",
+                    avif: "image/avif",
                 };
                 return imageTypes[extension || ""] || "image/jpeg";
             }
@@ -183,6 +185,21 @@ export class ImageProcessor {
                         return "image/webp";
                     }
                 }
+                // SVG: check for "<svg" or "<?xml" at start (text-based)
+                if (input.length >= 4) {
+                    const start = input.subarray(0, 4).toString();
+                    if (start === "<svg" || start === "<?xm") {
+                        return "image/svg+xml";
+                    }
+                }
+                // AVIF: check for "ftypavif" signature at bytes 4-11
+                if (input.length >= 12) {
+                    const ftyp = input.subarray(4, 8).toString();
+                    const brand = input.subarray(8, 12).toString();
+                    if (ftyp === "ftyp" && brand === "avif") {
+                        return "image/avif";
+                    }
+                }
             }
             return "image/jpeg"; // Default fallback
         }
@@ -217,6 +234,8 @@ export class ImageProcessor {
             "image/webp",
             "image/bmp",
             "image/tiff",
+            "image/svg+xml",
+            "image/avif",
         ];
         return supportedFormats.includes(mediaType.toLowerCase());
     }
@@ -332,14 +351,7 @@ export const imageUtils = {
     /**
      * Check if a string is base64 encoded
      */
-    isBase64: (str) => {
-        try {
-            return btoa(atob(str)) === str;
-        }
-        catch {
-            return false;
-        }
-    },
+    isBase64: (str) => imageUtils.isValidBase64(str),
     /**
      * Extract file extension from filename or URL
      */
@@ -359,4 +371,143 @@ export const imageUtils = {
         const i = Math.floor(Math.log(bytes) / Math.log(k));
         return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
     },
+    /**
+     * Convert Buffer to base64 string
+     */
+    bufferToBase64: (buffer) => {
+        return buffer.toString("base64");
+    },
+    /**
+     * Convert base64 string to Buffer
+     */
+    base64ToBuffer: (base64) => {
+        // Remove data URI prefix if present
+        const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
+        return Buffer.from(cleanBase64, "base64");
+    },
+    /**
+     * Convert file path to base64 data URI
+     */
+    fileToBase64DataUri: async (filePath, maxBytes = 10 * 1024 * 1024) => {
+        try {
+            const fs = await import("fs/promises");
+            // File existence and type validation
+            const stat = await fs.stat(filePath);
+            if (!stat.isFile()) {
+                throw new Error("Not a file");
+            }
+            // Size check before reading - prevent memory exhaustion
+            if (stat.size > maxBytes) {
+                throw new Error(`File too large: ${stat.size} bytes (max: ${maxBytes} bytes)`);
+            }
+            const buffer = await fs.readFile(filePath);
+            // Enhanced MIME detection: try buffer content first, fallback to filename
+            const mimeType = ImageProcessor.detectImageType(buffer) ||
+                ImageProcessor.detectImageType(filePath);
+            const base64 = buffer.toString("base64");
+            return `data:${mimeType};base64,${base64}`;
+        }
+        catch (error) {
+            throw new Error(`Failed to convert file to base64: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    },
+    /**
+     * Convert URL to base64 data URI by downloading the image
+     */
+    urlToBase64DataUri: async (url, { timeoutMs = 15000, maxBytes = 10 * 1024 * 1024 } = {}) => {
+        try {
+            // Basic protocol whitelist
+            if (!/^https?:\/\//i.test(url)) {
+                throw new Error("Unsupported protocol");
+            }
+            const controller = new AbortController();
+            const t = setTimeout(() => controller.abort(), timeoutMs);
+            try {
+                const response = await fetch(url, { signal: controller.signal });
+                if (!response.ok) {
+                    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+                }
+                const contentType = response.headers.get("content-type") || "";
+                if (!/^image\//i.test(contentType)) {
+                    throw new Error(`Unsupported content-type: ${contentType || "unknown"}`);
+                }
+                const len = Number(response.headers.get("content-length") || 0);
+                if (len && len > maxBytes) {
+                    throw new Error(`Content too large: ${len} bytes`);
+                }
+                const buffer = await response.arrayBuffer();
+                if (buffer.byteLength > maxBytes) {
+                    throw new Error(`Downloaded content too large: ${buffer.byteLength} bytes`);
+                }
+                const base64 = Buffer.from(buffer).toString("base64");
+                return `data:${contentType || "image/jpeg"};base64,${base64}`;
+            }
+            finally {
+                clearTimeout(t);
+            }
+        }
+        catch (error) {
+            throw new Error(`Failed to download and convert URL to base64: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    },
+    /**
+     * Extract base64 data from data URI
+     */
+    extractBase64FromDataUri: (dataUri) => {
+        if (!dataUri.includes(",")) {
+            return dataUri; // Already just base64
+        }
+        return dataUri.split(",")[1];
+    },
+    /**
+     * Extract MIME type from data URI
+     */
+    extractMimeTypeFromDataUri: (dataUri) => {
+        const match = dataUri.match(/^data:([^;]+);base64,/);
+        return match ? match[1] : "image/jpeg";
+    },
+    /**
+     * Create data URI from base64 and MIME type
+     */
+    createDataUri: (base64, mimeType = "image/jpeg") => {
+        // Remove data URI prefix if already present
+        const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
+        return `data:${mimeType};base64,${cleanBase64}`;
+    },
+    /**
+     * Validate base64 string format
+     */
+    isValidBase64: (str) => {
+        try {
+            // Remove data URI prefix if present
+            const cleanBase64 = str.includes(",") ? str.split(",")[1] : str;
+            // Check if it's valid base64
+            const decoded = Buffer.from(cleanBase64, "base64");
+            const reencoded = decoded.toString("base64");
+            // Remove padding for comparison (base64 can have different padding)
+            const normalizeBase64 = (b64) => b64.replace(/=+$/, "");
+            return normalizeBase64(cleanBase64) === normalizeBase64(reencoded);
+        }
+        catch {
+            return false;
+        }
+    },
+    /**
+     * Get base64 string size in bytes
+     */
+    getBase64Size: (base64) => {
+        // Remove data URI prefix if present
+        const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
+        return Buffer.byteLength(cleanBase64, "base64");
+    },
+    /**
+     * Compress base64 image by reducing quality (basic implementation)
+     * Note: This is a placeholder - for production use, consider using sharp or similar
+     */
+    compressBase64: (base64, _quality = 0.8) => {
+        // This is a basic implementation that just returns the original
+        // In a real implementation, you'd use an image processing library
+        logger.warn("Base64 compression not implemented - returning original");
+        return base64;
+    },
 };

package/dist/utils/messageBuilder.d.ts

@@ -7,13 +7,12 @@ import type { MultimodalChatMessage } from "../types/conversation.js";
 import type { TextGenerationOptions } from "../types/index.js";
 import type { StreamOptions } from "../types/streamTypes.js";
 import type { GenerateOptions } from "../types/generateTypes.js";
+import type { CoreMessage } from "ai";
 /**
- * Core message type compatible with AI SDK
+ * Type-safe conversion from MultimodalChatMessage[] to CoreMessage[]
+ * Filters out invalid content and ensures strict CoreMessage contract compliance
  */
-type CoreMessage = {
-    role: "user" | "assistant" | "system";
-    content: string;
-};
+export declare function convertToCoreMessages(messages: MultimodalChatMessage[]): CoreMessage[];
 /**
  * Build a properly formatted message array for AI providers
  * Combines system prompt, conversation history, and current user prompt
@@ -25,4 +24,3 @@ export declare function buildMessagesArray(options: TextGenerationOptions | Stre
  * Detects when images are present and routes through provider adapter
  */
 export declare function buildMultimodalMessagesArray(options: GenerateOptions, provider: string, model: string): Promise<MultimodalChatMessage[]>;
-export {};

package/dist/utils/messageBuilder.js

@@ -8,6 +8,147 @@ import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerIma
 import { logger } from "./logger.js";
 import { request } from "undici";
 import { readFileSync, existsSync } from "fs";
+/**
+ * Type guard for validating message roles
+ */
+function isValidRole(role) {
+    return (typeof role === "string" &&
+        (role === "user" || role === "assistant" || role === "system"));
+}
+/**
+ * Type guard for validating content items
+ */
+function isValidContentItem(item) {
+    if (!item || typeof item !== "object") {
+        return false;
+    }
+    const contentItem = item;
+    if (contentItem.type === "text") {
+        return typeof contentItem.text === "string";
+    }
+    if (contentItem.type === "image") {
+        return (typeof contentItem.image === "string" &&
+            (contentItem.mimeType === undefined ||
+                typeof contentItem.mimeType === "string"));
+    }
+    return false;
+}
+/**
+ * Safely convert content item to AI SDK content format
+ */
+function convertContentItem(item) {
+    if (!isValidContentItem(item)) {
+        return null;
+    }
+    const contentItem = item;
+    if (contentItem.type === "text" && typeof contentItem.text === "string") {
+        return { type: "text", text: contentItem.text };
+    }
+    if (contentItem.type === "image" && typeof contentItem.image === "string") {
+        return {
+            type: "image",
+            image: contentItem.image,
+            ...(contentItem.mimeType && { mimeType: contentItem.mimeType }),
+        };
+    }
+    return null;
+}
+/**
+ * Type-safe conversion from MultimodalChatMessage[] to CoreMessage[]
+ * Filters out invalid content and ensures strict CoreMessage contract compliance
+ */
+export function convertToCoreMessages(messages) {
+    return messages
+        .map((msg) => {
+        // Validate role
+        if (!isValidRole(msg.role)) {
+            logger.warn("Invalid message role found, skipping", { role: msg.role });
+            return null;
+        }
+        // Handle string content
+        if (typeof msg.content === "string") {
+            // Create properly typed discriminated union messages
+            if (msg.role === "system") {
+                return {
+                    role: "system",
+                    content: msg.content,
+                };
+            }
+            else if (msg.role === "user") {
+                return {
+                    role: "user",
+                    content: msg.content,
+                };
+            }
+            else if (msg.role === "assistant") {
+                return {
+                    role: "assistant",
+                    content: msg.content,
+                };
+            }
+        }
+        // Handle array content (multimodal) - only user messages support full multimodal content
+        if (Array.isArray(msg.content)) {
+            const validContent = msg.content
+                .map(convertContentItem)
+                .filter((item) => item !== null);
+            // If no valid content items, skip the message
+            if (validContent.length === 0) {
+                logger.warn("No valid content items found in multimodal message, skipping");
+                return null;
+            }
+            if (msg.role === "user") {
+                // User messages support both text and image content
+                return {
+                    role: "user",
+                    content: validContent,
+                };
+            }
+            else if (msg.role === "assistant") {
+                // Assistant messages only support text content, filter out images
+                const textOnlyContent = validContent.filter((item) => item.type === "text");
+                if (textOnlyContent.length === 0) {
+                    // If no text content, convert to empty string
+                    return {
+                        role: "assistant",
+                        content: "",
+                    };
+                }
+                else if (textOnlyContent.length === 1) {
+                    // Single text item, use string content
+                    return {
+                        role: "assistant",
+                        content: textOnlyContent[0].text,
+                    };
+                }
+                else {
+                    // Multiple text items, concatenate them
+                    const combinedText = textOnlyContent
+                        .map((item) => item.text)
+                        .join(" ");
+                    return {
+                        role: "assistant",
+                        content: combinedText,
+                    };
+                }
+            }
+            else {
+                // System messages cannot have multimodal content, convert to text
+                const textContent = validContent.find((item) => item.type === "text")?.text || "";
+                return {
+                    role: "system",
+                    content: textContent,
+                };
+            }
+        }
+        // Invalid content type
+        logger.warn("Invalid message content type found, skipping", {
+            contentType: typeof msg.content,
+        });
+        return null;
+    })
+        .filter((msg) => msg !== null);
+}
 /**
  * Convert ChatMessage to CoreMessage for AI SDK compatibility
  */
@@ -84,7 +225,10 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     // If no images, use standard message building and convert to MultimodalChatMessage[]
     if (!hasImages) {
         const standardMessages = buildMessagesArray(options);
-        return standardMessages.map((msg) => ({ ...msg, content: msg.content }));
+        return standardMessages.map((msg) => ({
+            role: msg.role,
+            content: typeof msg.content === "string" ? msg.content : msg.content,
+        }));
     }
     // Validate provider supports vision
     if (!ProviderImageAdapter.supportsVision(provider, model)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "7.46.0",
+  "version": "7.47.0",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",