@juspay/neurolink 7.46.0 → 7.47.1

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [7.47.1](https://github.com/juspay/neurolink/compare/v7.47.0...v7.47.1) (2025-09-26)
+
+### Bug Fixes
+
+- **(tools):** Unregistered tools getting called ([45fd67a](https://github.com/juspay/neurolink/commit/45fd67af418b5e458ce6a261a7891234a8d489b8))
+
+## [7.47.0](https://github.com/juspay/neurolink/compare/v7.46.0...v7.47.0) (2025-09-25)
+
+### Features
+
+- **(chat):** Implement multimodal UI and extend SDK support ([12a2f59](https://github.com/juspay/neurolink/commit/12a2f59c4826e82ab1feb1347d08980682748ad2))
+
 ## [7.46.0](https://github.com/juspay/neurolink/compare/v7.45.0...v7.46.0) (2025-09-24)
 
 ### Features

package/dist/adapters/providerImageAdapter.js

@@ -34,6 +34,14 @@ const VISION_CAPABILITIES = {
         "claude-3-sonnet",
         "claude-3-haiku",
     ],
+    azure: [
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4-turbo",
+        "gpt-4-vision-preview",
+        "gpt-4.1",
+        "gpt-4",
+    ],
     vertex: [
         // Gemini models on Vertex AI
         "gemini-2.5-pro",
@@ -78,6 +86,10 @@ export class ProviderImageAdapter {
                 case "openai":
                     adaptedPayload = this.formatForOpenAI(text, images);
                     break;
+                case "azure":
+                case "azure-openai":
+                    adaptedPayload = this.formatForOpenAI(text, images);
+                    break;
                 case "google-ai":
                 case "google":
                     adaptedPayload = this.formatForGoogleAI(text, images);

package/dist/core/constants.js

@@ -51,7 +51,7 @@ export const PROVIDER_MAX_TOKENS = {
         default: 64000,
     },
     azure: {
-        default: 64000,
+        default: 32000,
     },
     ollama: {
         default: 64000,

package/dist/factories/providerRegistry.js

@@ -87,7 +87,7 @@ export class ProviderRegistry {
             // Register Amazon SageMaker provider
             ProviderFactory.registerProvider(AIProviderName.SAGEMAKER, async (modelName, _providerName, _sdk, region) => {
                 const { AmazonSageMakerProvider } = await import("../providers/amazonSagemaker.js");
-                return new AmazonSageMakerProvider(modelName, region);
+                return new AmazonSageMakerProvider(modelName, undefined, region);
             }, process.env.SAGEMAKER_MODEL || "sagemaker-model", ["sagemaker", "aws-sagemaker"]);
             logger.debug("All providers registered successfully");
             this.registered = true;

package/dist/lib/adapters/providerImageAdapter.js

@@ -34,6 +34,14 @@ const VISION_CAPABILITIES = {
         "claude-3-sonnet",
         "claude-3-haiku",
     ],
+    azure: [
+        "gpt-4o",
+        "gpt-4o-mini",
+        "gpt-4-turbo",
+        "gpt-4-vision-preview",
+        "gpt-4.1",
+        "gpt-4",
+    ],
     vertex: [
         // Gemini models on Vertex AI
         "gemini-2.5-pro",
@@ -78,6 +86,10 @@ export class ProviderImageAdapter {
                 case "openai":
                     adaptedPayload = this.formatForOpenAI(text, images);
                     break;
+                case "azure":
+                case "azure-openai":
+                    adaptedPayload = this.formatForOpenAI(text, images);
+                    break;
                 case "google-ai":
                 case "google":
                     adaptedPayload = this.formatForGoogleAI(text, images);

package/dist/lib/core/constants.js

@@ -51,7 +51,7 @@ export const PROVIDER_MAX_TOKENS = {
         default: 64000,
     },
     azure: {
-        default: 64000,
+        default: 32000,
     },
     ollama: {
         default: 64000,

package/dist/lib/factories/providerRegistry.js

@@ -87,7 +87,7 @@ export class ProviderRegistry {
             // Register Amazon SageMaker provider
             ProviderFactory.registerProvider(AIProviderName.SAGEMAKER, async (modelName, _providerName, _sdk, region) => {
                 const { AmazonSageMakerProvider } = await import("../providers/amazonSagemaker.js");
-                return new AmazonSageMakerProvider(modelName, region);
+                return new AmazonSageMakerProvider(modelName, undefined, region);
             }, process.env.SAGEMAKER_MODEL || "sagemaker-model", ["sagemaker", "aws-sagemaker"]);
             logger.debug("All providers registered successfully");
             this.registered = true;

package/dist/lib/neurolink.d.ts

@@ -6,6 +6,7 @@
  * Uses real MCP infrastructure for tool discovery and execution.
  */
 import type { TextGenerationOptions, TextGenerationResult } from "./types/index.js";
+import { MCPToolRegistry } from "./mcp/toolRegistry.js";
 import type { GenerateOptions, GenerateResult } from "./types/generateTypes.js";
 import type { StreamOptions, StreamResult } from "./types/streamTypes.js";
 import type { MCPServerInfo, MCPExecutableTool } from "./types/mcpTypes.js";
@@ -46,6 +47,7 @@ export interface MCPStatus {
 export declare class NeuroLink {
     private mcpInitialized;
     private emitter;
+    private toolRegistry;
     private autoDiscoveredServerInfos;
     private externalServerManager;
     private toolCache;
@@ -99,6 +101,7 @@ export declare class NeuroLink {
      * @param config.hitl.dangerousActions - Keywords that trigger confirmation (default: ['delete', 'remove', 'drop'])
      * @param config.hitl.timeout - Confirmation timeout in milliseconds (default: 30000)
      * @param config.hitl.allowArgumentModification - Allow users to modify tool parameters (default: true)
+     * @param config.toolRegistry - Optional tool registry instance for advanced use cases (default: new MCPToolRegistry())
      *
      * @example
      * ```typescript
@@ -139,6 +142,7 @@ export declare class NeuroLink {
         conversationMemory?: Partial<ConversationMemoryConfig>;
         enableOrchestration?: boolean;
         hitl?: HITLConfig;
+        toolRegistry?: MCPToolRegistry;
     });
     /**
      * Initialize provider registry with security settings

package/dist/lib/neurolink.js

@@ -18,7 +18,7 @@ import { mcpLogger } from "./utils/logger.js";
 import { SYSTEM_LIMITS } from "./core/constants.js";
 import { NANOSECOND_TO_MS_DIVISOR, TOOL_TIMEOUTS, RETRY_ATTEMPTS, RETRY_DELAYS, CIRCUIT_BREAKER, CIRCUIT_BREAKER_RESET_MS, MEMORY_THRESHOLDS, PROVIDER_TIMEOUTS, PERFORMANCE_THRESHOLDS, } from "./constants/index.js";
 import pLimit from "p-limit";
-import { toolRegistry } from "./mcp/toolRegistry.js";
+import { MCPToolRegistry } from "./mcp/toolRegistry.js";
 import { logger } from "./utils/logger.js";
 import { getBestProvider } from "./utils/providerUtils.js";
 import { ProviderRegistry } from "./factories/providerRegistry.js";
@@ -45,6 +45,7 @@ import { isZodSchema } from "./utils/schemaConversion.js";
 export class NeuroLink {
     mcpInitialized = false;
     emitter = new EventEmitter();
+    toolRegistry;
     autoDiscoveredServerInfos = [];
     // External MCP server management
     externalServerManager;
@@ -140,6 +141,7 @@ export class NeuroLink {
      * @param config.hitl.dangerousActions - Keywords that trigger confirmation (default: ['delete', 'remove', 'drop'])
      * @param config.hitl.timeout - Confirmation timeout in milliseconds (default: 30000)
      * @param config.hitl.allowArgumentModification - Allow users to modify tool parameters (default: true)
+     * @param config.toolRegistry - Optional tool registry instance for advanced use cases (default: new MCPToolRegistry())
      *
      * @example
      * ```typescript
@@ -177,6 +179,7 @@ export class NeuroLink {
      * @throws {Error} When HITL configuration is invalid (if enabled)
      */
     constructor(config) {
+        this.toolRegistry = config?.toolRegistry || new MCPToolRegistry();
         // Initialize orchestration setting
         this.enableOrchestration = config?.enableOrchestration ?? false;
         // Read tool cache duration from environment variables, with a default
@@ -278,7 +281,7 @@ export class NeuroLink {
                 // Initialize HITL manager
                 this.hitlManager = new HITLManager(config.hitl);
                 // Inject HITL manager into tool registry
-                toolRegistry.setHITLManager(this.hitlManager);
+                this.toolRegistry.setHITLManager(this.hitlManager);
                 // Inject HITL manager into external server manager
                 this.externalServerManager.setHITLManager(this.hitlManager);
                 // Set up HITL event forwarding to main emitter
@@ -627,7 +630,7 @@ export class NeuroLink {
                 mcpLogger.debug("Direct tools server are disabled via environment variable.");
             }
             else {
-                await toolRegistry.registerServer("neurolink-direct", directToolsServer);
+                await this.toolRegistry.registerServer("neurolink-direct", directToolsServer);
                 mcpLogger.debug("[NeuroLink] Direct tools server registered successfully", {
                     serverId: "neurolink-direct",
                 });
@@ -1371,7 +1374,7 @@ export class NeuroLink {
                     mcpInitialized: this.mcpInitialized,
                     mcpComponents: {
                         hasExternalServerManager: !!this.externalServerManager,
-                        hasToolRegistry: !!toolRegistry,
+                        hasToolRegistry: !!this.toolRegistry,
                         hasProviderRegistry: !!AIProviderFactory,
                     },
                     fallbackReason: "MCP_NOT_INITIALIZED",
@@ -2403,7 +2406,7 @@ export class NeuroLink {
             // SMART DEFAULTS: Use utility to eliminate boilerplate creation
             const mcpServerInfo = createCustomToolServerInfo(name, convertedTool);
             // Register with toolRegistry using MCPServerInfo directly
-            toolRegistry.registerServer(mcpServerInfo);
+            this.toolRegistry.registerServer(mcpServerInfo);
             // Emit tool registration success event
             this.emitter.emit("tools-register:end", {
                 toolName: name,
@@ -2475,7 +2478,7 @@ export class NeuroLink {
     unregisterTool(name) {
         this.invalidateToolCache(); // Invalidate cache when a tool is unregistered
         const serverId = `custom-tool-${name}`;
-        const removed = toolRegistry.unregisterServer(serverId);
+        const removed = this.toolRegistry.unregisterServer(serverId);
         if (removed) {
             logger.info(`Unregistered custom tool: ${name}`);
         }
@@ -2487,7 +2490,7 @@ export class NeuroLink {
      */
     getCustomTools() {
         // Get tools from toolRegistry with smart category detection
-        const customTools = toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true }));
+        const customTools = this.toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true }));
         const toolMap = new Map();
         for (const tool of customTools) {
             const effectiveSchema = tool.inputSchema || tool.parameters;
@@ -2545,7 +2548,7 @@ export class NeuroLink {
                         hasShopId: !!executionContext.shopId,
                         sessionId: executionContext.sessionId,
                     });
-                    return await toolRegistry.executeTool(tool.name, params, executionContext);
+                    return await this.toolRegistry.executeTool(tool.name, params, executionContext);
                 },
             });
         }
@@ -2566,7 +2569,7 @@ export class NeuroLink {
                 serverInfo.tools = [];
             }
             // ZERO CONVERSIONS: Pass MCPServerInfo directly to toolRegistry
-            await toolRegistry.registerServer(serverInfo);
+            await this.toolRegistry.registerServer(serverInfo);
             mcpLogger.info(`[NeuroLink] Successfully registered in-memory server: ${serverId}`, {
                 category: serverInfo.metadata?.category,
                 provider: serverInfo.metadata?.provider,
@@ -2584,7 +2587,7 @@ export class NeuroLink {
      */
     getInMemoryServers() {
         // Get in-memory servers from toolRegistry
-        const serverInfos = toolRegistry.getBuiltInServerInfos();
+        const serverInfos = this.toolRegistry.getBuiltInServerInfos();
         const serverMap = new Map();
         for (const serverInfo of serverInfos) {
             if (detectCategory({
@@ -2603,7 +2606,7 @@ export class NeuroLink {
      */
     getInMemoryServerInfos() {
         // Get in-memory servers from centralized tool registry
-        const allServers = toolRegistry.getBuiltInServerInfos();
+        const allServers = this.toolRegistry.getBuiltInServerInfos();
         return allServers.filter((server) => detectCategory({
             existingCategory: server.metadata?.category,
             serverId: server.id,
@@ -2855,7 +2858,7 @@ export class NeuroLink {
                 storedContextKeys: Object.keys(storedContext),
                 finalContextKeys: Object.keys(context),
             });
-            const result = (await toolRegistry.executeTool(toolName, params, context));
+            const result = (await this.toolRegistry.executeTool(toolName, params, context));
             // ADD: Check if result indicates a failure and emit error event
             if (result &&
                 typeof result === "object" &&
@@ -2905,9 +2908,9 @@ export class NeuroLink {
             getAllToolsHrTimeStart: getAllToolsHrTimeStart.toString(),
             // 🔧 Tool registry state
             toolRegistryState: {
-                hasToolRegistry: !!toolRegistry,
+                hasToolRegistry: !!this.toolRegistry,
                 toolRegistrySize: 0, // Not accessible as size property
-                toolRegistryType: toolRegistry?.constructor?.name || "NOT_SET",
+                toolRegistryType: this.toolRegistry?.constructor?.name || "NOT_SET",
                 hasExternalServerManager: !!this.externalServerManager,
                 externalServerManagerType: this.externalServerManager?.constructor?.name || "NOT_SET",
             },
@@ -2926,7 +2929,7 @@ export class NeuroLink {
             // Optimized: Collect all tools with minimal object creation
             const allTools = new Map();
             // 1. Add MCP server tools (built-in direct tools)
-            const mcpToolsRaw = await toolRegistry.listTools();
+            const mcpToolsRaw = await this.toolRegistry.listTools();
             for (const tool of mcpToolsRaw) {
                 if (!allTools.has(tool.name)) {
                     const optimizedTool = optimizeToolForCollection(tool, {
@@ -2936,7 +2939,7 @@ export class NeuroLink {
                 }
             }
             // 2. Add custom tools from this NeuroLink instance
-            const customToolsRaw = toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true }));
+            const customToolsRaw = this.toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true }));
             for (const tool of customToolsRaw) {
                 if (!allTools.has(tool.name)) {
                     const optimizedTool = optimizeToolForCollection(tool, {
@@ -2952,7 +2955,7 @@ export class NeuroLink {
                 }
             }
             // 3. Add tools from in-memory MCP servers
-            const inMemoryToolsRaw = toolRegistry.getToolsByCategory("in-memory");
+            const inMemoryToolsRaw = this.toolRegistry.getToolsByCategory("in-memory");
             for (const tool of inMemoryToolsRaw) {
                 if (!allTools.has(tool.name)) {
                     const optimizedTool = optimizeToolForCollection(tool, {
@@ -3231,13 +3234,13 @@ export class NeuroLink {
             // Initialize MCP if not already initialized (loads external servers from config)
             await this.initializeMCP();
             // Get built-in tools
-            const allTools = await toolRegistry.listTools();
+            const allTools = await this.toolRegistry.listTools();
             // Get external MCP server statistics
             const externalStats = this.externalServerManager.getStatistics();
             // DIRECT RETURNS - ZERO conversion
             const externalMCPServers = this.externalServerManager.listServers();
             const inMemoryServerInfos = this.getInMemoryServerInfos();
-            const builtInServerInfos = toolRegistry.getBuiltInServerInfos();
+            const builtInServerInfos = this.toolRegistry.getBuiltInServerInfos();
             const autoDiscoveredServerInfos = this.getAutoDiscoveredServerInfos();
             // Calculate totals
             const totalServers = externalMCPServers.length +
@@ -3255,7 +3258,7 @@ export class NeuroLink {
                 autoDiscoveredCount: autoDiscoveredServerInfos.length,
                 totalTools,
                 autoDiscoveredServers: autoDiscoveredServerInfos,
-                customToolsCount: toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true })).length,
+                customToolsCount: this.toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true })).length,
                 inMemoryServersCount: inMemoryServerInfos.length,
                 externalMCPServersCount: externalMCPServers.length,
                 externalMCPConnectedCount: externalStats.connectedServers,
@@ -3271,7 +3274,7 @@ export class NeuroLink {
                 autoDiscoveredCount: 0,
                 totalTools: 0,
                 autoDiscoveredServers: [],
-                customToolsCount: toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true })).length,
+                customToolsCount: this.toolRegistry.getToolsByCategory(detectCategory({ isCustomTool: true })).length,
                 inMemoryServersCount: 0,
                 externalMCPServersCount: 0,
                 externalMCPConnectedCount: 0,
@@ -3290,7 +3293,7 @@ export class NeuroLink {
         return [
             ...this.externalServerManager.listServers(), // Direct return
             ...this.getInMemoryServerInfos(), // Direct return
-            ...toolRegistry.getBuiltInServerInfos(), // Direct return
+            ...this.toolRegistry.getBuiltInServerInfos(), // Direct return
             ...this.getAutoDiscoveredServerInfos(), // Direct return
         ];
     }
@@ -3303,7 +3306,7 @@ export class NeuroLink {
         try {
             // Test built-in tools
             if (serverId === "neurolink-direct") {
-                const tools = await toolRegistry.listTools();
+                const tools = await this.toolRegistry.listTools();
                 return tools.length > 0;
             }
             // Test in-memory servers
@@ -3480,7 +3483,7 @@ export class NeuroLink {
         const tools = {};
         let healthyCount = 0;
         // Get all tool names from toolRegistry
-        const allTools = await toolRegistry.listTools();
+        const allTools = await this.toolRegistry.listTools();
         const allToolNames = new Set(allTools.map((tool) => tool.name));
         for (const toolName of allToolNames) {
             const metrics = this.toolExecutionMetrics.get(toolName);
@@ -3908,7 +3911,7 @@ export class NeuroLink {
         try {
             const externalTools = this.externalServerManager.getServerTools(serverId);
             for (const tool of externalTools) {
-                toolRegistry.removeTool(tool.name);
+                this.toolRegistry.removeTool(tool.name);
                 mcpLogger.debug(`[NeuroLink] Unregistered external MCP tool from main registry: ${tool.name}`);
             }
         }
@@ -3921,7 +3924,7 @@ export class NeuroLink {
      */
     unregisterExternalMCPToolFromRegistry(toolName) {
         try {
-            toolRegistry.removeTool(toolName);
+            this.toolRegistry.removeTool(toolName);
             mcpLogger.debug(`[NeuroLink] Unregistered external MCP tool from main registry: ${toolName}`);
         }
         catch (error) {
@@ -3979,7 +3982,7 @@ export class NeuroLink {
         try {
             const externalTools = this.externalServerManager.getAllTools();
             for (const tool of externalTools) {
-                toolRegistry.removeTool(tool.name);
+                this.toolRegistry.removeTool(tool.name);
             }
             mcpLogger.debug(`[NeuroLink] Unregistered ${externalTools.length} external MCP tools from main registry`);
         }

package/dist/lib/providers/azureOpenai.js

@@ -4,7 +4,7 @@ import { BaseProvider } from "../core/baseProvider.js";
 import { APIVersions } from "../types/providers.js";
 import { validateApiKey, createAzureAPIKeyConfig, createAzureEndpointConfig, } from "../utils/providerConfig.js";
 import { logger } from "../utils/logger.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 export class AzureOpenAIProvider extends BaseProvider {
@@ -109,8 +109,41 @@ export class AzureOpenAIProvider extends BaseProvider {
                     })),
                 });
             }
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Azure OpenAI: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Azure OpenAI: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options);
             const stream = await streamText({
                 model,

package/dist/lib/providers/googleAiStudio.js

@@ -7,8 +7,9 @@ import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } from "../types/errors.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 // Google AI Live API types now imported from ../types/providerSpecific.js
+// Import proper types for multimodal message handling
 // Create Google GenAI client
 async function createGoogleGenAIClient(apiKey) {
     const mod = await import("@google/genai");
@@ -90,8 +91,41 @@ export class GoogleAIStudioProvider extends BaseProvider {
             // Get tools consistently with generate method
             const shouldUseTools = !options.disableTools && this.supportsTools();
             const tools = shouldUseTools ? await this.getAllTools() : {};
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Google AI Studio: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Google AI Studio: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const result = await streamText({
                 model,
                 messages: messages,

package/dist/lib/providers/googleVertex.js

@@ -11,8 +11,9 @@ import fs from "fs";
 import path from "path";
 import os from "os";
 import dns from "dns";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
+// Import proper types for multimodal message handling
 // Enhanced Anthropic support with direct imports
 // Using the dual provider architecture from Vercel AI SDK
 const hasAnthropicSupport = () => {
@@ -594,8 +595,41 @@ export class GoogleVertexProvider extends BaseProvider {
         try {
             // Validate stream options
             this.validateStreamOptionsOnly(options);
-            // Build message array from options
-            const messages = buildMessagesArray(options);
+            // Build message array from options with multimodal support
+            const hasMultimodalInput = !!(options.input?.images?.length || options.input?.content?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`${functionTag}: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                });
+                // Create multimodal options for buildMultimodalMessagesArray
+                const multimodalOptions = {
+                    input: {
+                        text: options.input?.text || "",
+                        images: options.input?.images,
+                        content: options.input?.content,
+                    },
+                    systemPrompt: options.systemPrompt,
+                    conversationHistory: options.conversationMessages,
+                    provider: this.providerName,
+                    model: this.modelName,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`${functionTag}: Text-only input, using standard message builder`);
+                messages = buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options); // This is where network connection happens!
             // Get all available tools (direct + MCP + external) for streaming
             const shouldUseTools = !options.disableTools && this.supportsTools();

package/dist/lib/utils/imageProcessor.d.ts

@@ -81,4 +81,48 @@ export declare const imageUtils: {
      * Convert file size to human readable format
      */
     formatFileSize: (bytes: number) => string;
+    /**
+     * Convert Buffer to base64 string
+     */
+    bufferToBase64: (buffer: Buffer) => string;
+    /**
+     * Convert base64 string to Buffer
+     */
+    base64ToBuffer: (base64: string) => Buffer;
+    /**
+     * Convert file path to base64 data URI
+     */
+    fileToBase64DataUri: (filePath: string, maxBytes?: number) => Promise<string>;
+    /**
+     * Convert URL to base64 data URI by downloading the image
+     */
+    urlToBase64DataUri: (url: string, { timeoutMs, maxBytes }?: {
+        timeoutMs?: number | undefined;
+        maxBytes?: number | undefined;
+    }) => Promise<string>;
+    /**
+     * Extract base64 data from data URI
+     */
+    extractBase64FromDataUri: (dataUri: string) => string;
+    /**
+     * Extract MIME type from data URI
+     */
+    extractMimeTypeFromDataUri: (dataUri: string) => string;
+    /**
+     * Create data URI from base64 and MIME type
+     */
+    createDataUri: (base64: string, mimeType?: string) => string;
+    /**
+     * Validate base64 string format
+     */
+    isValidBase64: (str: string) => boolean;
+    /**
+     * Get base64 string size in bytes
+     */
+    getBase64Size: (base64: string) => number;
+    /**
+     * Compress base64 image by reducing quality (basic implementation)
+     * Note: This is a placeholder - for production use, consider using sharp or similar
+     */
+    compressBase64: (base64: string, _quality?: number) => string;
 };