@juspay/neurolink 7.49.0 → 7.51.0

package/dist/providers/amazonBedrock.js

@@ -1,8 +1,12 @@
-import { BedrockRuntimeClient, ConverseCommand, ConverseStreamCommand, } from "@aws-sdk/client-bedrock-runtime";
+import { BedrockRuntimeClient, ConverseCommand, ConverseStreamCommand, ImageFormat, } from "@aws-sdk/client-bedrock-runtime";
 import { BedrockClient, ListFoundationModelsCommand, } from "@aws-sdk/client-bedrock";
 import { BaseProvider } from "../core/baseProvider.js";
 import { logger } from "../utils/logger.js";
 import { convertZodToJsonSchema } from "../utils/schemaConversion.js";
+import { buildMultimodalMessagesArray } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
+import { DEFAULT_MAX_STEPS } from "../core/constants.js";
+import { createAnalytics } from "../core/analytics.js";
 // Bedrock-specific types now imported from ../types/providerSpecific.js
 export class AmazonBedrockProvider extends BaseProvider {
     bedrockClient;
@@ -349,6 +353,16 @@ export class AmazonBedrockProvider extends BaseProvider {
                         text: item.text,
                     };
                 }
+                if (item.image) {
+                    return {
+                        image: item.image,
+                    };
+                }
+                if (item.document) {
+                    return {
+                        document: item.document,
+                    };
+                }
                 if (item.toolUse) {
                     return {
                         toolUse: {
@@ -520,6 +534,73 @@ export class AmazonBedrockProvider extends BaseProvider {
         logger.debug(`[AmazonBedrockProvider] Formatted ${bedrockTools.length} tools for Bedrock`);
         return { tools: bedrockTools };
     }
+    // Convert multimodal messages to Bedrock format
+    convertToBedrockMessages(messages) {
+        return messages.map((msg) => {
+            const bedrockMessage = {
+                role: msg.role === "system" ? "user" : msg.role,
+                content: [],
+            };
+            if (typeof msg.content === "string") {
+                bedrockMessage.content.push({ text: msg.content });
+            }
+            else {
+                msg.content.forEach((contentItem) => {
+                    if (contentItem.type === "text" && contentItem.text) {
+                        bedrockMessage.content.push({ text: contentItem.text });
+                    }
+                    else if (contentItem.type === "image" && contentItem.image) {
+                        const imageData = typeof contentItem.image === "string"
+                            ? Buffer.from(contentItem.image.replace(/^data:image\/\w+;base64,/, ""), "base64")
+                            : contentItem.image;
+                        let format = contentItem.mimeType?.split("/")[1] || "png";
+                        if (format === "jpg") {
+                            format = "jpeg";
+                        }
+                        bedrockMessage.content.push({
+                            image: {
+                                format: format === "jpeg"
+                                    ? ImageFormat.JPEG
+                                    : format === "png"
+                                        ? ImageFormat.PNG
+                                        : format === "gif"
+                                            ? ImageFormat.GIF
+                                            : ImageFormat.WEBP,
+                                source: {
+                                    bytes: imageData,
+                                },
+                            },
+                        });
+                    }
+                    else if (contentItem.type === "document" ||
+                        contentItem.type === "pdf" ||
+                        (contentItem.type === "file" &&
+                            contentItem.mimeType?.toLowerCase().startsWith("application/pdf"))) {
+                        let docData;
+                        if (typeof contentItem.data === "string") {
+                            const pdfString = contentItem.data.replace(/^data:application\/pdf;base64,/i, "");
+                            docData = Buffer.from(pdfString, "base64");
+                        }
+                        else {
+                            docData = contentItem.data;
+                        }
+                        bedrockMessage.content.push({
+                            document: {
+                                format: "pdf",
+                                name: typeof contentItem.name === "string" && contentItem.name
+                                    ? contentItem.name
+                                    : "document.pdf",
+                                source: {
+                                    bytes: docData,
+                                },
+                            },
+                        });
+                    }
+                });
+            }
+            return bedrockMessage;
+        });
+    }
     // Bedrock-MCP-Connector compatibility
     getBedrockClient() {
         return this.bedrockClient;
@@ -529,16 +610,43 @@ export class AmazonBedrockProvider extends BaseProvider {
         logger.info("🚀 [AmazonBedrockProvider] Attempting real streaming with ConverseStreamCommand");
         try {
             logger.debug("🟢 [TRACE] executeStream TRY block - about to call streamingConversationLoop");
-            // CRITICAL FIX: Initialize conversation history like generate() does
             // Clear conversation history for new streaming session
             this.conversationHistory = [];
-            // Add user message to conversation - exactly like generate() does
-            const userMessage = {
-                role: "user",
-                content: [{ text: options.input.text }],
-            };
-            this.conversationHistory.push(userMessage);
-            logger.debug(`[AmazonBedrockProvider] Starting streaming conversation with prompt: ${options.input.text}`);
+            // Check for multimodal input (images, PDFs, CSVs, files)
+            const hasMultimodalInput = !!(options.input?.images?.length ||
+                options.input?.content?.length ||
+                options.input?.files?.length ||
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
+            if (hasMultimodalInput) {
+                logger.debug(`[AmazonBedrockProvider] Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                    hasFiles: !!options.input?.files?.length,
+                    fileCount: options.input?.files?.length || 0,
+                    hasCSVFiles: !!options.input?.csvFiles?.length,
+                    csvFileCount: options.input?.csvFiles?.length || 0,
+                    hasPDFFiles: !!options.input?.pdfFiles?.length,
+                    pdfFileCount: options.input?.pdfFiles?.length || 0,
+                });
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
+                const multimodalMessages = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert to Bedrock format
+                this.conversationHistory =
+                    this.convertToBedrockMessages(multimodalMessages);
+            }
+            else {
+                logger.debug(`[AmazonBedrockProvider] Text-only input, using simple message builder`);
+                // Add user message to conversation - simple text-only case
+                const userMessage = {
+                    role: "user",
+                    content: [{ text: options.input.text }],
+                };
+                this.conversationHistory.push(userMessage);
+            }
+            logger.debug(`[AmazonBedrockProvider] Starting streaming conversation with ${this.conversationHistory.length} message(s)`);
             // Call the actual streaming implementation that already exists
             logger.debug("🟢 [TRACE] executeStream - calling streamingConversationLoop NOW");
             const result = await this.streamingConversationLoop(options);
@@ -615,7 +723,8 @@ export class AmazonBedrockProvider extends BaseProvider {
     }
     async streamingConversationLoop(options) {
         logger.debug("🟦 [TRACE] streamingConversationLoop ENTRY");
-        const maxIterations = 10;
+        const startTime = Date.now();
+        const maxIterations = options.maxSteps || DEFAULT_MAX_STEPS;
         let iteration = 0;
         // The REAL issue: ReadableStream errors don't bubble up to the caller
         // So we need to make the first streaming call synchronously to test permissions
@@ -650,7 +759,7 @@ export class AmazonBedrockProvider extends BaseProvider {
                             logger.debug(`[AmazonBedrockProvider] Streaming iteration ${iteration}`);
                             const commandInput = await this.prepareStreamCommand(options);
                             const { stopReason, assistantMessage } = await this.processStreamResponse(commandInput, controller);
-                            const shouldContinue = await this.handleStreamStopReason(stopReason, assistantMessage, controller);
+                            const shouldContinue = await this.handleStreamStopReason(stopReason, assistantMessage, controller, options);
                             if (!shouldContinue) {
                                 break;
                             }
@@ -665,11 +774,22 @@ export class AmazonBedrockProvider extends BaseProvider {
                     }
                 },
             });
+            // Create analytics promise (without token tracking for now due to AWS SDK limitations)
+            const analyticsPromise = Promise.resolve(createAnalytics(this.providerName, this.modelName || this.getDefaultModel(), { usage: { input: 0, output: 0, total: 0 } }, Date.now() - startTime, {
+                requestId: `bedrock-stream-${Date.now()}`,
+                streamingMode: true,
+                note: "Token usage not available from AWS SDK streaming responses",
+            }));
             return {
                 stream: this.convertToAsyncIterable(stream),
                 usage: { total: 0, input: 0, output: 0 },
                 model: this.modelName || this.getDefaultModel(),
                 provider: this.getProviderName(),
+                analytics: analyticsPromise,
+                metadata: {
+                    startTime,
+                    streamId: `bedrock-${Date.now()}`,
+                },
             };
         }
         catch (error) {
@@ -825,7 +945,7 @@ export class AmazonBedrockProvider extends BaseProvider {
         this.conversationHistory.push(assistantMessage);
         return { stopReason, assistantMessage };
     }
-    async handleStreamStopReason(stopReason, assistantMessage, controller) {
+    async handleStreamStopReason(stopReason, assistantMessage, controller, options) {
         if (stopReason === "end_turn" || stopReason === "stop_sequence") {
             // Conversation completed
             controller.close();
@@ -833,7 +953,7 @@ export class AmazonBedrockProvider extends BaseProvider {
         }
         else if (stopReason === "tool_use") {
             logger.debug(`🛠️ [AmazonBedrockProvider] Tool use detected in streaming - executing tools`);
-            await this.executeStreamTools(assistantMessage.content);
+            await this.executeStreamTools(assistantMessage.content, options);
             return true; // Continue conversation loop
         }
         else if (stopReason === "max_tokens") {
@@ -851,10 +971,13 @@ export class AmazonBedrockProvider extends BaseProvider {
             return false;
         }
     }
-    async executeStreamTools(messageContent) {
+    async executeStreamTools(messageContent, options) {
         // Execute all tool uses in the message - ensure 1:1 mapping like Bedrock-MCP-Connector
         const toolResults = [];
         let toolUseCount = 0;
+        // Track tool calls and results for storage (similar to Vertex onStepFinish)
+        const toolCalls = [];
+        const toolResultsForStorage = [];
         // Count toolUse blocks first to ensure 1:1 mapping
         for (const contentItem of messageContent) {
             if (contentItem.toolUse) {
@@ -865,9 +988,23 @@ export class AmazonBedrockProvider extends BaseProvider {
         for (const contentItem of messageContent) {
             if (contentItem.toolUse) {
                 logger.debug(`🔧 [AmazonBedrockProvider] Executing tool: ${contentItem.toolUse.name}`);
+                // Track tool call
+                toolCalls.push({
+                    type: "tool-call",
+                    toolCallId: contentItem.toolUse.toolUseId,
+                    toolName: contentItem.toolUse.name,
+                    args: contentItem.toolUse.input || {},
+                });
                 try {
                     const toolResult = await this.executeSingleTool(contentItem.toolUse.name, contentItem.toolUse.input || {}, contentItem.toolUse.toolUseId);
                     logger.debug(`✅ [AmazonBedrockProvider] Tool execution successful: ${contentItem.toolUse.name}`);
+                    // Track tool result for storage
+                    toolResultsForStorage.push({
+                        type: "tool-result",
+                        toolCallId: contentItem.toolUse.toolUseId,
+                        toolName: contentItem.toolUse.name,
+                        result: toolResult,
+                    });
                     // Ensure exact structure matching Bedrock-MCP-Connector
                     toolResults.push({
                         toolResult: {
@@ -880,6 +1017,13 @@ export class AmazonBedrockProvider extends BaseProvider {
                 catch (error) {
                     logger.error(`❌ [AmazonBedrockProvider] Tool execution failed: ${contentItem.toolUse.name}`, error);
                     const errorMessage = error instanceof Error ? error.message : String(error);
+                    // Track failed tool result
+                    toolResultsForStorage.push({
+                        type: "tool-result",
+                        toolCallId: contentItem.toolUse.toolUseId,
+                        toolName: contentItem.toolUse.name,
+                        result: { error: errorMessage },
+                    });
                     toolResults.push({
                         toolResult: {
                             toolUseId: contentItem.toolUse.toolUseId,
@@ -908,6 +1052,13 @@ export class AmazonBedrockProvider extends BaseProvider {
             };
             this.conversationHistory.push(userMessageWithToolResults);
             logger.debug(`📤 [AmazonBedrockProvider] Added ${toolResults.length} tool results to conversation (1:1 mapping validated)`);
+            // Store tool execution for analytics and debugging (similar to Vertex onStepFinish)
+            this.handleToolExecutionStorage(toolCalls, toolResultsForStorage, options, new Date()).catch((error) => {
+                logger.warn("[AmazonBedrockProvider] Failed to store tool executions", {
+                    provider: this.providerName,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            });
         }
     }
     /**

package/dist/providers/anthropic.js

@@ -8,6 +8,7 @@ import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } fro
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { validateApiKey, createAnthropicConfig, getProviderModel, } from "../utils/providerConfig.js";
 import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 // Configuration helpers - now using consolidated utility
 const getAnthropicApiKey = () => {
@@ -96,7 +97,8 @@ export class AnthropicProvider extends BaseProvider {
             const hasMultimodalInput = !!(options.input?.images?.length ||
                 options.input?.content?.length ||
                 options.input?.files?.length ||
-                options.input?.csvFiles?.length);
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
             let messages;
             if (hasMultimodalInput) {
                 logger.debug(`Anthropic: Detected multimodal input, using multimodal message builder`, {
@@ -108,27 +110,10 @@ export class AnthropicProvider extends BaseProvider {
                     fileCount: options.input?.files?.length || 0,
                     hasCSVFiles: !!options.input?.csvFiles?.length,
                     csvFileCount: options.input?.csvFiles?.length || 0,
+                    hasPDFFiles: !!options.input?.pdfFiles?.length,
+                    pdfFileCount: options.input?.pdfFiles?.length || 0,
                 });
-                // Create multimodal options for buildMultimodalMessagesArray
-                const multimodalOptions = {
-                    input: {
-                        text: options.input?.text || "",
-                        images: options.input?.images,
-                        content: options.input?.content,
-                        files: options.input?.files,
-                        csvFiles: options.input?.csvFiles,
-                    },
-                    csvOptions: options.csvOptions,
-                    systemPrompt: options.systemPrompt,
-                    conversationHistory: options.conversationMessages,
-                    provider: this.providerName,
-                    model: this.modelName,
-                    temperature: options.temperature,
-                    maxTokens: options.maxTokens,
-                    enableAnalytics: options.enableAnalytics,
-                    enableEvaluation: options.enableEvaluation,
-                    context: options.context,
-                };
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
                 const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
                 // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
                 messages = convertToCoreMessages(mm);
@@ -147,6 +132,7 @@ export class AnthropicProvider extends BaseProvider {
                 maxSteps: options.maxSteps || DEFAULT_MAX_STEPS,
                 toolChoice: shouldUseTools ? "auto" : "none",
                 abortSignal: timeoutController?.controller.signal,
+                experimental_telemetry: this.getStreamTelemetryConfig(options),
                 onStepFinish: ({ toolCalls, toolResults }) => {
                     this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
                         logger.warn("[AnthropicProvider] Failed to store tool executions", {

package/dist/providers/azureOpenai.js

@@ -5,6 +5,7 @@ import { APIVersions } from "../types/providers.js";
 import { validateApiKey, createAzureAPIKeyConfig, createAzureEndpointConfig, } from "../utils/providerConfig.js";
 import { logger } from "../utils/logger.js";
 import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 export class AzureOpenAIProvider extends BaseProvider {
@@ -113,7 +114,8 @@ export class AzureOpenAIProvider extends BaseProvider {
             const hasMultimodalInput = !!(options.input?.images?.length ||
                 options.input?.content?.length ||
                 options.input?.files?.length ||
-                options.input?.csvFiles?.length);
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
             let messages;
             if (hasMultimodalInput) {
                 logger.debug(`Azure OpenAI: Detected multimodal input, using multimodal message builder`, {
@@ -122,26 +124,7 @@ export class AzureOpenAIProvider extends BaseProvider {
                     hasContent: !!options.input?.content?.length,
                     contentCount: options.input?.content?.length || 0,
                 });
-                // Create multimodal options for buildMultimodalMessagesArray
-                const multimodalOptions = {
-                    input: {
-                        text: options.input?.text || "",
-                        images: options.input?.images,
-                        content: options.input?.content,
-                        files: options.input?.files,
-                        csvFiles: options.input?.csvFiles,
-                    },
-                    csvOptions: options.csvOptions,
-                    systemPrompt: options.systemPrompt,
-                    conversationHistory: options.conversationMessages,
-                    provider: this.providerName,
-                    model: this.modelName,
-                    temperature: options.temperature,
-                    maxTokens: options.maxTokens,
-                    enableAnalytics: options.enableAnalytics,
-                    enableEvaluation: options.enableEvaluation,
-                    context: options.context,
-                };
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
                 const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
                 // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
                 messages = convertToCoreMessages(mm);
@@ -162,6 +145,7 @@ export class AzureOpenAIProvider extends BaseProvider {
                     : {}),
                 tools,
                 toolChoice: shouldUseTools ? "auto" : "none",
+                experimental_telemetry: this.getStreamTelemetryConfig(options),
                 onStepFinish: ({ toolCalls, toolResults }) => {
                     this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
                         logger.warn("[AzureOpenaiProvider] Failed to store tool executions", {

package/dist/providers/googleAiStudio.js

@@ -8,6 +8,7 @@ import { AuthenticationError, NetworkError, ProviderError, RateLimitError, } fro
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
 import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
 // Google AI Live API types now imported from ../types/providerSpecific.js
 // Import proper types for multimodal message handling
 // Create Google GenAI client
@@ -95,7 +96,8 @@ export class GoogleAIStudioProvider extends BaseProvider {
             const hasMultimodalInput = !!(options.input?.images?.length ||
                 options.input?.content?.length ||
                 options.input?.files?.length ||
-                options.input?.csvFiles?.length);
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
             let messages;
             if (hasMultimodalInput) {
                 logger.debug(`Google AI Studio: Detected multimodal input, using multimodal message builder`, {
@@ -104,26 +106,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
                     hasContent: !!options.input?.content?.length,
                     contentCount: options.input?.content?.length || 0,
                 });
-                // Create multimodal options for buildMultimodalMessagesArray
-                const multimodalOptions = {
-                    input: {
-                        text: options.input?.text || "",
-                        images: options.input?.images,
-                        content: options.input?.content,
-                        files: options.input?.files,
-                        csvFiles: options.input?.csvFiles,
-                    },
-                    csvOptions: options.csvOptions,
-                    systemPrompt: options.systemPrompt,
-                    conversationHistory: options.conversationMessages,
-                    provider: this.providerName,
-                    model: this.modelName,
-                    temperature: options.temperature,
-                    maxTokens: options.maxTokens,
-                    enableAnalytics: options.enableAnalytics,
-                    enableEvaluation: options.enableEvaluation,
-                    context: options.context,
-                };
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
                 const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
                 // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
                 messages = convertToCoreMessages(mm);
@@ -141,6 +124,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
                 maxSteps: options.maxSteps || DEFAULT_MAX_STEPS,
                 toolChoice: shouldUseTools ? "auto" : "none",
                 abortSignal: timeoutController?.controller.signal,
+                experimental_telemetry: this.getStreamTelemetryConfig(options),
                 onStepFinish: ({ toolCalls, toolResults }) => {
                     this.handleToolExecutionStorage(toolCalls, toolResults, options, new Date()).catch((error) => {
                         logger.warn("[GoogleAiStudioProvider] Failed to store tool executions", {

package/dist/providers/googleVertex.js

@@ -599,7 +599,8 @@ export class GoogleVertexProvider extends BaseProvider {
             const hasMultimodalInput = !!(options.input?.images?.length ||
                 options.input?.content?.length ||
                 options.input?.files?.length ||
-                options.input?.csvFiles?.length);
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
             let messages;
             if (hasMultimodalInput) {
                 logger.debug(`${functionTag}: Detected multimodal input, using multimodal message builder`, {
@@ -607,6 +608,8 @@ export class GoogleVertexProvider extends BaseProvider {
                     imageCount: options.input?.images?.length || 0,
                     hasContent: !!options.input?.content?.length,
                     contentCount: options.input?.content?.length || 0,
+                    hasPDFs: !!options.input?.pdfFiles?.length,
+                    pdfCount: options.input?.pdfFiles?.length || 0,
                 });
                 // Create multimodal options for buildMultimodalMessagesArray
                 const multimodalOptions = {
@@ -616,6 +619,7 @@ export class GoogleVertexProvider extends BaseProvider {
                         content: options.input?.content,
                         files: options.input?.files,
                         csvFiles: options.input?.csvFiles,
+                        pdfFiles: options.input?.pdfFiles,
                     },
                     csvOptions: options.csvOptions,
                     systemPrompt: options.systemPrompt,
@@ -666,6 +670,7 @@ export class GoogleVertexProvider extends BaseProvider {
                     maxSteps: options.maxSteps || DEFAULT_MAX_STEPS,
                 }),
                 abortSignal: timeoutController?.controller.signal,
+                experimental_telemetry: this.getStreamTelemetryConfig(options),
                 onError: (event) => {
                     const error = event.error;
                     const errorMessage = error instanceof Error ? error.message : String(error);
@@ -1178,6 +1183,8 @@ export class GoogleVertexProvider extends BaseProvider {
             /^claude-sonnet-4@\d{8}$/,
             /^claude-sonnet-4-5@\d{8}$/,
             /^claude-opus-4@\d{8}$/,
+            /^claude-opus-4-1@\d{8}$/,
+            /^claude-3-7-sonnet@\d{8}$/,
             /^claude-3-5-sonnet-\d{8}$/,
             /^claude-3-5-haiku-\d{8}$/,
             /^claude-3-sonnet-\d{8}$/,

package/dist/providers/huggingFace.js

@@ -3,8 +3,10 @@ import { streamText, } from "ai";
 import { BaseProvider } from "../core/baseProvider.js";
 import { logger } from "../utils/logger.js";
 import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
+import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { validateApiKey, createHuggingFaceConfig, getProviderModel, } from "../utils/providerConfig.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 // Configuration helpers - now using consolidated utility
 const getHuggingFaceApiKey = () => {
@@ -113,13 +115,41 @@ export class HuggingFaceProvider extends BaseProvider {
         try {
             // Enhanced tool handling for HuggingFace models
             const streamOptions = this.prepareStreamOptions(options, analysisSchema);
-            // Build message array from options
-            const messages = await buildMessagesArray(options);
+            // Check for multimodal input (images, PDFs, CSVs, files)
+            const hasMultimodalInput = !!(options.input?.images?.length ||
+                options.input?.content?.length ||
+                options.input?.files?.length ||
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`HuggingFace: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                    hasFiles: !!options.input?.files?.length,
+                    fileCount: options.input?.files?.length || 0,
+                    hasCSVFiles: !!options.input?.csvFiles?.length,
+                    csvFileCount: options.input?.csvFiles?.length || 0,
+                    hasPDFFiles: !!options.input?.pdfFiles?.length,
+                    pdfFileCount: options.input?.pdfFiles?.length || 0,
+                });
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`HuggingFace: Text-only input, using standard message builder`);
+                messages = await buildMessagesArray(options);
+            }
             const result = await streamText({
                 model: this.model,
                 messages: messages,
                 temperature: options.temperature,
                 maxTokens: options.maxTokens, // No default limit - unlimited unless specified
+                maxSteps: options.maxSteps || DEFAULT_MAX_STEPS,
                 tools: streamOptions.tools, // Tools format conversion handled by prepareStreamOptions
                 toolChoice: streamOptions.toolChoice, // Tool choice handled by prepareStreamOptions
                 abortSignal: timeoutController?.controller.signal,

package/dist/providers/litellm.js

@@ -5,7 +5,9 @@ import { logger } from "../utils/logger.js";
 import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { getProviderModel } from "../utils/providerConfig.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { DEFAULT_MAX_STEPS } from "../core/constants.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 // Configuration helpers
 const getLiteLLMConfig = () => {
@@ -120,14 +122,42 @@ export class LiteLLMProvider extends BaseProvider {
         const timeout = this.getTimeout(options);
         const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
         try {
-            // Build message array from options
-            const messages = await buildMessagesArray(options);
+            // Check for multimodal input (images, PDFs, CSVs, files)
+            const hasMultimodalInput = !!(options.input?.images?.length ||
+                options.input?.content?.length ||
+                options.input?.files?.length ||
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`LiteLLM: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                    hasFiles: !!options.input?.files?.length,
+                    fileCount: options.input?.files?.length || 0,
+                    hasCSVFiles: !!options.input?.csvFiles?.length,
+                    csvFileCount: options.input?.csvFiles?.length || 0,
+                    hasPDFFiles: !!options.input?.pdfFiles?.length,
+                    pdfFileCount: options.input?.pdfFiles?.length || 0,
+                });
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`LiteLLM: Text-only input, using standard message builder`);
+                messages = await buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options); // This is where network connection happens!
             const result = streamText({
                 model: model,
                 messages: messages,
                 temperature: options.temperature,
                 maxTokens: options.maxTokens, // No default limit - unlimited unless specified
+                maxSteps: options.maxSteps || DEFAULT_MAX_STEPS,
                 tools: options.tools,
                 toolChoice: "auto",
                 abortSignal: timeoutController?.controller.signal,

package/dist/providers/mistral.js

@@ -6,7 +6,8 @@ import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { DEFAULT_MAX_STEPS } from "../core/constants.js";
 import { validateApiKey, createMistralConfig, getProviderModel, } from "../utils/providerConfig.js";
 import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, convertToCoreMessages, } from "../utils/messageBuilder.js";
+import { buildMultimodalOptions } from "../utils/multimodalOptionsBuilder.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 // Configuration helpers - now using consolidated utility
 const getMistralApiKey = () => {
@@ -49,7 +50,35 @@ export class MistralProvider extends BaseProvider {
             // Get tools consistently with generate method
             const shouldUseTools = !options.disableTools && this.supportsTools();
             const tools = shouldUseTools ? await this.getAllTools() : {};
-            const messages = await buildMessagesArray(options);
+            // Check for multimodal input (images, PDFs, CSVs, files)
+            const hasMultimodalInput = !!(options.input?.images?.length ||
+                options.input?.content?.length ||
+                options.input?.files?.length ||
+                options.input?.csvFiles?.length ||
+                options.input?.pdfFiles?.length);
+            let messages;
+            if (hasMultimodalInput) {
+                logger.debug(`Mistral: Detected multimodal input, using multimodal message builder`, {
+                    hasImages: !!options.input?.images?.length,
+                    imageCount: options.input?.images?.length || 0,
+                    hasContent: !!options.input?.content?.length,
+                    contentCount: options.input?.content?.length || 0,
+                    hasFiles: !!options.input?.files?.length,
+                    fileCount: options.input?.files?.length || 0,
+                    hasCSVFiles: !!options.input?.csvFiles?.length,
+                    csvFileCount: options.input?.csvFiles?.length || 0,
+                    hasPDFFiles: !!options.input?.pdfFiles?.length,
+                    pdfFileCount: options.input?.pdfFiles?.length || 0,
+                });
+                const multimodalOptions = buildMultimodalOptions(options, this.providerName, this.modelName);
+                const mm = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+                // Convert multimodal messages to Vercel AI SDK format (CoreMessage[])
+                messages = convertToCoreMessages(mm);
+            }
+            else {
+                logger.debug(`Mistral: Text-only input, using standard message builder`);
+                messages = await buildMessagesArray(options);
+            }
             const model = await this.getAISDKModelWithMiddleware(options); // This is where network connection happens!
             const result = await streamText({
                 model,