@juspay/neurolink 7.34.0 → 7.36.0

package/dist/lib/utils/messageBuilder.js

@@ -1,8 +1,13 @@
 /**
  * Message Builder Utility
  * Centralized logic for building message arrays from TextGenerationOptions
+ * Enhanced with multimodal support for images
  */
 import { CONVERSATION_INSTRUCTIONS } from "../config/conversationMemory.js";
+import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerImageAdapter.js";
+import { logger } from "./logger.js";
+import { request } from "undici";
+import { readFileSync, existsSync } from "fs";
 /**
  * Build a properly formatted message array for AI providers
  * Combines system prompt, conversation history, and current user prompt
@@ -46,3 +51,277 @@ export function buildMessagesArray(options) {
     }
     return messages;
 }
+/**
+ * Build multimodal message array with image support
+ * Detects when images are present and routes through provider adapter
+ */
+export async function buildMultimodalMessagesArray(options, provider, model) {
+    // Check if this is a multimodal request
+    const hasImages = (options.input.images && options.input.images.length > 0) ||
+        (options.input.content &&
+            options.input.content.some((c) => c.type === "image"));
+    // If no images, use standard message building and convert to MultimodalChatMessage[]
+    if (!hasImages) {
+        const standardMessages = buildMessagesArray(options);
+        return standardMessages.map((msg) => ({ ...msg, content: msg.content }));
+    }
+    // Validate provider supports vision
+    if (!ProviderImageAdapter.supportsVision(provider, model)) {
+        throw new Error(`Provider ${provider} with model ${model} does not support vision processing. ` +
+            `Supported providers: ${ProviderImageAdapter.getVisionProviders().join(", ")}`);
+    }
+    const messages = [];
+    // Build enhanced system prompt
+    let systemPrompt = options.systemPrompt?.trim() || "";
+    // Add conversation-aware instructions when history exists
+    const hasConversationHistory = options.conversationHistory && options.conversationHistory.length > 0;
+    if (hasConversationHistory) {
+        systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
+    }
+    // Add system message if we have one
+    if (systemPrompt.trim()) {
+        messages.push({
+            role: "system",
+            content: systemPrompt.trim(),
+        });
+    }
+    // Add conversation history if available
+    if (hasConversationHistory && options.conversationHistory) {
+        // Convert conversation history to MultimodalChatMessage format
+        options.conversationHistory.forEach((msg) => {
+            messages.push({
+                role: msg.role,
+                content: msg.content,
+            });
+        });
+    }
+    // Handle multimodal content
+    try {
+        let userContent;
+        if (options.input.content && options.input.content.length > 0) {
+            // Advanced content format - convert to provider-specific format
+            userContent = await convertContentToProviderFormat(options.input.content, provider, model);
+        }
+        else if (options.input.images && options.input.images.length > 0) {
+            // Simple images format - convert to provider-specific format
+            userContent = await convertSimpleImagesToProviderFormat(options.input.text, options.input.images, provider, model);
+        }
+        else {
+            // Text-only fallback
+            userContent = options.input.text;
+        }
+        // 🔧 CRITICAL FIX: Handle multimodal content properly for Vercel AI SDK
+        if (typeof userContent === "string") {
+            // Simple text content - use standard MultimodalChatMessage format
+            messages.push({
+                role: "user",
+                content: userContent,
+            });
+        }
+        else {
+            // 🔧 MULTIMODAL CONTENT: Wrap the content array in a proper message object
+            // The Vercel AI SDK expects messages with multimodal content arrays
+            messages.push({
+                role: "user",
+                content: userContent,
+            });
+        }
+        return messages;
+    }
+    catch (error) {
+        MultimodalLogger.logError("MULTIMODAL_BUILD", error, {
+            provider,
+            model,
+            hasImages,
+            imageCount: options.input.images?.length || 0,
+        });
+        throw error;
+    }
+}
+/**
+ * Convert advanced content format to provider-specific format
+ */
+async function convertContentToProviderFormat(content, provider, _model) {
+    const textContent = content.find((c) => c.type === "text");
+    const imageContent = content.filter((c) => c.type === "image");
+    if (!textContent) {
+        throw new Error("Multimodal content must include at least one text element");
+    }
+    if (imageContent.length === 0) {
+        return textContent.text;
+    }
+    // Extract images as Buffer | string array
+    const images = imageContent.map((img) => img.data);
+    return await convertSimpleImagesToProviderFormat(textContent.text, images, provider, _model);
+}
+/**
+ * Check if a string is an internet URL
+ */
+function isInternetUrl(input) {
+    return input.startsWith("http://") || input.startsWith("https://");
+}
+/**
+ * Download image from URL and convert to base64 data URI
+ */
+async function downloadImageFromUrl(url) {
+    try {
+        const response = await request(url, {
+            method: "GET",
+            headersTimeout: 10000, // 10 second timeout for headers
+            bodyTimeout: 30000, // 30 second timeout for body
+            maxRedirections: 5,
+        });
+        if (response.statusCode !== 200) {
+            throw new Error(`HTTP ${response.statusCode}: Failed to download image from ${url}`);
+        }
+        // Get content type from headers
+        const contentType = response.headers["content-type"] || "image/jpeg";
+        // Validate it's an image
+        if (!contentType.startsWith("image/")) {
+            throw new Error(`URL does not point to an image. Content-Type: ${contentType}`);
+        }
+        // Read the response body
+        const chunks = [];
+        for await (const chunk of response.body) {
+            chunks.push(chunk);
+        }
+        const buffer = Buffer.concat(chunks);
+        // Check file size (limit to 10MB)
+        const maxSize = 10 * 1024 * 1024; // 10MB
+        if (buffer.length > maxSize) {
+            throw new Error(`Image too large: ${buffer.length} bytes (max: ${maxSize} bytes)`);
+        }
+        // Convert to base64 data URI
+        const base64 = buffer.toString("base64");
+        const dataUri = `data:${contentType};base64,${base64}`;
+        return dataUri;
+    }
+    catch (error) {
+        MultimodalLogger.logError("URL_DOWNLOAD_FAILED", error, { url });
+        throw new Error(`Failed to download image from ${url}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+/**
+ * Convert simple images format to Vercel AI SDK format with smart auto-detection
+ * - URLs: Downloaded and converted to base64 for Vercel AI SDK compatibility
+ * - Local files: Converted to base64 for Vercel AI SDK compatibility
+ * - Buffers/Data URIs: Processed normally
+ */
+async function convertSimpleImagesToProviderFormat(text, images, provider, _model) {
+    // For Vercel AI SDK, we need to return the content in the standard format
+    // The Vercel AI SDK will handle provider-specific formatting internally
+    // Smart auto-detection: separate URLs from actual image data
+    const urlImages = [];
+    const actualImages = [];
+    images.forEach((image, _index) => {
+        if (typeof image === "string" && isInternetUrl(image)) {
+            // Internet URL - will be downloaded and converted to base64
+            urlImages.push(image);
+        }
+        else {
+            // Actual image data (file path, Buffer, data URI) - process for Vercel AI SDK
+            actualImages.push(image);
+        }
+    });
+    // Download URL images and add to actual images
+    for (const url of urlImages) {
+        try {
+            const downloadedDataUri = await downloadImageFromUrl(url);
+            actualImages.push(downloadedDataUri);
+        }
+        catch (error) {
+            MultimodalLogger.logError("URL_DOWNLOAD_FAILED_SKIPPING", error, { url });
+            // Continue processing other images even if one URL fails
+            logger.warn(`Failed to download image from ${url}, skipping: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    const content = [{ type: "text", text }];
+    // Process all images (including downloaded URLs) for Vercel AI SDK
+    actualImages.forEach((image, index) => {
+        try {
+            // Vercel AI SDK expects { type: 'image', image: Buffer | string, mimeType?: string }
+            // For Vertex AI, we need to include mimeType
+            let imageData;
+            let mimeType = "image/jpeg"; // Default mime type
+            if (typeof image === "string") {
+                if (image.startsWith("data:")) {
+                    // Data URI (including downloaded URLs) - extract mime type and use directly
+                    const match = image.match(/^data:([^;]+);base64,(.+)$/);
+                    if (match) {
+                        mimeType = match[1];
+                        imageData = image; // Keep as data URI for Vercel AI SDK
+                    }
+                    else {
+                        imageData = image;
+                    }
+                }
+                else if (isInternetUrl(image)) {
+                    // This should not happen as URLs are processed separately above
+                    // But handle it gracefully just in case
+                    throw new Error(`Unprocessed URL found in actualImages: ${image}`);
+                }
+                else {
+                    // File path string - convert to base64 data URI
+                    try {
+                        if (existsSync(image)) {
+                            const buffer = readFileSync(image);
+                            const base64 = buffer.toString("base64");
+                            // Detect mime type from file extension
+                            const ext = image.toLowerCase().split(".").pop();
+                            switch (ext) {
+                                case "png":
+                                    mimeType = "image/png";
+                                    break;
+                                case "gif":
+                                    mimeType = "image/gif";
+                                    break;
+                                case "webp":
+                                    mimeType = "image/webp";
+                                    break;
+                                case "bmp":
+                                    mimeType = "image/bmp";
+                                    break;
+                                case "tiff":
+                                case "tif":
+                                    mimeType = "image/tiff";
+                                    break;
+                                default:
+                                    mimeType = "image/jpeg";
+                                    break;
+                            }
+                            imageData = `data:${mimeType};base64,${base64}`;
+                        }
+                        else {
+                            throw new Error(`Image file not found: ${image}`);
+                        }
+                    }
+                    catch (error) {
+                        MultimodalLogger.logError("FILE_PATH_CONVERSION", error, {
+                            index,
+                            filePath: image,
+                        });
+                        throw new Error(`Failed to convert file path to base64: ${image}. ${error}`);
+                    }
+                }
+            }
+            else {
+                // Buffer - convert to base64 data URI
+                const base64 = image.toString("base64");
+                imageData = `data:${mimeType};base64,${base64}`;
+            }
+            content.push({
+                type: "image",
+                image: imageData,
+                mimeType: mimeType, // Add mimeType for Vertex AI compatibility
+            });
+        }
+        catch (error) {
+            MultimodalLogger.logError("ADD_IMAGE_TO_CONTENT", error, {
+                index,
+                provider,
+            });
+            throw error;
+        }
+    });
+    return content;
+}

package/dist/neurolink.js

@@ -854,7 +854,7 @@ export class NeuroLink {
                 // Continue with warning rather than throwing - graceful degradation
             }
         }
-        // Convert to TextGenerationOptions using factory utilities
+        // 🔧 CRITICAL FIX: Convert to TextGenerationOptions while preserving the input object for multimodal support
         const baseOptions = {
             prompt: options.input.text,
             provider: options.provider,
@@ -868,6 +868,7 @@ export class NeuroLink {
             context: options.context,
             evaluationDomain: options.evaluationDomain,
             toolUsageContext: options.toolUsageContext,
+            input: options.input, // This includes text, images, and content arrays
         };
         // Apply factory enhancement using centralized utilities
         const textOptions = enhanceTextGenerationOptions(baseOptions, factoryResult);
@@ -1664,7 +1665,9 @@ export class NeuroLink {
             const processedStream = (async function* (self) {
                 try {
                     for await (const chunk of mcpStream) {
-                        if (chunk && "content" in chunk && typeof chunk.content === "string") {
+                        if (chunk &&
+                            "content" in chunk &&
+                            typeof chunk.content === "string") {
                             accumulatedContent += chunk.content;
                             // Emit chunk event for compatibility
                             self.emitter.emit("response:chunk", chunk.content);
@@ -1941,7 +1944,9 @@ export class NeuroLink {
         const fallbackProcessedStream = (async function* (self) {
             try {
                 for await (const chunk of fallbackStreamResult.stream) {
-                    if (chunk && "content" in chunk && typeof chunk.content === "string") {
+                    if (chunk &&
+                        "content" in chunk &&
+                        typeof chunk.content === "string") {
                         fallbackAccumulatedContent += chunk.content;
                         // Emit chunk event
                         self.emitter.emit("response:chunk", chunk.content);

package/dist/types/content.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Content type definitions for multimodal support
+ * Supports text and image content with provider-specific formatting
+ */
+/**
+ * Text content type for multimodal messages
+ */
+export interface TextContent {
+    type: "text";
+    text: string;
+}
+/**
+ * Image content type for multimodal messages
+ */
+export interface ImageContent {
+    type: "image";
+    data: Buffer | string;
+    mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
+    metadata?: {
+        description?: string;
+        quality?: "low" | "high" | "auto";
+        dimensions?: {
+            width: number;
+            height: number;
+        };
+        filename?: string;
+    };
+}
+/**
+ * Union type for all content types
+ */
+export type Content = TextContent | ImageContent;
+/**
+ * Vision capability information for providers
+ */
+export interface VisionCapability {
+    provider: string;
+    supportedModels: string[];
+    maxImageSize?: number;
+    supportedFormats: string[];
+    maxImagesPerRequest?: number;
+}
+/**
+ * Provider-specific image format requirements
+ */
+export interface ProviderImageFormat {
+    provider: string;
+    format: "data_uri" | "base64" | "inline_data" | "source";
+    requiresPrefix?: boolean;
+    mimeTypeField?: string;
+    dataField?: string;
+}
+/**
+ * Image processing result
+ */
+export interface ProcessedImage {
+    data: string;
+    mediaType: string;
+    size: number;
+    format: "data_uri" | "base64" | "inline_data" | "source";
+}
+/**
+ * Multimodal message structure for provider adapters
+ */
+export interface MultimodalMessage {
+    role: "user" | "assistant" | "system";
+    content: Content[];
+}
+/**
+ * Provider-specific multimodal payload
+ */
+export interface ProviderMultimodalPayload {
+    provider: string;
+    model: string;
+    messages?: MultimodalMessage[];
+    contents?: unknown[];
+    [key: string]: unknown;
+}

package/dist/types/content.js

@@ -0,0 +1,5 @@
+/**
+ * Content type definitions for multimodal support
+ * Supports text and image content with provider-specific formatting
+ */
+export {};

package/dist/types/conversation.d.ts

@@ -66,6 +66,25 @@ export interface ChatMessage {
     /** Content of the message */
     content: string;
 }
+/**
+ * Content format for multimodal messages (used internally)
+ */
+export interface MessageContent {
+    type: string;
+    text?: string;
+    image?: string;
+    mimeType?: string;
+    [key: string]: unknown;
+}
+/**
+ * Extended chat message for multimodal support (internal use)
+ */
+export interface MultimodalChatMessage {
+    /** Role of the message sender */
+    role: "user" | "assistant" | "system";
+    /** Content of the message - can be text or multimodal content array */
+    content: string | MessageContent[];
+}
 /**
  * Events emitted by conversation memory system
  */

package/dist/types/generateTypes.d.ts

@@ -6,13 +6,16 @@ import type { EvaluationData } from "./evaluation.js";
 import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 import type { JsonValue } from "./common.js";
+import type { TextContent, ImageContent } from "./content.js";
 /**
  * Generate function options type - Primary method for content generation
- * Future-ready for multi-modal capabilities while maintaining text focus
+ * Supports multimodal content while maintaining backward compatibility
  */
 export type GenerateOptions = {
     input: {
         text: string;
+        images?: Array<Buffer | string>;
+        content?: Array<TextContent | ImageContent>;
     };
     output?: {
         format?: "text" | "structured" | "json";

package/dist/types/streamTypes.d.ts

@@ -5,6 +5,7 @@ import type { AnalyticsData, TokenUsage } from "./analytics.js";
 import type { EvaluationData } from "./evaluation.js";
 import type { UnknownRecord, JsonValue } from "./common.js";
 import type { ChatMessage } from "./conversation.js";
+import type { TextContent, ImageContent } from "./content.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 /**
  * Progress tracking and metadata for streaming operations
@@ -118,10 +119,12 @@ export interface AudioChunk {
     channels: number;
     encoding: PCMEncoding;
 }
-export type StreamOptions = {
+export interface StreamOptions {
     input: {
-        text?: string;
+        text: string;
         audio?: AudioInputSpec;
+        images?: Array<Buffer | string>;
+        content?: Array<TextContent | ImageContent>;
     };
     output?: {
         format?: "text" | "structured" | "json";
@@ -166,7 +169,7 @@ export type StreamOptions = {
     };
     conversationMessages?: ChatMessage[];
     middleware?: MiddlewareFactoryOptions;
-};
+}
 /**
  * Stream function result type - Primary output format for streaming
  * Future-ready for multi-modal outputs while maintaining text focus

package/dist/utils/imageProcessor.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Image processing utilities for multimodal support
+ * Handles format conversion for different AI providers
+ */
+import type { ProcessedImage } from "../types/content.js";
+/**
+ * Image processor class for handling provider-specific image formatting
+ */
+export declare class ImageProcessor {
+    /**
+     * Process image for OpenAI (requires data URI format)
+     */
+    static processImageForOpenAI(image: Buffer | string): string;
+    /**
+     * Process image for Google AI (requires base64 without data URI prefix)
+     */
+    static processImageForGoogle(image: Buffer | string): {
+        mimeType: string;
+        data: string;
+    };
+    /**
+     * Process image for Anthropic (requires base64 without data URI prefix)
+     */
+    static processImageForAnthropic(image: Buffer | string): {
+        mediaType: string;
+        data: string;
+    };
+    /**
+     * Process image for Vertex AI (model-specific routing)
+     */
+    static processImageForVertex(image: Buffer | string, model: string): {
+        mimeType?: string;
+        mediaType?: string;
+        data: string;
+    };
+    /**
+     * Detect image type from filename or data
+     */
+    static detectImageType(input: string | Buffer): string;
+    /**
+     * Validate image size (default 10MB limit)
+     */
+    static validateImageSize(data: Buffer | string, maxSize?: number): boolean;
+    /**
+     * Validate image format
+     */
+    static validateImageFormat(mediaType: string): boolean;
+    /**
+     * Get image dimensions from Buffer (basic implementation)
+     */
+    static getImageDimensions(buffer: Buffer): {
+        width: number;
+        height: number;
+    } | null;
+    /**
+     * Convert image to ProcessedImage format
+     */
+    static processImage(image: Buffer | string, provider: string, model?: string): ProcessedImage;
+}
+/**
+ * Utility functions for image handling
+ */
+export declare const imageUtils: {
+    /**
+     * Check if a string is a valid data URI
+     */
+    isDataUri: (str: string) => boolean;
+    /**
+     * Check if a string is a valid URL
+     */
+    isUrl: (str: string) => boolean;
+    /**
+     * Check if a string is base64 encoded
+     */
+    isBase64: (str: string) => boolean;
+    /**
+     * Extract file extension from filename or URL
+     */
+    getFileExtension: (filename: string) => string | null;
+    /**
+     * Convert file size to human readable format
+     */
+    formatFileSize: (bytes: number) => string;
+};