@juspay/neurolink 7.35.0 → 7.36.0

package/dist/neurolink.js

@@ -854,7 +854,7 @@ export class NeuroLink {
                 // Continue with warning rather than throwing - graceful degradation
             }
         }
-        // Convert to TextGenerationOptions using factory utilities
+        // 🔧 CRITICAL FIX: Convert to TextGenerationOptions while preserving the input object for multimodal support
         const baseOptions = {
             prompt: options.input.text,
             provider: options.provider,
@@ -868,6 +868,7 @@ export class NeuroLink {
             context: options.context,
             evaluationDomain: options.evaluationDomain,
             toolUsageContext: options.toolUsageContext,
+            input: options.input, // This includes text, images, and content arrays
         };
         // Apply factory enhancement using centralized utilities
         const textOptions = enhanceTextGenerationOptions(baseOptions, factoryResult);
@@ -1664,7 +1665,9 @@ export class NeuroLink {
             const processedStream = (async function* (self) {
                 try {
                     for await (const chunk of mcpStream) {
-                        if (chunk && "content" in chunk && typeof chunk.content === "string") {
+                        if (chunk &&
+                            "content" in chunk &&
+                            typeof chunk.content === "string") {
                             accumulatedContent += chunk.content;
                             // Emit chunk event for compatibility
                             self.emitter.emit("response:chunk", chunk.content);
@@ -1941,7 +1944,9 @@ export class NeuroLink {
         const fallbackProcessedStream = (async function* (self) {
             try {
                 for await (const chunk of fallbackStreamResult.stream) {
-                    if (chunk && "content" in chunk && typeof chunk.content === "string") {
+                    if (chunk &&
+                        "content" in chunk &&
+                        typeof chunk.content === "string") {
                         fallbackAccumulatedContent += chunk.content;
                         // Emit chunk event
                         self.emitter.emit("response:chunk", chunk.content);

package/dist/types/content.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Content type definitions for multimodal support
+ * Supports text and image content with provider-specific formatting
+ */
+/**
+ * Text content type for multimodal messages
+ */
+export interface TextContent {
+    type: "text";
+    text: string;
+}
+/**
+ * Image content type for multimodal messages
+ */
+export interface ImageContent {
+    type: "image";
+    data: Buffer | string;
+    mediaType?: "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp" | "image/tiff";
+    metadata?: {
+        description?: string;
+        quality?: "low" | "high" | "auto";
+        dimensions?: {
+            width: number;
+            height: number;
+        };
+        filename?: string;
+    };
+}
+/**
+ * Union type for all content types
+ */
+export type Content = TextContent | ImageContent;
+/**
+ * Vision capability information for providers
+ */
+export interface VisionCapability {
+    provider: string;
+    supportedModels: string[];
+    maxImageSize?: number;
+    supportedFormats: string[];
+    maxImagesPerRequest?: number;
+}
+/**
+ * Provider-specific image format requirements
+ */
+export interface ProviderImageFormat {
+    provider: string;
+    format: "data_uri" | "base64" | "inline_data" | "source";
+    requiresPrefix?: boolean;
+    mimeTypeField?: string;
+    dataField?: string;
+}
+/**
+ * Image processing result
+ */
+export interface ProcessedImage {
+    data: string;
+    mediaType: string;
+    size: number;
+    format: "data_uri" | "base64" | "inline_data" | "source";
+}
+/**
+ * Multimodal message structure for provider adapters
+ */
+export interface MultimodalMessage {
+    role: "user" | "assistant" | "system";
+    content: Content[];
+}
+/**
+ * Provider-specific multimodal payload
+ */
+export interface ProviderMultimodalPayload {
+    provider: string;
+    model: string;
+    messages?: MultimodalMessage[];
+    contents?: unknown[];
+    [key: string]: unknown;
+}

package/dist/types/content.js

@@ -0,0 +1,5 @@
+/**
+ * Content type definitions for multimodal support
+ * Supports text and image content with provider-specific formatting
+ */
+export {};

package/dist/types/conversation.d.ts

@@ -66,6 +66,25 @@ export interface ChatMessage {
     /** Content of the message */
     content: string;
 }
+/**
+ * Content format for multimodal messages (used internally)
+ */
+export interface MessageContent {
+    type: string;
+    text?: string;
+    image?: string;
+    mimeType?: string;
+    [key: string]: unknown;
+}
+/**
+ * Extended chat message for multimodal support (internal use)
+ */
+export interface MultimodalChatMessage {
+    /** Role of the message sender */
+    role: "user" | "assistant" | "system";
+    /** Content of the message - can be text or multimodal content array */
+    content: string | MessageContent[];
+}
 /**
  * Events emitted by conversation memory system
  */

package/dist/types/generateTypes.d.ts

@@ -6,13 +6,16 @@ import type { EvaluationData } from "./evaluation.js";
 import type { ChatMessage, ConversationMemoryConfig } from "./conversation.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 import type { JsonValue } from "./common.js";
+import type { TextContent, ImageContent } from "./content.js";
 /**
  * Generate function options type - Primary method for content generation
- * Future-ready for multi-modal capabilities while maintaining text focus
+ * Supports multimodal content while maintaining backward compatibility
  */
 export type GenerateOptions = {
     input: {
         text: string;
+        images?: Array<Buffer | string>;
+        content?: Array<TextContent | ImageContent>;
     };
     output?: {
         format?: "text" | "structured" | "json";

package/dist/types/streamTypes.d.ts

@@ -5,6 +5,7 @@ import type { AnalyticsData, TokenUsage } from "./analytics.js";
 import type { EvaluationData } from "./evaluation.js";
 import type { UnknownRecord, JsonValue } from "./common.js";
 import type { ChatMessage } from "./conversation.js";
+import type { TextContent, ImageContent } from "./content.js";
 import type { MiddlewareFactoryOptions } from "./middlewareTypes.js";
 /**
  * Progress tracking and metadata for streaming operations
@@ -118,10 +119,12 @@ export interface AudioChunk {
     channels: number;
     encoding: PCMEncoding;
 }
-export type StreamOptions = {
+export interface StreamOptions {
     input: {
-        text?: string;
+        text: string;
         audio?: AudioInputSpec;
+        images?: Array<Buffer | string>;
+        content?: Array<TextContent | ImageContent>;
     };
     output?: {
         format?: "text" | "structured" | "json";
@@ -166,7 +169,7 @@ export type StreamOptions = {
     };
     conversationMessages?: ChatMessage[];
     middleware?: MiddlewareFactoryOptions;
-};
+}
 /**
  * Stream function result type - Primary output format for streaming
  * Future-ready for multi-modal outputs while maintaining text focus

package/dist/utils/imageProcessor.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Image processing utilities for multimodal support
+ * Handles format conversion for different AI providers
+ */
+import type { ProcessedImage } from "../types/content.js";
+/**
+ * Image processor class for handling provider-specific image formatting
+ */
+export declare class ImageProcessor {
+    /**
+     * Process image for OpenAI (requires data URI format)
+     */
+    static processImageForOpenAI(image: Buffer | string): string;
+    /**
+     * Process image for Google AI (requires base64 without data URI prefix)
+     */
+    static processImageForGoogle(image: Buffer | string): {
+        mimeType: string;
+        data: string;
+    };
+    /**
+     * Process image for Anthropic (requires base64 without data URI prefix)
+     */
+    static processImageForAnthropic(image: Buffer | string): {
+        mediaType: string;
+        data: string;
+    };
+    /**
+     * Process image for Vertex AI (model-specific routing)
+     */
+    static processImageForVertex(image: Buffer | string, model: string): {
+        mimeType?: string;
+        mediaType?: string;
+        data: string;
+    };
+    /**
+     * Detect image type from filename or data
+     */
+    static detectImageType(input: string | Buffer): string;
+    /**
+     * Validate image size (default 10MB limit)
+     */
+    static validateImageSize(data: Buffer | string, maxSize?: number): boolean;
+    /**
+     * Validate image format
+     */
+    static validateImageFormat(mediaType: string): boolean;
+    /**
+     * Get image dimensions from Buffer (basic implementation)
+     */
+    static getImageDimensions(buffer: Buffer): {
+        width: number;
+        height: number;
+    } | null;
+    /**
+     * Convert image to ProcessedImage format
+     */
+    static processImage(image: Buffer | string, provider: string, model?: string): ProcessedImage;
+}
+/**
+ * Utility functions for image handling
+ */
+export declare const imageUtils: {
+    /**
+     * Check if a string is a valid data URI
+     */
+    isDataUri: (str: string) => boolean;
+    /**
+     * Check if a string is a valid URL
+     */
+    isUrl: (str: string) => boolean;
+    /**
+     * Check if a string is base64 encoded
+     */
+    isBase64: (str: string) => boolean;
+    /**
+     * Extract file extension from filename or URL
+     */
+    getFileExtension: (filename: string) => string | null;
+    /**
+     * Convert file size to human readable format
+     */
+    formatFileSize: (bytes: number) => string;
+};

package/dist/utils/imageProcessor.js

@@ -0,0 +1,362 @@
+/**
+ * Image processing utilities for multimodal support
+ * Handles format conversion for different AI providers
+ */
+import { logger } from "./logger.js";
+/**
+ * Image processor class for handling provider-specific image formatting
+ */
+export class ImageProcessor {
+    /**
+     * Process image for OpenAI (requires data URI format)
+     */
+    static processImageForOpenAI(image) {
+        try {
+            if (typeof image === "string") {
+                // Handle URLs
+                if (image.startsWith("http")) {
+                    return image;
+                }
+                // Handle data URIs
+                if (image.startsWith("data:")) {
+                    return image;
+                }
+                // Handle base64 - convert to data URI
+                return `data:image/jpeg;base64,${image}`;
+            }
+            // Handle Buffer - convert to data URI
+            const base64 = image.toString("base64");
+            return `data:image/jpeg;base64,${base64}`;
+        }
+        catch (error) {
+            logger.error("Failed to process image for OpenAI:", error);
+            throw new Error(`Image processing failed for OpenAI: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Process image for Google AI (requires base64 without data URI prefix)
+     */
+    static processImageForGoogle(image) {
+        try {
+            let base64Data;
+            let mimeType = "image/jpeg"; // Default
+            if (typeof image === "string") {
+                if (image.startsWith("data:")) {
+                    // Extract mime type and base64 from data URI
+                    const match = image.match(/^data:([^;]+);base64,(.+)$/);
+                    if (match) {
+                        mimeType = match[1];
+                        base64Data = match[2];
+                    }
+                    else {
+                        base64Data = image.split(",")[1] || image;
+                    }
+                }
+                else {
+                    base64Data = image;
+                }
+            }
+            else {
+                base64Data = image.toString("base64");
+            }
+            return {
+                mimeType,
+                data: base64Data, // Google wants base64 WITHOUT data URI prefix
+            };
+        }
+        catch (error) {
+            logger.error("Failed to process image for Google AI:", error);
+            throw new Error(`Image processing failed for Google AI: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Process image for Anthropic (requires base64 without data URI prefix)
+     */
+    static processImageForAnthropic(image) {
+        try {
+            let base64Data;
+            let mediaType = "image/jpeg"; // Default
+            if (typeof image === "string") {
+                if (image.startsWith("data:")) {
+                    // Extract mime type and base64 from data URI
+                    const match = image.match(/^data:([^;]+);base64,(.+)$/);
+                    if (match) {
+                        mediaType = match[1];
+                        base64Data = match[2];
+                    }
+                    else {
+                        base64Data = image.split(",")[1] || image;
+                    }
+                }
+                else {
+                    base64Data = image;
+                }
+            }
+            else {
+                base64Data = image.toString("base64");
+            }
+            return {
+                mediaType,
+                data: base64Data, // Anthropic wants base64 WITHOUT data URI prefix
+            };
+        }
+        catch (error) {
+            logger.error("Failed to process image for Anthropic:", error);
+            throw new Error(`Image processing failed for Anthropic: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Process image for Vertex AI (model-specific routing)
+     */
+    static processImageForVertex(image, model) {
+        try {
+            // Route based on model type
+            if (model.includes("gemini")) {
+                // Use Google AI format for Gemini models
+                return ImageProcessor.processImageForGoogle(image);
+            }
+            else if (model.includes("claude")) {
+                // Use Anthropic format for Claude models
+                return ImageProcessor.processImageForAnthropic(image);
+            }
+            else {
+                // Default to Google format
+                return ImageProcessor.processImageForGoogle(image);
+            }
+        }
+        catch (error) {
+            logger.error("Failed to process image for Vertex AI:", error);
+            throw new Error(`Image processing failed for Vertex AI: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+    /**
+     * Detect image type from filename or data
+     */
+    static detectImageType(input) {
+        try {
+            if (typeof input === "string") {
+                // Check if it's a data URI
+                if (input.startsWith("data:")) {
+                    const match = input.match(/^data:([^;]+);/);
+                    return match ? match[1] : "image/jpeg";
+                }
+                // Check if it's a filename
+                const extension = input.toLowerCase().split(".").pop();
+                const imageTypes = {
+                    jpg: "image/jpeg",
+                    jpeg: "image/jpeg",
+                    png: "image/png",
+                    gif: "image/gif",
+                    webp: "image/webp",
+                    bmp: "image/bmp",
+                    tiff: "image/tiff",
+                    tif: "image/tiff",
+                };
+                return imageTypes[extension || ""] || "image/jpeg";
+            }
+            // For Buffer, try to detect from magic bytes
+            if (input.length >= 4) {
+                const header = input.subarray(0, 4);
+                // PNG: 89 50 4E 47
+                if (header[0] === 0x89 &&
+                    header[1] === 0x50 &&
+                    header[2] === 0x4e &&
+                    header[3] === 0x47) {
+                    return "image/png";
+                }
+                // JPEG: FF D8 FF
+                if (header[0] === 0xff && header[1] === 0xd8 && header[2] === 0xff) {
+                    return "image/jpeg";
+                }
+                // GIF: 47 49 46 38
+                if (header[0] === 0x47 &&
+                    header[1] === 0x49 &&
+                    header[2] === 0x46 &&
+                    header[3] === 0x38) {
+                    return "image/gif";
+                }
+                // WebP: check for RIFF and WEBP
+                if (input.length >= 12) {
+                    const riff = input.subarray(0, 4);
+                    const webp = input.subarray(8, 12);
+                    if (riff.toString() === "RIFF" && webp.toString() === "WEBP") {
+                        return "image/webp";
+                    }
+                }
+            }
+            return "image/jpeg"; // Default fallback
+        }
+        catch (error) {
+            logger.warn("Failed to detect image type, using default:", error);
+            return "image/jpeg";
+        }
+    }
+    /**
+     * Validate image size (default 10MB limit)
+     */
+    static validateImageSize(data, maxSize = 10 * 1024 * 1024) {
+        try {
+            const size = typeof data === "string"
+                ? Buffer.byteLength(data, "base64")
+                : data.length;
+            return size <= maxSize;
+        }
+        catch (error) {
+            logger.warn("Failed to validate image size:", error);
+            return false;
+        }
+    }
+    /**
+     * Validate image format
+     */
+    static validateImageFormat(mediaType) {
+        const supportedFormats = [
+            "image/jpeg",
+            "image/png",
+            "image/gif",
+            "image/webp",
+            "image/bmp",
+            "image/tiff",
+        ];
+        return supportedFormats.includes(mediaType.toLowerCase());
+    }
+    /**
+     * Get image dimensions from Buffer (basic implementation)
+     */
+    static getImageDimensions(buffer) {
+        try {
+            // Basic PNG dimension extraction
+            if (buffer.length >= 24 &&
+                buffer.subarray(0, 8).toString("hex") === "89504e470d0a1a0a") {
+                const width = buffer.readUInt32BE(16);
+                const height = buffer.readUInt32BE(20);
+                return { width, height };
+            }
+            // Basic JPEG dimension extraction (simplified)
+            if (buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xd8) {
+                // This is a very basic implementation
+                // For production, consider using a proper image library
+                return null;
+            }
+            return null;
+        }
+        catch (error) {
+            logger.warn("Failed to extract image dimensions:", error);
+            return null;
+        }
+    }
+    /**
+     * Convert image to ProcessedImage format
+     */
+    static processImage(image, provider, model) {
+        try {
+            const mediaType = ImageProcessor.detectImageType(image);
+            const size = typeof image === "string"
+                ? Buffer.byteLength(image, "base64")
+                : image.length;
+            let data;
+            let format;
+            switch (provider.toLowerCase()) {
+                case "openai":
+                    data = ImageProcessor.processImageForOpenAI(image);
+                    format = "data_uri";
+                    break;
+                case "google-ai":
+                case "google": {
+                    const googleResult = ImageProcessor.processImageForGoogle(image);
+                    data = googleResult.data;
+                    format = "base64";
+                    break;
+                }
+                case "anthropic": {
+                    const anthropicResult = ImageProcessor.processImageForAnthropic(image);
+                    data = anthropicResult.data;
+                    format = "base64";
+                    break;
+                }
+                case "vertex": {
+                    const vertexResult = ImageProcessor.processImageForVertex(image, model || "");
+                    data = vertexResult.data;
+                    format = "base64";
+                    break;
+                }
+                default:
+                    // Default to base64
+                    if (typeof image === "string") {
+                        data = image.startsWith("data:")
+                            ? image.split(",")[1] || image
+                            : image;
+                    }
+                    else {
+                        data = image.toString("base64");
+                    }
+                    format = "base64";
+            }
+            return {
+                data,
+                mediaType,
+                size,
+                format,
+            };
+        }
+        catch (error) {
+            logger.error(`Failed to process image for ${provider}:`, error);
+            throw new Error(`Image processing failed: ${error instanceof Error ? error.message : "Unknown error"}`);
+        }
+    }
+}
+/**
+ * Utility functions for image handling
+ */
+export const imageUtils = {
+    /**
+     * Check if a string is a valid data URI
+     */
+    isDataUri: (str) => {
+        return (typeof str === "string" &&
+            str.startsWith("data:") &&
+            str.includes("base64,"));
+    },
+    /**
+     * Check if a string is a valid URL
+     */
+    isUrl: (str) => {
+        try {
+            new URL(str);
+            return str.startsWith("http://") || str.startsWith("https://");
+        }
+        catch {
+            return false;
+        }
+    },
+    /**
+     * Check if a string is base64 encoded
+     */
+    isBase64: (str) => {
+        try {
+            return btoa(atob(str)) === str;
+        }
+        catch {
+            return false;
+        }
+    },
+    /**
+     * Extract file extension from filename or URL
+     */
+    getFileExtension: (filename) => {
+        const match = filename.match(/\.([^.]+)$/);
+        return match ? match[1].toLowerCase() : null;
+    },
+    /**
+     * Convert file size to human readable format
+     */
+    formatFileSize: (bytes) => {
+        if (bytes === 0) {
+            return "0 Bytes";
+        }
+        const k = 1024;
+        const sizes = ["Bytes", "KB", "MB", "GB"];
+        const i = Math.floor(Math.log(bytes) / Math.log(k));
+        return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
+    },
+};

package/dist/utils/messageBuilder.d.ts

@@ -1,13 +1,20 @@
 /**
  * Message Builder Utility
  * Centralized logic for building message arrays from TextGenerationOptions
+ * Enhanced with multimodal support for images
  */
-import type { ChatMessage } from "../types/conversation.js";
+import type { ChatMessage, MultimodalChatMessage } from "../types/conversation.js";
 import type { TextGenerationOptions } from "../types/index.js";
 import type { StreamOptions } from "../types/streamTypes.js";
+import type { GenerateOptions } from "../types/generateTypes.js";
 /**
  * Build a properly formatted message array for AI providers
  * Combines system prompt, conversation history, and current user prompt
  * Supports both TextGenerationOptions and StreamOptions
  */
 export declare function buildMessagesArray(options: TextGenerationOptions | StreamOptions): ChatMessage[];
+/**
+ * Build multimodal message array with image support
+ * Detects when images are present and routes through provider adapter
+ */
+export declare function buildMultimodalMessagesArray(options: GenerateOptions, provider: string, model: string): Promise<MultimodalChatMessage[]>;