@juspay/neurolink 7.35.0 → 7.37.0

package/dist/lib/adapters/providerImageAdapter.js

@@ -0,0 +1,257 @@
+/**
+ * Provider Image Adapter - Smart routing for multimodal content
+ * Handles provider-specific image formatting and vision capability validation
+ */
+import { logger } from "../utils/logger.js";
+import { ImageProcessor } from "../utils/imageProcessor.js";
+/**
+ * Simplified logger for essential error reporting only
+ */
+export class MultimodalLogger {
+    static logError(step, error, context) {
+        logger.error(`Multimodal ${step} failed: ${error.message}`);
+        if (process.env.NODE_ENV === "development") {
+            logger.error("Context:", JSON.stringify(context, null, 2));
+            logger.error("Stack:", error.stack);
+        }
+    }
+}
+/**
+ * Vision capability definitions for each provider
+ */
+const VISION_CAPABILITIES = {
+    openai: ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-4-vision-preview"],
+    "google-ai": [
+        "gemini-2.5-pro",
+        "gemini-2.5-flash",
+        "gemini-1.5-pro",
+        "gemini-1.5-flash",
+        "gemini-pro-vision",
+    ],
+    anthropic: [
+        "claude-3-5-sonnet",
+        "claude-3-opus",
+        "claude-3-sonnet",
+        "claude-3-haiku",
+    ],
+    vertex: [
+        // Gemini models on Vertex AI
+        "gemini-2.5-pro",
+        "gemini-2.5-flash",
+        "gemini-1.5-pro",
+        "gemini-1.5-flash",
+        // Claude models on Vertex AI (with actual Vertex naming patterns)
+        "claude-3-5-sonnet",
+        "claude-3-opus",
+        "claude-3-sonnet",
+        "claude-3-haiku",
+        "claude-sonnet-3",
+        "claude-sonnet-4",
+        "claude-opus-3",
+        "claude-haiku-3",
+        // Additional Vertex AI Claude model patterns
+        "claude-3.5-sonnet",
+        "claude-3.5-haiku",
+        "claude-3.0-sonnet",
+        "claude-3.0-opus",
+        // Versioned model names (e.g., claude-sonnet-4@20250514)
+        "claude-sonnet-4@",
+        "claude-opus-3@",
+        "claude-haiku-3@",
+        "claude-3-5-sonnet@",
+    ],
+};
+/**
+ * Provider Image Adapter - Smart routing and formatting
+ */
+export class ProviderImageAdapter {
+    /**
+     * Main adapter method - routes to provider-specific formatting
+     */
+    static async adaptForProvider(text, images, provider, model) {
+        try {
+            // Validate provider supports vision
+            this.validateVisionSupport(provider, model);
+            let adaptedPayload;
+            // Process images based on provider requirements
+            switch (provider.toLowerCase()) {
+                case "openai":
+                    adaptedPayload = this.formatForOpenAI(text, images);
+                    break;
+                case "google-ai":
+                case "google":
+                    adaptedPayload = this.formatForGoogleAI(text, images);
+                    break;
+                case "anthropic":
+                    adaptedPayload = this.formatForAnthropic(text, images);
+                    break;
+                case "vertex":
+                    adaptedPayload = this.formatForVertex(text, images, model);
+                    break;
+                default:
+                    throw new Error(`Vision not supported for provider: ${provider}`);
+            }
+            return adaptedPayload;
+        }
+        catch (error) {
+            MultimodalLogger.logError("ADAPTATION", error, {
+                provider,
+                model,
+                imageCount: images.length,
+            });
+            throw error;
+        }
+    }
+    /**
+     * Format content for OpenAI (GPT-4o format)
+     */
+    static formatForOpenAI(text, images) {
+        const content = [{ type: "text", text }];
+        images.forEach((image, index) => {
+            try {
+                const imageUrl = ImageProcessor.processImageForOpenAI(image);
+                content.push({
+                    type: "image_url",
+                    image_url: { url: imageUrl },
+                });
+            }
+            catch (error) {
+                MultimodalLogger.logError("PROCESS_IMAGE", error, {
+                    index,
+                    provider: "openai",
+                });
+                throw error;
+            }
+        });
+        return { messages: [{ role: "user", content }] };
+    }
+    /**
+     * Format content for Google AI (Gemini format)
+     */
+    static formatForGoogleAI(text, images) {
+        const parts = [{ text }];
+        images.forEach((image, index) => {
+            try {
+                const { mimeType, data } = ImageProcessor.processImageForGoogle(image);
+                parts.push({
+                    inlineData: { mimeType, data },
+                });
+            }
+            catch (error) {
+                MultimodalLogger.logError("PROCESS_IMAGE", error, {
+                    index,
+                    provider: "google-ai",
+                });
+                throw error;
+            }
+        });
+        return { contents: [{ parts }] };
+    }
+    /**
+     * Format content for Anthropic (Claude format)
+     */
+    static formatForAnthropic(text, images) {
+        const content = [{ type: "text", text }];
+        images.forEach((image, index) => {
+            try {
+                const { mediaType, data } = ImageProcessor.processImageForAnthropic(image);
+                content.push({
+                    type: "image",
+                    source: {
+                        type: "base64",
+                        media_type: mediaType,
+                        data,
+                    },
+                });
+            }
+            catch (error) {
+                MultimodalLogger.logError("PROCESS_IMAGE", error, {
+                    index,
+                    provider: "anthropic",
+                });
+                throw error;
+            }
+        });
+        return { messages: [{ role: "user", content }] };
+    }
+    /**
+     * Format content for Vertex AI (model-specific routing)
+     */
+    static formatForVertex(text, images, model) {
+        // Route based on model type
+        if (model.includes("gemini")) {
+            return this.formatForGoogleAI(text, images);
+        }
+        else if (model.includes("claude")) {
+            return this.formatForAnthropic(text, images);
+        }
+        else {
+            return this.formatForGoogleAI(text, images);
+        }
+    }
+    /**
+     * Validate that provider and model support vision
+     */
+    static validateVisionSupport(provider, model) {
+        const normalizedProvider = provider.toLowerCase();
+        const supportedModels = VISION_CAPABILITIES[normalizedProvider];
+        if (!supportedModels) {
+            throw new Error(`Provider ${provider} does not support vision processing. ` +
+                `Supported providers: ${Object.keys(VISION_CAPABILITIES).join(", ")}`);
+        }
+        const isSupported = supportedModels.some((supportedModel) => model.toLowerCase().includes(supportedModel.toLowerCase()));
+        if (!isSupported) {
+            throw new Error(`Provider ${provider} with model ${model} does not support vision processing. ` +
+                `Supported models for ${provider}: ${supportedModels.join(", ")}`);
+        }
+    }
+    /**
+     * Convert simple images array to advanced content format
+     */
+    static convertToContent(text, images) {
+        const content = [{ type: "text", text }];
+        if (images && images.length > 0) {
+            images.forEach((image) => {
+                content.push({
+                    type: "image",
+                    data: image,
+                    mediaType: ImageProcessor.detectImageType(image),
+                });
+            });
+        }
+        return content;
+    }
+    /**
+     * Check if provider supports multimodal content
+     */
+    static supportsVision(provider, model) {
+        try {
+            const normalizedProvider = provider.toLowerCase();
+            const supportedModels = VISION_CAPABILITIES[normalizedProvider];
+            if (!supportedModels) {
+                return false;
+            }
+            if (!model) {
+                return true; // Provider supports vision, but need to check specific model
+            }
+            return supportedModels.some((supportedModel) => model.toLowerCase().includes(supportedModel.toLowerCase()));
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get supported models for a provider
+     */
+    static getSupportedModels(provider) {
+        const normalizedProvider = provider.toLowerCase();
+        const models = VISION_CAPABILITIES[normalizedProvider];
+        return models ? [...models] : [];
+    }
+    /**
+     * Get all vision-capable providers
+     */
+    static getVisionProviders() {
+        return Object.keys(VISION_CAPABILITIES);
+    }
+}

package/dist/lib/config/taskClassificationConfig.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Task Classification Configuration
+ * Contains patterns, keywords, and scoring weights for task classification
+ */
+/**
+ * Regular expression patterns that indicate fast response tasks
+ */
+export declare const FAST_PATTERNS: RegExp[];
+/**
+ * Regular expression patterns that indicate reasoning tasks
+ */
+export declare const REASONING_PATTERNS: RegExp[];
+/**
+ * Keywords that indicate fast tasks regardless of context
+ */
+export declare const FAST_KEYWORDS: string[];
+/**
+ * Keywords that indicate reasoning tasks regardless of context
+ */
+export declare const REASONING_KEYWORDS: string[];
+/**
+ * Scoring weights for different classification factors
+ */
+export declare const SCORING_WEIGHTS: {
+    readonly SHORT_PROMPT_BONUS: 2;
+    readonly LONG_PROMPT_BONUS: 1;
+    readonly PATTERN_MATCH_SCORE: 3;
+    readonly KEYWORD_MATCH_SCORE: 1;
+    readonly MULTIPLE_QUESTIONS_BONUS: 1;
+    readonly MULTI_SENTENCE_BONUS: 1;
+    readonly TECHNICAL_DOMAIN_BONUS: 1;
+    readonly SIMPLE_DEFINITION_BONUS: 2;
+};
+/**
+ * Classification thresholds and constraints
+ */
+export declare const CLASSIFICATION_THRESHOLDS: {
+    readonly SHORT_PROMPT_LENGTH: 50;
+    readonly LONG_PROMPT_LENGTH: 200;
+    readonly SIMPLE_DEFINITION_LENGTH: 100;
+    readonly MIN_CONFIDENCE: 0.6;
+    readonly MAX_CONFIDENCE: 0.95;
+    readonly DEFAULT_CONFIDENCE: 0.5;
+};
+/**
+ * Domain-specific patterns for enhanced classification
+ */
+export declare const DOMAIN_PATTERNS: {
+    readonly TECHNICAL: RegExp;
+    readonly SIMPLE_DEFINITION: RegExp;
+};

package/dist/lib/config/taskClassificationConfig.js

@@ -0,0 +1,148 @@
+/**
+ * Task Classification Configuration
+ * Contains patterns, keywords, and scoring weights for task classification
+ */
+/**
+ * Regular expression patterns that indicate fast response tasks
+ */
+export const FAST_PATTERNS = [
+    // Greetings and social
+    /^(hi|hello|hey|good morning|good afternoon|good evening)/i,
+    /^(thanks?|thank you|thx)/i,
+    /^(yes|no|ok|okay|sure|fine)/i,
+    // Simple questions
+    /^what is\s+\w+\??$/i,
+    /^how are you/i,
+    /^tell me about\s+\w+$/i,
+    // Simple requests
+    /^(list|show|display)\s+/i,
+    /^give me\s+/i,
+    /^can you\s+(help|assist)/i,
+    // Simple definitions
+    /^define\s+/i,
+    /^meaning of\s+/i,
+    /^what does\s+\w+\s+mean/i,
+    // Quick facts
+    /^when (is|was|did)/i,
+    /^where (is|was)/i,
+    /^who (is|was)/i,
+    // Simple translations
+    /^translate\s+["'].*["']\s+to\s+\w+/i,
+    /^how do you say\s+/i,
+];
+/**
+ * Regular expression patterns that indicate reasoning tasks
+ */
+export const REASONING_PATTERNS = [
+    // Analysis and comparison
+    /\b(analyz|compar|evaluat|assess|examin)\w*/i,
+    /\b(pros and cons|advantages and disadvantages)/i,
+    /\b(better|worse|best|worst)\b.*\b(than|versus|vs)\b/i,
+    // Problem solving
+    /\b(solve|solution|problem|issue|challenge)\b/i,
+    /\b(how to|step by step|strategy|approach)\b/i,
+    /\b(optimize|improve|enhance|maximize|minimize)\b/i,
+    // Planning and design
+    /\b(plan|design|architect|structure|framework)\b/i,
+    /\b(implement|develop|build|create|construct)\b/i,
+    /\b(roadmap|timeline|schedule|phases)\b/i,
+    // Complex questions
+    /\b(why|explain|reason|cause|effect|impact)\b/i,
+    /\b(implications|consequences|considerations)\b/i,
+    /\b(should I|would you recommend|what if)\b/i,
+    // Research and investigation
+    /\b(research|investigate|explore|discover)\b/i,
+    /\b(evidence|proof|validate|verify)\b/i,
+    /\b(trends|patterns|insights|conclusions)\b/i,
+    // Business and strategy
+    /\b(business|strategy|market|competitive|financial)\b/i,
+    /\b(ROI|revenue|profit|investment|budget)\b/i,
+    /\b(stakeholder|customer|user experience|UX)\b/i,
+    // Technical complexity
+    /\b(algorithm|architecture|system|infrastructure)\b/i,
+    /\b(performance|scalability|security|reliability)\b/i,
+    /\b(integration|API|database|server)\b/i,
+];
+/**
+ * Keywords that indicate fast tasks regardless of context
+ */
+export const FAST_KEYWORDS = [
+    "quick",
+    "simple",
+    "brief",
+    "short",
+    "summary",
+    "overview",
+    "definition",
+    "meaning",
+    "list",
+    "show",
+    "display",
+    "name",
+    "tell",
+    "what",
+    "when",
+    "where",
+    "who",
+    "how many",
+    "count",
+];
+/**
+ * Keywords that indicate reasoning tasks regardless of context
+ */
+export const REASONING_KEYWORDS = [
+    "complex",
+    "detailed",
+    "comprehensive",
+    "thorough",
+    "in-depth",
+    "analyze",
+    "compare",
+    "evaluate",
+    "assess",
+    "research",
+    "investigate",
+    "strategy",
+    "plan",
+    "design",
+    "solve",
+    "optimize",
+    "recommend",
+    "explain",
+    "why",
+    "justify",
+    "pros",
+    "cons",
+    "trade-offs",
+];
+/**
+ * Scoring weights for different classification factors
+ */
+export const SCORING_WEIGHTS = {
+    SHORT_PROMPT_BONUS: 2,
+    LONG_PROMPT_BONUS: 1,
+    PATTERN_MATCH_SCORE: 3,
+    KEYWORD_MATCH_SCORE: 1,
+    MULTIPLE_QUESTIONS_BONUS: 1,
+    MULTI_SENTENCE_BONUS: 1,
+    TECHNICAL_DOMAIN_BONUS: 1,
+    SIMPLE_DEFINITION_BONUS: 2,
+};
+/**
+ * Classification thresholds and constraints
+ */
+export const CLASSIFICATION_THRESHOLDS = {
+    SHORT_PROMPT_LENGTH: 50,
+    LONG_PROMPT_LENGTH: 200,
+    SIMPLE_DEFINITION_LENGTH: 100,
+    MIN_CONFIDENCE: 0.6,
+    MAX_CONFIDENCE: 0.95,
+    DEFAULT_CONFIDENCE: 0.5,
+};
+/**
+ * Domain-specific patterns for enhanced classification
+ */
+export const DOMAIN_PATTERNS = {
+    TECHNICAL: /\b(code|programming|development|software)\b/i,
+    SIMPLE_DEFINITION: /\b(definition|meaning|what is)\b/i,
+};

package/dist/lib/core/baseProvider.js

@@ -1,3 +1,4 @@
+import { generateText } from "ai";
 import { MiddlewareFactory } from "../middleware/factory.js";
 import { logger } from "../utils/logger.js";
 import { DEFAULT_MAX_STEPS, STEP_LIMITS } from "../core/constants.js";
@@ -5,12 +6,11 @@ import { directAgentTools } from "../agent/directTools.js";
 import { getSafeMaxTokens } from "../utils/tokenLimits.js";
 import { createTimeoutController, TimeoutError } from "../utils/timeout.js";
 import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
-import { buildMessagesArray } from "../utils/messageBuilder.js";
+import { buildMessagesArray, buildMultimodalMessagesArray, } from "../utils/messageBuilder.js";
 import { getKeysAsString, getKeyCount } from "../utils/transformationUtils.js";
 import { validateStreamOptions as validateStreamOpts, validateTextGenerationOptions, ValidationError, createValidationSummary, } from "../utils/parameterValidation.js";
 import { recordProviderPerformanceFromMetrics, getPerformanceOptimizedProvider, } from "./evaluationProviders.js";
 import { modelConfig } from "./modelConfiguration.js";
-// Provider types moved to ../types/providers.js
 /**
  * Abstract base class for all AI providers
  * Tools are integrated as first-class citizens - always available by default
@@ -166,7 +166,7 @@ export class BaseProvider {
         try {
             // Import streamText dynamically to avoid circular dependencies
             // Using streamText instead of generateText for unified implementation
-            const { streamText } = await import("ai");
+            // const { streamText } = await import("ai");
             // Get ALL available tools (direct + MCP + external from options)
             const shouldUseTools = !options.disableTools && this.supportsTools();
             const baseTools = shouldUseTools ? await this.getAllTools() : {};
@@ -211,42 +211,86 @@ export class BaseProvider {
             });
             const model = await this.getAISDKModelWithMiddleware(options);
             // Build proper message array with conversation history
-            const messages = buildMessagesArray(options);
-            // Use streamText and accumulate results instead of generateText
-            const streamResult = await streamText({
+            // Check if this is a multimodal request (images or content present)
+            let messages;
+            // Type guard to check if options has multimodal input
+            const hasMultimodalInput = (opts) => {
+                const input = opts.input;
+                const hasImages = !!input?.images?.length;
+                const hasContent = !!input?.content?.length;
+                return hasImages || hasContent;
+            };
+            if (hasMultimodalInput(options)) {
+                if (process.env.NEUROLINK_DEBUG === "true") {
+                    logger.info("🖼️ [MULTIMODAL-REQUEST] Detected multimodal input, using multimodal message builder");
+                }
+                // This is a multimodal request - use multimodal message builder
+                // Convert TextGenerationOptions to GenerateOptions format for multimodal processing
+                const input = options.input;
+                const multimodalOptions = {
+                    input: {
+                        text: options.prompt || options.input?.text || "",
+                        images: input?.images,
+                        content: input?.content,
+                    },
+                    provider: options.provider,
+                    model: options.model,
+                    temperature: options.temperature,
+                    maxTokens: options.maxTokens,
+                    systemPrompt: options.systemPrompt,
+                    enableAnalytics: options.enableAnalytics,
+                    enableEvaluation: options.enableEvaluation,
+                    context: options.context,
+                };
+                messages = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
+            }
+            else {
+                if (process.env.NEUROLINK_DEBUG === "true") {
+                    logger.info("📝 [TEXT-ONLY-REQUEST] No multimodal input detected, using standard message builder");
+                }
+                // Standard text-only request
+                messages = buildMessagesArray(options);
+            }
+            // Convert messages to Vercel AI SDK format
+            const aiSDKMessages = messages.map((msg) => {
+                if (typeof msg.content === "string") {
+                    // Simple text content
+                    return {
+                        role: msg.role,
+                        content: msg.content,
+                    };
+                }
+                else {
+                    // Multimodal content array - convert to Vercel AI SDK format
+                    // The Vercel AI SDK expects content to be in a specific format
+                    return {
+                        role: msg.role,
+                        content: msg.content.map((item) => {
+                            if (item.type === "text") {
+                                return { type: "text", text: item.text || "" };
+                            }
+                            else if (item.type === "image") {
+                                return { type: "image", image: item.image || "" };
+                            }
+                            return item;
+                        }),
+                    };
+                }
+            });
+            const generateResult = await generateText({
                 model,
-                messages: messages,
+                messages: aiSDKMessages,
                 tools,
                 maxSteps: options.maxSteps || DEFAULT_MAX_STEPS,
                 toolChoice: shouldUseTools ? "auto" : "none",
                 temperature: options.temperature,
                 maxTokens: options.maxTokens, // No default limit - unlimited unless specified
             });
-            // Accumulate the streamed content
-            let accumulatedContent = "";
-            // Wait for the stream to complete and accumulate content
-            try {
-                for await (const chunk of streamResult.textStream) {
-                    accumulatedContent += chunk;
-                }
-            }
-            catch (streamError) {
-                logger.error(`Error reading text stream for ${this.providerName}:`, streamError);
-                throw streamError;
-            }
-            // Get the final result - this should include usage, toolCalls, etc.
-            const usage = await streamResult.usage;
-            const toolCalls = await streamResult.toolCalls;
-            const toolResults = await streamResult.toolResults;
             const responseTime = Date.now() - startTime;
-            // Create a result object compatible with generateText format
-            const result = {
-                text: accumulatedContent,
-                usage: usage,
-                toolCalls: toolCalls,
-                toolResults: toolResults,
-                steps: streamResult.steps, // Include steps for tool execution tracking
-            };
+            // Extract properties from generateResult
+            const usage = generateResult.usage;
+            const toolCalls = generateResult.toolCalls;
+            const toolResults = generateResult.toolResults;
             try {
                 const actualCost = await this.calculateActualCost(usage || { promptTokens: 0, completionTokens: 0, totalTokens: 0 });
                 recordProviderPerformanceFromMetrics(this.providerName, {
@@ -273,14 +317,14 @@ export class BaseProvider {
             // First check direct tool calls (fallback)
             if (toolCalls && toolCalls.length > 0) {
                 toolsUsed.push(...toolCalls.map((tc) => {
-                    return tc.toolName || "unknown";
+                    return tc.toolName || tc.name || "unknown";
                 }));
             }
             // Then check steps for tool calls (primary source for multi-step)
-            if (result.steps &&
-                Array.isArray(result.steps)) {
-                for (const step of result.steps ||
-                    []) {
+            if (generateResult.steps &&
+                Array.isArray(generateResult.steps)) {
+                for (const step of generateResult
+                    .steps || []) {
                     if (step?.toolCalls && Array.isArray(step.toolCalls)) {
                         toolsUsed.push(...step.toolCalls.map((tc) => {
                             return tc.toolName || tc.name || "unknown";
@@ -295,10 +339,10 @@ export class BaseProvider {
             // Create a map of tool calls to their arguments for matching with results
             const toolCallArgsMap = new Map();
             // Extract tool executions from AI SDK result steps
-            if (result.steps &&
-                Array.isArray(result.steps)) {
-                for (const step of result.steps ||
-                    []) {
+            if (generateResult.steps &&
+                Array.isArray(generateResult.steps)) {
+                for (const step of generateResult
+                    .steps || []) {
                     // First, collect tool calls and their arguments
                     if (step?.toolCalls && Array.isArray(step.toolCalls)) {
                         for (const toolCall of step.toolCalls) {
@@ -359,11 +403,11 @@ export class BaseProvider {
             }
             // Format the result with tool executions included
             const enhancedResult = {
-                content: result.text,
+                content: generateResult.text,
                 usage: {
-                    input: result.usage?.promptTokens || 0,
-                    output: result.usage?.completionTokens || 0,
-                    total: result.usage?.totalTokens || 0,
+                    input: generateResult.usage?.promptTokens || 0,
+                    output: generateResult.usage?.completionTokens || 0,
+                    total: generateResult.usage?.totalTokens || 0,
                 },
                 provider: this.providerName,
                 model: this.modelName,
@@ -943,13 +987,23 @@ export class BaseProvider {
         const providerName = optionsOrPrompt.provider || this.providerName;
         // Apply safe maxTokens based on provider and model
         const safeMaxTokens = getSafeMaxTokens(providerName, modelName, optionsOrPrompt.maxTokens);
-        return {
+        // CRITICAL FIX: Preserve the entire input object for multimodal support
+        // This ensures images and content arrays are not lost during normalization
+        const normalizedOptions = {
             ...optionsOrPrompt,
             prompt,
             provider: providerName,
             model: modelName,
             maxTokens: safeMaxTokens,
         };
+        // Ensure input object is preserved if it exists (for multimodal support)
+        if (optionsOrPrompt.input) {
+            normalizedOptions.input = {
+                ...optionsOrPrompt.input,
+                text: prompt, // Ensure text is consistent
+            };
+        }
+        return normalizedOptions;
     }
     normalizeStreamOptions(optionsOrPrompt) {
         if (typeof optionsOrPrompt === "string") {

package/dist/lib/core/types.d.ts

@@ -175,11 +175,14 @@ export interface StreamingOptions {
 }
 /**
  * Text generation options interface
+ * Extended to support multimodal content with zero breaking changes
  */
 export interface TextGenerationOptions {
     prompt?: string;
     input?: {
         text: string;
+        images?: Array<Buffer | string>;
+        content?: Array<import("../types/content.js").TextContent | import("../types/content.js").ImageContent>;
     };
     provider?: AIProviderName;
     model?: string;