@juspay/neurolink 7.35.0 → 7.37.0

package/dist/utils/messageBuilder.js

@@ -1,8 +1,13 @@
 /**
  * Message Builder Utility
  * Centralized logic for building message arrays from TextGenerationOptions
+ * Enhanced with multimodal support for images
  */
 import { CONVERSATION_INSTRUCTIONS } from "../config/conversationMemory.js";
+import { ProviderImageAdapter, MultimodalLogger, } from "../adapters/providerImageAdapter.js";
+import { logger } from "./logger.js";
+import { request } from "undici";
+import { readFileSync, existsSync } from "fs";
 /**
  * Build a properly formatted message array for AI providers
  * Combines system prompt, conversation history, and current user prompt
@@ -46,3 +51,277 @@ export function buildMessagesArray(options) {
     }
     return messages;
 }
+/**
+ * Build multimodal message array with image support
+ * Detects when images are present and routes through provider adapter
+ */
+export async function buildMultimodalMessagesArray(options, provider, model) {
+    // Check if this is a multimodal request
+    const hasImages = (options.input.images && options.input.images.length > 0) ||
+        (options.input.content &&
+            options.input.content.some((c) => c.type === "image"));
+    // If no images, use standard message building and convert to MultimodalChatMessage[]
+    if (!hasImages) {
+        const standardMessages = buildMessagesArray(options);
+        return standardMessages.map((msg) => ({ ...msg, content: msg.content }));
+    }
+    // Validate provider supports vision
+    if (!ProviderImageAdapter.supportsVision(provider, model)) {
+        throw new Error(`Provider ${provider} with model ${model} does not support vision processing. ` +
+            `Supported providers: ${ProviderImageAdapter.getVisionProviders().join(", ")}`);
+    }
+    const messages = [];
+    // Build enhanced system prompt
+    let systemPrompt = options.systemPrompt?.trim() || "";
+    // Add conversation-aware instructions when history exists
+    const hasConversationHistory = options.conversationHistory && options.conversationHistory.length > 0;
+    if (hasConversationHistory) {
+        systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
+    }
+    // Add system message if we have one
+    if (systemPrompt.trim()) {
+        messages.push({
+            role: "system",
+            content: systemPrompt.trim(),
+        });
+    }
+    // Add conversation history if available
+    if (hasConversationHistory && options.conversationHistory) {
+        // Convert conversation history to MultimodalChatMessage format
+        options.conversationHistory.forEach((msg) => {
+            messages.push({
+                role: msg.role,
+                content: msg.content,
+            });
+        });
+    }
+    // Handle multimodal content
+    try {
+        let userContent;
+        if (options.input.content && options.input.content.length > 0) {
+            // Advanced content format - convert to provider-specific format
+            userContent = await convertContentToProviderFormat(options.input.content, provider, model);
+        }
+        else if (options.input.images && options.input.images.length > 0) {
+            // Simple images format - convert to provider-specific format
+            userContent = await convertSimpleImagesToProviderFormat(options.input.text, options.input.images, provider, model);
+        }
+        else {
+            // Text-only fallback
+            userContent = options.input.text;
+        }
+        // 🔧 CRITICAL FIX: Handle multimodal content properly for Vercel AI SDK
+        if (typeof userContent === "string") {
+            // Simple text content - use standard MultimodalChatMessage format
+            messages.push({
+                role: "user",
+                content: userContent,
+            });
+        }
+        else {
+            // 🔧 MULTIMODAL CONTENT: Wrap the content array in a proper message object
+            // The Vercel AI SDK expects messages with multimodal content arrays
+            messages.push({
+                role: "user",
+                content: userContent,
+            });
+        }
+        return messages;
+    }
+    catch (error) {
+        MultimodalLogger.logError("MULTIMODAL_BUILD", error, {
+            provider,
+            model,
+            hasImages,
+            imageCount: options.input.images?.length || 0,
+        });
+        throw error;
+    }
+}
+/**
+ * Convert advanced content format to provider-specific format
+ */
+async function convertContentToProviderFormat(content, provider, _model) {
+    const textContent = content.find((c) => c.type === "text");
+    const imageContent = content.filter((c) => c.type === "image");
+    if (!textContent) {
+        throw new Error("Multimodal content must include at least one text element");
+    }
+    if (imageContent.length === 0) {
+        return textContent.text;
+    }
+    // Extract images as Buffer | string array
+    const images = imageContent.map((img) => img.data);
+    return await convertSimpleImagesToProviderFormat(textContent.text, images, provider, _model);
+}
+/**
+ * Check if a string is an internet URL
+ */
+function isInternetUrl(input) {
+    return input.startsWith("http://") || input.startsWith("https://");
+}
+/**
+ * Download image from URL and convert to base64 data URI
+ */
+async function downloadImageFromUrl(url) {
+    try {
+        const response = await request(url, {
+            method: "GET",
+            headersTimeout: 10000, // 10 second timeout for headers
+            bodyTimeout: 30000, // 30 second timeout for body
+            maxRedirections: 5,
+        });
+        if (response.statusCode !== 200) {
+            throw new Error(`HTTP ${response.statusCode}: Failed to download image from ${url}`);
+        }
+        // Get content type from headers
+        const contentType = response.headers["content-type"] || "image/jpeg";
+        // Validate it's an image
+        if (!contentType.startsWith("image/")) {
+            throw new Error(`URL does not point to an image. Content-Type: ${contentType}`);
+        }
+        // Read the response body
+        const chunks = [];
+        for await (const chunk of response.body) {
+            chunks.push(chunk);
+        }
+        const buffer = Buffer.concat(chunks);
+        // Check file size (limit to 10MB)
+        const maxSize = 10 * 1024 * 1024; // 10MB
+        if (buffer.length > maxSize) {
+            throw new Error(`Image too large: ${buffer.length} bytes (max: ${maxSize} bytes)`);
+        }
+        // Convert to base64 data URI
+        const base64 = buffer.toString("base64");
+        const dataUri = `data:${contentType};base64,${base64}`;
+        return dataUri;
+    }
+    catch (error) {
+        MultimodalLogger.logError("URL_DOWNLOAD_FAILED", error, { url });
+        throw new Error(`Failed to download image from ${url}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
+/**
+ * Convert simple images format to Vercel AI SDK format with smart auto-detection
+ * - URLs: Downloaded and converted to base64 for Vercel AI SDK compatibility
+ * - Local files: Converted to base64 for Vercel AI SDK compatibility
+ * - Buffers/Data URIs: Processed normally
+ */
+async function convertSimpleImagesToProviderFormat(text, images, provider, _model) {
+    // For Vercel AI SDK, we need to return the content in the standard format
+    // The Vercel AI SDK will handle provider-specific formatting internally
+    // Smart auto-detection: separate URLs from actual image data
+    const urlImages = [];
+    const actualImages = [];
+    images.forEach((image, _index) => {
+        if (typeof image === "string" && isInternetUrl(image)) {
+            // Internet URL - will be downloaded and converted to base64
+            urlImages.push(image);
+        }
+        else {
+            // Actual image data (file path, Buffer, data URI) - process for Vercel AI SDK
+            actualImages.push(image);
+        }
+    });
+    // Download URL images and add to actual images
+    for (const url of urlImages) {
+        try {
+            const downloadedDataUri = await downloadImageFromUrl(url);
+            actualImages.push(downloadedDataUri);
+        }
+        catch (error) {
+            MultimodalLogger.logError("URL_DOWNLOAD_FAILED_SKIPPING", error, { url });
+            // Continue processing other images even if one URL fails
+            logger.warn(`Failed to download image from ${url}, skipping: ${error instanceof Error ? error.message : String(error)}`);
+        }
+    }
+    const content = [{ type: "text", text }];
+    // Process all images (including downloaded URLs) for Vercel AI SDK
+    actualImages.forEach((image, index) => {
+        try {
+            // Vercel AI SDK expects { type: 'image', image: Buffer | string, mimeType?: string }
+            // For Vertex AI, we need to include mimeType
+            let imageData;
+            let mimeType = "image/jpeg"; // Default mime type
+            if (typeof image === "string") {
+                if (image.startsWith("data:")) {
+                    // Data URI (including downloaded URLs) - extract mime type and use directly
+                    const match = image.match(/^data:([^;]+);base64,(.+)$/);
+                    if (match) {
+                        mimeType = match[1];
+                        imageData = image; // Keep as data URI for Vercel AI SDK
+                    }
+                    else {
+                        imageData = image;
+                    }
+                }
+                else if (isInternetUrl(image)) {
+                    // This should not happen as URLs are processed separately above
+                    // But handle it gracefully just in case
+                    throw new Error(`Unprocessed URL found in actualImages: ${image}`);
+                }
+                else {
+                    // File path string - convert to base64 data URI
+                    try {
+                        if (existsSync(image)) {
+                            const buffer = readFileSync(image);
+                            const base64 = buffer.toString("base64");
+                            // Detect mime type from file extension
+                            const ext = image.toLowerCase().split(".").pop();
+                            switch (ext) {
+                                case "png":
+                                    mimeType = "image/png";
+                                    break;
+                                case "gif":
+                                    mimeType = "image/gif";
+                                    break;
+                                case "webp":
+                                    mimeType = "image/webp";
+                                    break;
+                                case "bmp":
+                                    mimeType = "image/bmp";
+                                    break;
+                                case "tiff":
+                                case "tif":
+                                    mimeType = "image/tiff";
+                                    break;
+                                default:
+                                    mimeType = "image/jpeg";
+                                    break;
+                            }
+                            imageData = `data:${mimeType};base64,${base64}`;
+                        }
+                        else {
+                            throw new Error(`Image file not found: ${image}`);
+                        }
+                    }
+                    catch (error) {
+                        MultimodalLogger.logError("FILE_PATH_CONVERSION", error, {
+                            index,
+                            filePath: image,
+                        });
+                        throw new Error(`Failed to convert file path to base64: ${image}. ${error}`);
+                    }
+                }
+            }
+            else {
+                // Buffer - convert to base64 data URI
+                const base64 = image.toString("base64");
+                imageData = `data:${mimeType};base64,${base64}`;
+            }
+            content.push({
+                type: "image",
+                image: imageData,
+                mimeType: mimeType, // Add mimeType for Vertex AI compatibility
+            });
+        }
+        catch (error) {
+            MultimodalLogger.logError("ADD_IMAGE_TO_CONTENT", error, {
+                index,
+                provider,
+            });
+            throw error;
+        }
+    });
+    return content;
+}

package/dist/utils/modelRouter.d.ts

@@ -0,0 +1,107 @@
+/**
+ * Model Router for NeuroLink Orchestration
+ * Routes tasks to optimal models based on classification and requirements
+ */
+import type { TaskType } from "../types/taskClassificationTypes.js";
+export interface ModelRoute {
+    provider: string;
+    model: string;
+    reasoning: string;
+    confidence: number;
+}
+export interface ModelRoutingOptions {
+    /** Override the task classification */
+    forceTaskType?: TaskType;
+    /** Require specific performance characteristics */
+    requireFast?: boolean;
+    /** Require specific capability (reasoning, creativity, etc.) */
+    requireCapability?: string;
+    /** Fallback strategy if primary choice fails */
+    fallbackStrategy?: "fast" | "reasoning" | "auto";
+}
+/**
+ * Model configurations for different task types and providers
+ */
+declare const MODEL_CONFIGS: {
+    readonly fast: {
+        readonly primary: {
+            readonly provider: "vertex";
+            readonly model: "gemini-2.5-flash";
+            readonly capabilities: readonly ["speed", "general", "code", "basic-reasoning"];
+            readonly avgResponseTime: 800;
+            readonly costPerToken: 0.0001;
+            readonly reasoning: "Optimized for speed and efficiency via Vertex AI";
+        };
+        readonly fallback: {
+            readonly provider: "vertex";
+            readonly model: "gemini-2.5-pro";
+            readonly capabilities: readonly ["speed", "general", "basic-reasoning"];
+            readonly avgResponseTime: 1200;
+            readonly costPerToken: 0.0002;
+            readonly reasoning: "Vertex AI Gemini Pro fallback";
+        };
+    };
+    readonly reasoning: {
+        readonly primary: {
+            readonly provider: "vertex";
+            readonly model: "claude-sonnet-4@20250514";
+            readonly capabilities: readonly ["reasoning", "analysis", "complex-logic", "code", "creativity"];
+            readonly avgResponseTime: 3000;
+            readonly costPerToken: 0.003;
+            readonly reasoning: "Advanced reasoning and analysis via Claude Sonnet 4 on Vertex AI";
+        };
+        readonly fallback: {
+            readonly provider: "vertex";
+            readonly model: "claude-opus-4@20250514";
+            readonly capabilities: readonly ["reasoning", "analysis", "complex-logic", "code", "creativity", "agentic"];
+            readonly avgResponseTime: 4000;
+            readonly costPerToken: 0.005;
+            readonly reasoning: "Claude Opus 4 fallback on Vertex AI for most complex tasks";
+        };
+    };
+};
+/**
+ * Model Router
+ * Intelligently routes tasks to optimal models based on classification
+ */
+export declare class ModelRouter {
+    /**
+     * Route a prompt to the optimal model configuration
+     */
+    static route(prompt: string, options?: ModelRoutingOptions): ModelRoute;
+    /**
+     * Get fallback route if primary route fails
+     */
+    static getFallbackRoute(prompt: string, primaryRoute: ModelRoute, options?: ModelRoutingOptions): ModelRoute;
+    /**
+     * Determine task type from a model route
+     */
+    private static getTaskTypeFromRoute;
+    /**
+     * Get all available model configurations
+     */
+    static getAvailableModels(): typeof MODEL_CONFIGS;
+    /**
+     * Validate model availability for a given route
+     */
+    static validateRoute(route: ModelRoute): Promise<boolean>;
+    /**
+     * Get routing statistics for multiple prompts
+     */
+    static getRoutingStats(prompts: string[]): {
+        total: number;
+        fastRoutes: number;
+        reasoningRoutes: number;
+        averageConfidence: number;
+        providerDistribution: Record<string, number>;
+    };
+    /**
+     * Estimate cost and performance for a route
+     */
+    static getRouteEstimates(route: ModelRoute, estimatedTokens?: number): {
+        estimatedCost: number;
+        estimatedResponseTime: number;
+        capabilities: string[];
+    };
+}
+export {};

package/dist/utils/modelRouter.js

@@ -0,0 +1,292 @@
+/**
+ * Model Router for NeuroLink Orchestration
+ * Routes tasks to optimal models based on classification and requirements
+ */
+import { logger } from "./logger.js";
+import { BinaryTaskClassifier } from "./taskClassifier.js";
+import { redactForRouting } from "./promptRedaction.js";
+/**
+ * Routing configuration constants
+ */
+const ROUTING_CONFIG = {
+    minRouteConfidence: 0.3,
+    maxRouteConfidence: 0.95,
+    confidenceBoost: 0.1,
+};
+/**
+ * Model configurations for different task types and providers
+ */
+const MODEL_CONFIGS = {
+    fast: {
+        primary: {
+            provider: "vertex",
+            model: "gemini-2.5-flash",
+            capabilities: ["speed", "general", "code", "basic-reasoning"],
+            avgResponseTime: 800, // ms
+            costPerToken: 0.0001,
+            reasoning: "Optimized for speed and efficiency via Vertex AI",
+        },
+        fallback: {
+            provider: "vertex",
+            model: "gemini-2.5-pro",
+            capabilities: ["speed", "general", "basic-reasoning"],
+            avgResponseTime: 1200,
+            costPerToken: 0.0002,
+            reasoning: "Vertex AI Gemini Pro fallback",
+        },
+    },
+    reasoning: {
+        primary: {
+            provider: "vertex",
+            model: "claude-sonnet-4@20250514",
+            capabilities: [
+                "reasoning",
+                "analysis",
+                "complex-logic",
+                "code",
+                "creativity",
+            ],
+            avgResponseTime: 3000, // ms
+            costPerToken: 0.003,
+            reasoning: "Advanced reasoning and analysis via Claude Sonnet 4 on Vertex AI",
+        },
+        fallback: {
+            provider: "vertex",
+            model: "claude-opus-4@20250514",
+            capabilities: [
+                "reasoning",
+                "analysis",
+                "complex-logic",
+                "code",
+                "creativity",
+                "agentic",
+            ],
+            avgResponseTime: 4000,
+            costPerToken: 0.005,
+            reasoning: "Claude Opus 4 fallback on Vertex AI for most complex tasks",
+        },
+    },
+};
+/**
+ * Model Router
+ * Intelligently routes tasks to optimal models based on classification
+ */
+export class ModelRouter {
+    /**
+     * Route a prompt to the optimal model configuration
+     */
+    static route(prompt, options = {}) {
+        const startTime = Date.now();
+        // 1. Classify the task if not overridden
+        let classification;
+        if (options.forceTaskType) {
+            classification = {
+                type: options.forceTaskType,
+                confidence: ROUTING_CONFIG.maxRouteConfidence, // Use maxRouteConfidence instead of 1.0
+                reasoning: "forced task type",
+            };
+        }
+        else {
+            classification = BinaryTaskClassifier.classify(prompt);
+        }
+        // 2. Apply special requirements
+        let taskType = classification.type;
+        const reasons = [classification.reasoning];
+        if (options.requireFast) {
+            taskType = "fast";
+            reasons.push("speed required");
+        }
+        if (options.requireCapability) {
+            // Check if the capability suggests a specific task type
+            const capability = options.requireCapability.toLowerCase();
+            if (["analysis", "reasoning", "complex", "research"].some((c) => capability.includes(c))) {
+                taskType = "reasoning";
+                reasons.push(`capability: ${capability}`);
+            }
+            else if (["speed", "quick", "fast", "simple"].some((c) => capability.includes(c))) {
+                taskType = "fast";
+                reasons.push(`capability: ${capability}`);
+            }
+        }
+        // 3. Select model configuration
+        const config = MODEL_CONFIGS[taskType];
+        const selectedConfig = config.primary;
+        // 4. Calculate confidence based on multiple factors
+        let confidence = classification.confidence;
+        // Adjust confidence based on prompt characteristics
+        if (taskType === "fast" && prompt.length < 30) {
+            confidence = Math.min(ROUTING_CONFIG.maxRouteConfidence, confidence + ROUTING_CONFIG.confidenceBoost);
+            reasons.push("very short prompt");
+        }
+        if (taskType === "reasoning" && prompt.length > 150) {
+            confidence = Math.min(ROUTING_CONFIG.maxRouteConfidence, confidence + ROUTING_CONFIG.confidenceBoost);
+            reasons.push("detailed prompt");
+        }
+        // Ensure final confidence is within configured bounds
+        confidence = Math.max(ROUTING_CONFIG.minRouteConfidence, Math.min(ROUTING_CONFIG.maxRouteConfidence, confidence));
+        // 5. Create route result
+        const route = {
+            provider: selectedConfig.provider,
+            model: selectedConfig.model,
+            reasoning: reasons.join(", "),
+            confidence,
+        };
+        const routingTime = Date.now() - startTime;
+        logger.debug("Model routing decision", {
+            prompt: redactForRouting(prompt),
+            taskType,
+            route: {
+                provider: route.provider,
+                model: route.model,
+                confidence: route.confidence.toFixed(2),
+            },
+            reasoning: route.reasoning,
+            routingTime: `${routingTime}ms`,
+            options: Object.keys(options).length > 0 ? options : undefined,
+        });
+        return route;
+    }
+    /**
+     * Get fallback route if primary route fails
+     */
+    static getFallbackRoute(prompt, primaryRoute, options = {}) {
+        // Determine fallback strategy
+        let fallbackType;
+        if (options.fallbackStrategy) {
+            if (options.fallbackStrategy === "auto") {
+                // Use opposite of primary for fallback
+                const primaryType = this.getTaskTypeFromRoute(primaryRoute);
+                fallbackType = primaryType === "fast" ? "reasoning" : "fast";
+            }
+            else {
+                fallbackType = options.fallbackStrategy;
+            }
+        }
+        else {
+            // Default: use fallback model of same type
+            fallbackType = this.getTaskTypeFromRoute(primaryRoute);
+        }
+        const config = MODEL_CONFIGS[fallbackType];
+        const fallbackConfig = config.fallback;
+        const route = {
+            provider: fallbackConfig.provider,
+            model: fallbackConfig.model,
+            reasoning: `fallback from ${primaryRoute.provider}/${primaryRoute.model}`,
+            confidence: Math.max(ROUTING_CONFIG.minRouteConfidence, Math.min(ROUTING_CONFIG.maxRouteConfidence, primaryRoute.confidence - 0.2)),
+        };
+        logger.debug("Fallback route selected", {
+            originalRoute: `${primaryRoute.provider}/${primaryRoute.model}`,
+            fallbackRoute: `${route.provider}/${route.model}`,
+            fallbackType,
+            strategy: options.fallbackStrategy || "default",
+        });
+        return route;
+    }
+    /**
+     * Determine task type from a model route
+     */
+    static getTaskTypeFromRoute(route) {
+        // Check which config matches this route
+        for (const [taskType, config] of Object.entries(MODEL_CONFIGS)) {
+            if (config.primary.provider === route.provider &&
+                config.primary.model === route.model) {
+                return taskType;
+            }
+            if (config.fallback.provider === route.provider &&
+                config.fallback.model === route.model) {
+                return taskType;
+            }
+        }
+        // Default fallback based on model name patterns
+        if (route.model.includes("flash") || route.model.includes("mini")) {
+            return "fast";
+        }
+        return "reasoning";
+    }
+    /**
+     * Get all available model configurations
+     */
+    static getAvailableModels() {
+        return MODEL_CONFIGS;
+    }
+    /**
+     * Validate model availability for a given route
+     */
+    static async validateRoute(route) {
+        try {
+            // This would typically check provider availability
+            // For now, just validate the configuration exists
+            const configs = Object.values(MODEL_CONFIGS).flatMap((config) => [
+                config.primary,
+                config.fallback,
+            ]);
+            return configs.some((config) => config.provider === route.provider && config.model === route.model);
+        }
+        catch (error) {
+            logger.error("Route validation failed", {
+                route,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            return false;
+        }
+    }
+    /**
+     * Get routing statistics for multiple prompts
+     */
+    static getRoutingStats(prompts) {
+        const routes = prompts.map((prompt) => this.route(prompt));
+        // Handle empty prompts array to avoid divide-by-zero
+        if (routes.length === 0) {
+            const stats = {
+                total: 0,
+                fastRoutes: 0,
+                reasoningRoutes: 0,
+                averageConfidence: 0,
+                providerDistribution: {},
+            };
+            logger.debug("Routing statistics", stats);
+            return stats;
+        }
+        const stats = {
+            total: routes.length,
+            fastRoutes: routes.filter((r) => {
+                const taskType = this.getTaskTypeFromRoute(r);
+                return taskType === "fast";
+            }).length,
+            reasoningRoutes: routes.filter((r) => {
+                const taskType = this.getTaskTypeFromRoute(r);
+                return taskType === "reasoning";
+            }).length,
+            averageConfidence: routes.reduce((sum, r) => sum + r.confidence, 0) / routes.length,
+            providerDistribution: routes.reduce((dist, r) => {
+                dist[r.provider] = (dist[r.provider] || 0) + 1;
+                return dist;
+            }, {}),
+        };
+        logger.debug("Routing statistics", stats);
+        return stats;
+    }
+    /**
+     * Estimate cost and performance for a route
+     */
+    static getRouteEstimates(route, estimatedTokens = 500) {
+        // Find the config for this route
+        const allConfigs = Object.values(MODEL_CONFIGS).flatMap((config) => [
+            config.primary,
+            config.fallback,
+        ]);
+        const config = allConfigs.find((c) => c.provider === route.provider && c.model === route.model);
+        if (!config) {
+            return {
+                estimatedCost: 0,
+                estimatedResponseTime: 2000,
+                capabilities: [],
+            };
+        }
+        return {
+            estimatedCost: config.costPerToken * estimatedTokens,
+            estimatedResponseTime: config.avgResponseTime,
+            capabilities: [...config.capabilities],
+        };
+    }
+}