@juspay/neurolink 7.35.0 → 7.37.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [7.37.0](https://github.com/juspay/neurolink/compare/v7.36.0...v7.37.0) (2025-09-10)
+
+### Features
+
+- **(sdk):** Add advanced orchestration of model and providers BZ-43839 ([840d697](https://github.com/juspay/neurolink/commit/840d697aa6ef3e5e4c511a9482fc7e80006d2534))
+
+## [7.36.0](https://github.com/juspay/neurolink/compare/v7.35.0...v7.36.0) (2025-09-10)
+
+### Features
+
+- **(image):** added support for multimodality(image) in cli and sdk ([678b61b](https://github.com/juspay/neurolink/commit/678b61bfef3d0622029d40b8ab06dca9836bcb6c))
+
 ## [7.35.0](https://github.com/juspay/neurolink/compare/v7.34.0...v7.35.0) (2025-09-09)
 
 ### Features

package/dist/adapters/providerImageAdapter.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Provider Image Adapter - Smart routing for multimodal content
+ * Handles provider-specific image formatting and vision capability validation
+ */
+import type { Content } from "../types/content.js";
+/**
+ * Simplified logger for essential error reporting only
+ */
+export declare class MultimodalLogger {
+    static logError(step: string, error: Error, context: unknown): void;
+}
+/**
+ * Provider Image Adapter - Smart routing and formatting
+ */
+export declare class ProviderImageAdapter {
+    /**
+     * Main adapter method - routes to provider-specific formatting
+     */
+    static adaptForProvider(text: string, images: Array<Buffer | string>, provider: string, model: string): Promise<unknown>;
+    /**
+     * Format content for OpenAI (GPT-4o format)
+     */
+    private static formatForOpenAI;
+    /**
+     * Format content for Google AI (Gemini format)
+     */
+    private static formatForGoogleAI;
+    /**
+     * Format content for Anthropic (Claude format)
+     */
+    private static formatForAnthropic;
+    /**
+     * Format content for Vertex AI (model-specific routing)
+     */
+    private static formatForVertex;
+    /**
+     * Validate that provider and model support vision
+     */
+    private static validateVisionSupport;
+    /**
+     * Convert simple images array to advanced content format
+     */
+    static convertToContent(text: string, images?: Array<Buffer | string>): Content[];
+    /**
+     * Check if provider supports multimodal content
+     */
+    static supportsVision(provider: string, model?: string): boolean;
+    /**
+     * Get supported models for a provider
+     */
+    static getSupportedModels(provider: string): string[];
+    /**
+     * Get all vision-capable providers
+     */
+    static getVisionProviders(): string[];
+}

package/dist/adapters/providerImageAdapter.js

@@ -0,0 +1,257 @@
+/**
+ * Provider Image Adapter - Smart routing for multimodal content
+ * Handles provider-specific image formatting and vision capability validation
+ */
+import { logger } from "../utils/logger.js";
+import { ImageProcessor } from "../utils/imageProcessor.js";
+/**
+ * Simplified logger for essential error reporting only
+ */
+export class MultimodalLogger {
+    static logError(step, error, context) {
+        logger.error(`Multimodal ${step} failed: ${error.message}`);
+        if (process.env.NODE_ENV === "development") {
+            logger.error("Context:", JSON.stringify(context, null, 2));
+            logger.error("Stack:", error.stack);
+        }
+    }
+}
+/**
+ * Vision capability definitions for each provider
+ */
+const VISION_CAPABILITIES = {
+    openai: ["gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-4-vision-preview"],
+    "google-ai": [
+        "gemini-2.5-pro",
+        "gemini-2.5-flash",
+        "gemini-1.5-pro",
+        "gemini-1.5-flash",
+        "gemini-pro-vision",
+    ],
+    anthropic: [
+        "claude-3-5-sonnet",
+        "claude-3-opus",
+        "claude-3-sonnet",
+        "claude-3-haiku",
+    ],
+    vertex: [
+        // Gemini models on Vertex AI
+        "gemini-2.5-pro",
+        "gemini-2.5-flash",
+        "gemini-1.5-pro",
+        "gemini-1.5-flash",
+        // Claude models on Vertex AI (with actual Vertex naming patterns)
+        "claude-3-5-sonnet",
+        "claude-3-opus",
+        "claude-3-sonnet",
+        "claude-3-haiku",
+        "claude-sonnet-3",
+        "claude-sonnet-4",
+        "claude-opus-3",
+        "claude-haiku-3",
+        // Additional Vertex AI Claude model patterns
+        "claude-3.5-sonnet",
+        "claude-3.5-haiku",
+        "claude-3.0-sonnet",
+        "claude-3.0-opus",
+        // Versioned model names (e.g., claude-sonnet-4@20250514)
+        "claude-sonnet-4@",
+        "claude-opus-3@",
+        "claude-haiku-3@",
+        "claude-3-5-sonnet@",
+    ],
+};
+/**
+ * Provider Image Adapter - Smart routing and formatting
+ */
+export class ProviderImageAdapter {
+    /**
+     * Main adapter method - routes to provider-specific formatting
+     */
+    static async adaptForProvider(text, images, provider, model) {
+        try {
+            // Validate provider supports vision
+            this.validateVisionSupport(provider, model);
+            let adaptedPayload;
+            // Process images based on provider requirements
+            switch (provider.toLowerCase()) {
+                case "openai":
+                    adaptedPayload = this.formatForOpenAI(text, images);
+                    break;
+                case "google-ai":
+                case "google":
+                    adaptedPayload = this.formatForGoogleAI(text, images);
+                    break;
+                case "anthropic":
+                    adaptedPayload = this.formatForAnthropic(text, images);
+                    break;
+                case "vertex":
+                    adaptedPayload = this.formatForVertex(text, images, model);
+                    break;
+                default:
+                    throw new Error(`Vision not supported for provider: ${provider}`);
+            }
+            return adaptedPayload;
+        }
+        catch (error) {
+            MultimodalLogger.logError("ADAPTATION", error, {
+                provider,
+                model,
+                imageCount: images.length,
+            });
+            throw error;
+        }
+    }
+    /**
+     * Format content for OpenAI (GPT-4o format)
+     */
+    static formatForOpenAI(text, images) {
+        const content = [{ type: "text", text }];
+        images.forEach((image, index) => {
+            try {
+                const imageUrl = ImageProcessor.processImageForOpenAI(image);
+                content.push({
+                    type: "image_url",
+                    image_url: { url: imageUrl },
+                });
+            }
+            catch (error) {
+                MultimodalLogger.logError("PROCESS_IMAGE", error, {
+                    index,
+                    provider: "openai",
+                });
+                throw error;
+            }
+        });
+        return { messages: [{ role: "user", content }] };
+    }
+    /**
+     * Format content for Google AI (Gemini format)
+     */
+    static formatForGoogleAI(text, images) {
+        const parts = [{ text }];
+        images.forEach((image, index) => {
+            try {
+                const { mimeType, data } = ImageProcessor.processImageForGoogle(image);
+                parts.push({
+                    inlineData: { mimeType, data },
+                });
+            }
+            catch (error) {
+                MultimodalLogger.logError("PROCESS_IMAGE", error, {
+                    index,
+                    provider: "google-ai",
+                });
+                throw error;
+            }
+        });
+        return { contents: [{ parts }] };
+    }
+    /**
+     * Format content for Anthropic (Claude format)
+     */
+    static formatForAnthropic(text, images) {
+        const content = [{ type: "text", text }];
+        images.forEach((image, index) => {
+            try {
+                const { mediaType, data } = ImageProcessor.processImageForAnthropic(image);
+                content.push({
+                    type: "image",
+                    source: {
+                        type: "base64",
+                        media_type: mediaType,
+                        data,
+                    },
+                });
+            }
+            catch (error) {
+                MultimodalLogger.logError("PROCESS_IMAGE", error, {
+                    index,
+                    provider: "anthropic",
+                });
+                throw error;
+            }
+        });
+        return { messages: [{ role: "user", content }] };
+    }
+    /**
+     * Format content for Vertex AI (model-specific routing)
+     */
+    static formatForVertex(text, images, model) {
+        // Route based on model type
+        if (model.includes("gemini")) {
+            return this.formatForGoogleAI(text, images);
+        }
+        else if (model.includes("claude")) {
+            return this.formatForAnthropic(text, images);
+        }
+        else {
+            return this.formatForGoogleAI(text, images);
+        }
+    }
+    /**
+     * Validate that provider and model support vision
+     */
+    static validateVisionSupport(provider, model) {
+        const normalizedProvider = provider.toLowerCase();
+        const supportedModels = VISION_CAPABILITIES[normalizedProvider];
+        if (!supportedModels) {
+            throw new Error(`Provider ${provider} does not support vision processing. ` +
+                `Supported providers: ${Object.keys(VISION_CAPABILITIES).join(", ")}`);
+        }
+        const isSupported = supportedModels.some((supportedModel) => model.toLowerCase().includes(supportedModel.toLowerCase()));
+        if (!isSupported) {
+            throw new Error(`Provider ${provider} with model ${model} does not support vision processing. ` +
+                `Supported models for ${provider}: ${supportedModels.join(", ")}`);
+        }
+    }
+    /**
+     * Convert simple images array to advanced content format
+     */
+    static convertToContent(text, images) {
+        const content = [{ type: "text", text }];
+        if (images && images.length > 0) {
+            images.forEach((image) => {
+                content.push({
+                    type: "image",
+                    data: image,
+                    mediaType: ImageProcessor.detectImageType(image),
+                });
+            });
+        }
+        return content;
+    }
+    /**
+     * Check if provider supports multimodal content
+     */
+    static supportsVision(provider, model) {
+        try {
+            const normalizedProvider = provider.toLowerCase();
+            const supportedModels = VISION_CAPABILITIES[normalizedProvider];
+            if (!supportedModels) {
+                return false;
+            }
+            if (!model) {
+                return true; // Provider supports vision, but need to check specific model
+            }
+            return supportedModels.some((supportedModel) => model.toLowerCase().includes(supportedModel.toLowerCase()));
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Get supported models for a provider
+     */
+    static getSupportedModels(provider) {
+        const normalizedProvider = provider.toLowerCase();
+        const models = VISION_CAPABILITIES[normalizedProvider];
+        return models ? [...models] : [];
+    }
+    /**
+     * Get all vision-capable providers
+     */
+    static getVisionProviders() {
+        return Object.keys(VISION_CAPABILITIES);
+    }
+}

package/dist/cli/commands/config.d.ts

@@ -18,9 +18,9 @@ declare const ConfigSchema: z.ZodObject<{
             apiKey?: string | undefined;
             baseURL?: string | undefined;
         }, {
+            model?: string | undefined;
             apiKey?: string | undefined;
             baseURL?: string | undefined;
-            model?: string | undefined;
         }>>;
         bedrock: z.ZodOptional<z.ZodObject<{
             region: z.ZodOptional<z.ZodString>;
@@ -73,8 +73,8 @@ declare const ConfigSchema: z.ZodObject<{
             model: string;
             apiKey?: string | undefined;
         }, {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         }>>;
         azure: z.ZodOptional<z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
@@ -87,8 +87,8 @@ declare const ConfigSchema: z.ZodObject<{
             endpoint?: string | undefined;
             deploymentId?: string | undefined;
         }, {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
             endpoint?: string | undefined;
             deploymentId?: string | undefined;
         }>>;
@@ -99,8 +99,8 @@ declare const ConfigSchema: z.ZodObject<{
             model: string;
             apiKey?: string | undefined;
         }, {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         }>>;
         huggingface: z.ZodOptional<z.ZodObject<{
             apiKey: z.ZodOptional<z.ZodString>;
@@ -109,8 +109,8 @@ declare const ConfigSchema: z.ZodObject<{
             model: string;
             apiKey?: string | undefined;
         }, {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         }>>;
         ollama: z.ZodOptional<z.ZodObject<{
             baseUrl: z.ZodDefault<z.ZodString>;
@@ -132,8 +132,8 @@ declare const ConfigSchema: z.ZodObject<{
             model: string;
             apiKey?: string | undefined;
         }, {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         }>>;
     }, "strip", z.ZodTypeAny, {
         openai?: {
@@ -186,13 +186,13 @@ declare const ConfigSchema: z.ZodObject<{
         } | undefined;
     }, {
         openai?: {
+            model?: string | undefined;
             apiKey?: string | undefined;
             baseURL?: string | undefined;
-            model?: string | undefined;
         } | undefined;
         anthropic?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
         vertex?: {
             model?: string | undefined;
@@ -204,8 +204,8 @@ declare const ConfigSchema: z.ZodObject<{
             privateKey?: string | undefined;
         } | undefined;
         "google-ai"?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
         bedrock?: {
             model?: string | undefined;
@@ -215,14 +215,14 @@ declare const ConfigSchema: z.ZodObject<{
             sessionToken?: string | undefined;
         } | undefined;
         azure?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
             endpoint?: string | undefined;
             deploymentId?: string | undefined;
         } | undefined;
         huggingface?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
         ollama?: {
             timeout?: number | undefined;
@@ -230,8 +230,8 @@ declare const ConfigSchema: z.ZodObject<{
             baseUrl?: string | undefined;
         } | undefined;
         mistral?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
     }>>;
     profiles: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodAny>>;
@@ -505,7 +505,7 @@ declare const ConfigSchema: z.ZodObject<{
             apiKey?: string | undefined;
         } | undefined;
     };
-    defaultProvider: "openai" | "anthropic" | "vertex" | "google-ai" | "bedrock" | "azure" | "huggingface" | "ollama" | "mistral" | "auto";
+    defaultProvider: "openai" | "anthropic" | "vertex" | "google-ai" | "auto" | "bedrock" | "azure" | "huggingface" | "ollama" | "mistral";
     profiles: Record<string, any>;
     preferences: {
         temperature: number;
@@ -555,13 +555,13 @@ declare const ConfigSchema: z.ZodObject<{
 }, {
     providers?: {
         openai?: {
+            model?: string | undefined;
             apiKey?: string | undefined;
             baseURL?: string | undefined;
-            model?: string | undefined;
         } | undefined;
         anthropic?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
         vertex?: {
             model?: string | undefined;
@@ -573,8 +573,8 @@ declare const ConfigSchema: z.ZodObject<{
             privateKey?: string | undefined;
         } | undefined;
         "google-ai"?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
         bedrock?: {
             model?: string | undefined;
@@ -584,14 +584,14 @@ declare const ConfigSchema: z.ZodObject<{
             sessionToken?: string | undefined;
         } | undefined;
         azure?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
             endpoint?: string | undefined;
             deploymentId?: string | undefined;
         } | undefined;
         huggingface?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
         ollama?: {
             timeout?: number | undefined;
@@ -599,11 +599,11 @@ declare const ConfigSchema: z.ZodObject<{
             baseUrl?: string | undefined;
         } | undefined;
         mistral?: {
-            apiKey?: string | undefined;
             model?: string | undefined;
+            apiKey?: string | undefined;
         } | undefined;
     } | undefined;
-    defaultProvider?: "openai" | "anthropic" | "vertex" | "google-ai" | "bedrock" | "azure" | "huggingface" | "ollama" | "mistral" | "auto" | undefined;
+    defaultProvider?: "openai" | "anthropic" | "vertex" | "google-ai" | "auto" | "bedrock" | "azure" | "huggingface" | "ollama" | "mistral" | undefined;
     profiles?: Record<string, any> | undefined;
     preferences?: {
         maxTokens?: number | undefined;

package/dist/cli/factories/commandFactory.d.ts

@@ -5,6 +5,7 @@ import type { CommandModule } from "yargs";
 export declare class CLICommandFactory {
     private static readonly commonOptions;
     private static buildOptions;
+    private static processCliImages;
     private static processOptions;
     private static handleOutput;
     private static isValidTokenUsage;

package/dist/cli/factories/commandFactory.js

@@ -15,7 +15,6 @@ import chalk from "chalk";
 import { logger } from "../../lib/utils/logger.js";
 import fs from "fs";
 import { handleSetup } from "../commands/setup.js";
-// Use specific command interfaces from cli.ts instead of universal interface
 /**
  * CLI Command Factory for generate commands
  */
@@ -44,6 +43,11 @@ export class CLICommandFactory {
             description: "AI provider to use (auto-selects best available)",
             alias: "p",
         },
+        image: {
+            type: "string",
+            description: "Add image file for multimodal analysis (can be used multiple times)",
+            alias: "i",
+        },
         model: {
             type: "string",
             description: "Specific model to use (e.g. gemini-2.5-pro, gemini-2.5-flash)",
@@ -172,6 +176,17 @@ export class CLICommandFactory {
             ...additionalOptions,
         });
     }
+    // Helper method to process CLI images with smart auto-detection
+    static processCliImages(images) {
+        if (!images) {
+            return undefined;
+        }
+        const imagePaths = Array.isArray(images) ? images : [images];
+        // Return as-is - let the smart message builder handle URL vs file detection
+        // URLs will be detected and appended to prompt text
+        // File paths will be converted to base64 by the message builder
+        return imagePaths;
+    }
     // Helper method to process common options
     static processOptions(argv) {
         // Handle noColor option by disabling chalk
@@ -927,8 +942,12 @@ export class CLICommandFactory {
                     toolsEnabled: !options.disableTools,
                 });
             }
+            // Process CLI images if provided
+            const imageBuffers = CLICommandFactory.processCliImages(argv.image);
             const result = await sdk.generate({
-                input: { text: inputText },
+                input: imageBuffers
+                    ? { text: inputText, images: imageBuffers }
+                    : { text: inputText },
                 provider: enhancedOptions.provider,
                 model: enhancedOptions.model,
                 temperature: enhancedOptions.temperature,
@@ -1139,8 +1158,12 @@ export class CLICommandFactory {
             const context = sessionId
                 ? { ...contextMetadata, sessionId }
                 : contextMetadata;
+            // Process CLI images if provided
+            const imageBuffers = CLICommandFactory.processCliImages(argv.image);
             const stream = await sdk.stream({
-                input: { text: inputText },
+                input: imageBuffers
+                    ? { text: inputText, images: imageBuffers }
+                    : { text: inputText },
                 provider: enhancedOptions.provider,
                 model: enhancedOptions.model,
                 temperature: enhancedOptions.temperature,

package/dist/config/taskClassificationConfig.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Task Classification Configuration
+ * Contains patterns, keywords, and scoring weights for task classification
+ */
+/**
+ * Regular expression patterns that indicate fast response tasks
+ */
+export declare const FAST_PATTERNS: RegExp[];
+/**
+ * Regular expression patterns that indicate reasoning tasks
+ */
+export declare const REASONING_PATTERNS: RegExp[];
+/**
+ * Keywords that indicate fast tasks regardless of context
+ */
+export declare const FAST_KEYWORDS: string[];
+/**
+ * Keywords that indicate reasoning tasks regardless of context
+ */
+export declare const REASONING_KEYWORDS: string[];
+/**
+ * Scoring weights for different classification factors
+ */
+export declare const SCORING_WEIGHTS: {
+    readonly SHORT_PROMPT_BONUS: 2;
+    readonly LONG_PROMPT_BONUS: 1;
+    readonly PATTERN_MATCH_SCORE: 3;
+    readonly KEYWORD_MATCH_SCORE: 1;
+    readonly MULTIPLE_QUESTIONS_BONUS: 1;
+    readonly MULTI_SENTENCE_BONUS: 1;
+    readonly TECHNICAL_DOMAIN_BONUS: 1;
+    readonly SIMPLE_DEFINITION_BONUS: 2;
+};
+/**
+ * Classification thresholds and constraints
+ */
+export declare const CLASSIFICATION_THRESHOLDS: {
+    readonly SHORT_PROMPT_LENGTH: 50;
+    readonly LONG_PROMPT_LENGTH: 200;
+    readonly SIMPLE_DEFINITION_LENGTH: 100;
+    readonly MIN_CONFIDENCE: 0.6;
+    readonly MAX_CONFIDENCE: 0.95;
+    readonly DEFAULT_CONFIDENCE: 0.5;
+};
+/**
+ * Domain-specific patterns for enhanced classification
+ */
+export declare const DOMAIN_PATTERNS: {
+    readonly TECHNICAL: RegExp;
+    readonly SIMPLE_DEFINITION: RegExp;
+};