@juspay/neurolink 8.3.0 → 8.4.1

package/dist/types/multimodal.js

@@ -0,0 +1,100 @@
+/**
+ * Multimodal Content Types for NeuroLink
+ *
+ * Central registry for all multimodal input/output types.
+ * This file consolidates types from content.ts and conversation.ts
+ * to provide a single source of truth for multimodal functionality.
+ *
+ * @module types/multimodal
+ *
+ * @example Basic Multimodal Input
+ * ```typescript
+ * import type { MultimodalInput } from './types/multimodal.js';
+ *
+ * const input: MultimodalInput = {
+ *   text: "What's in this image?",
+ *   images: [imageBuffer, "https://example.com/image.jpg"],
+ *   pdfFiles: [pdfBuffer]
+ * };
+ * ```
+ *
+ * @example Audio/Video Input (Future)
+ * ```typescript
+ * const avInput: MultimodalInput = {
+ *   text: "Transcribe this audio and analyze this video",
+ *   audioFiles: [audioBuffer],
+ *   videoFiles: ["path/to/video.mp4"]
+ * };
+ * ```
+ *
+ * @example Advanced Content Array
+ * ```typescript
+ * const advanced: MultimodalInput = {
+ *   text: "irrelevant", // ignored when content[] is provided
+ *   content: [
+ *     { type: "text", text: "Analyze these items:" },
+ *     { type: "image", data: imageBuffer, mediaType: "image/jpeg" },
+ *     { type: "pdf", data: pdfBuffer, metadata: { filename: "report.pdf" } }
+ *   ]
+ * };
+ * ```
+ */
+// ============================================
+// TYPE GUARDS
+// ============================================
+/**
+ * Type guard to check if content is TextContent
+ */
+export function isTextContent(content) {
+    return content.type === "text";
+}
+/**
+ * Type guard to check if content is ImageContent
+ */
+export function isImageContent(content) {
+    return content.type === "image";
+}
+/**
+ * Type guard to check if content is CSVContent
+ */
+export function isCSVContent(content) {
+    return content.type === "csv";
+}
+/**
+ * Type guard to check if content is PDFContent
+ */
+export function isPDFContent(content) {
+    return content.type === "pdf";
+}
+/**
+ * Type guard to check if content is AudioContent
+ */
+export function isAudioContent(content) {
+    return content.type === "audio";
+}
+/**
+ * Type guard to check if content is VideoContent
+ */
+export function isVideoContent(content) {
+    return content.type === "video";
+}
+/**
+ * Type guard to check if input contains multimodal content
+ * Now includes audio and video detection
+ */
+export function isMultimodalInput(input) {
+    const maybeInput = input;
+    return !!(maybeInput?.images?.length ||
+        maybeInput?.csvFiles?.length ||
+        maybeInput?.pdfFiles?.length ||
+        maybeInput?.files?.length ||
+        maybeInput?.content?.length ||
+        maybeInput?.audioFiles?.length ||
+        maybeInput?.videoFiles?.length);
+}
+/**
+ * Type guard to check if message content is multimodal (array)
+ */
+export function isMultimodalMessageContent(content) {
+    return Array.isArray(content);
+}

package/dist/types/streamTypes.d.ts

@@ -1,7 +1,7 @@
 import type { Tool } from "ai";
 import type { ValidationSchema, StandardRecord } from "./typeAliases.js";
 import type { AIModelProviderConfig } from "./providers.js";
-import type { TextContent, ImageContent } from "./content.js";
+import type { Content } from "./content.js";
 import type { AnalyticsData, ToolExecutionEvent, ToolExecutionSummary } from "../types/index.js";
 import { AIProviderName } from "../constants/enums.js";
 import type { TokenUsage } from "./analytics.js";
@@ -129,7 +129,7 @@ export type StreamOptions = {
         csvFiles?: Array<Buffer | string>;
         pdfFiles?: Array<Buffer | string>;
         files?: Array<Buffer | string>;
-        content?: Array<TextContent | ImageContent>;
+        content?: Content[];
     };
     output?: {
         format?: "text" | "structured" | "json";

package/dist/utils/imageProcessor.d.ts

@@ -2,7 +2,7 @@
  * Image processing utilities for multimodal support
  * Handles format conversion for different AI providers
  */
-import type { ProcessedImage } from "../types/content.js";
+import type { ProcessedImage } from "../types/multimodal.js";
 import type { FileProcessingResult } from "../types/fileTypes.js";
 /**
  * Image processor class for handling provider-specific image formatting

package/dist/utils/messageBuilder.js

@@ -473,6 +473,26 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     if (hasConversationHistory) {
         systemPrompt = `${systemPrompt.trim()}${CONVERSATION_INSTRUCTIONS}`;
     }
+    // Add file handling guidance when multimodal files are present
+    const hasCSVFiles = (options.input.csvFiles && options.input.csvFiles.length > 0) ||
+        (options.input.files &&
+            options.input.files.some((f) => typeof f === "string" ? f.toLowerCase().endsWith(".csv") : false));
+    const hasPDFFiles = pdfFiles.length > 0;
+    if (hasCSVFiles || hasPDFFiles) {
+        const fileTypes = [];
+        if (hasPDFFiles) {
+            fileTypes.push("PDFs");
+        }
+        if (hasCSVFiles) {
+            fileTypes.push("CSVs");
+        }
+        systemPrompt += `\n\nIMPORTANT FILE HANDLING INSTRUCTIONS:
+- File content (${fileTypes.join(", ")}, images) is already processed and included in this message
+- DO NOT use GitHub tools (get_file_contents, search_code, etc.) for local files - they only work for remote repository files
+- Analyze the provided file content directly without attempting to fetch or read files using tools
+- GitHub MCP tools are ONLY for remote repository operations, not local filesystem access
+- Use the file content shown in this message for your analysis`;
+    }
     // Add system message if we have one
     if (systemPrompt.trim()) {
         messages.push({
@@ -739,7 +759,10 @@ async function convertMultimodalToProviderFormat(text, images, pdfFiles, provide
             });
         }
     }
-    // Add PDFs using Vercel AI SDK standard format (works for all providers)
+    // Add PDFs using Vercel AI SDK standard format (works for all providers except Mistral)
+    // NOTE: Mistral API has a fundamental limitation - it does NOT support PDFs in any form.
+    // The API strictly requires image content to start with data:image/, rejecting data:application/pdf
+    // See: MISTRAL_PDF_FIX_SUMMARY.md for full investigation details
     content.push(...pdfFiles.map((pdf) => {
         logger.info(`[PDF] ✅ Added to content (Vercel AI SDK format): ${pdf.filename}`);
         return {
@@ -769,5 +792,5 @@ function extractFilename(file, index = 0) {
     return `file-${index + 1}`;
 }
 function buildCSVToolInstructions(filePath) {
-    return `\n**IMPORTANT**: For counting, aggregation, or statistical operations, use the analyzeCSV tool with filePath="${filePath}". The tool reads the file directly - do NOT pass CSV content.\n\nExample: analyzeCSV(filePath="${filePath}", operation="count_by_column", column="merchant_id")\n\n`;
+    return `\n**NOTE**: You can perform calculations directly on the CSV data shown above. For advanced operations on the full file (counting by column, grouping, etc.), you may optionally use the analyzeCSV tool with filePath="${filePath}".\n\nExample: analyzeCSV(filePath="${filePath}", operation="count_by_column", column="merchant_id")\n\n`;
 }

package/dist/utils/multimodalOptionsBuilder.d.ts

@@ -45,7 +45,7 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
     input: {
         text: string;
         images: (string | Buffer<ArrayBufferLike>)[] | undefined;
-        content: (import("../types/content.js").TextContent | import("../types/content.js").ImageContent)[] | undefined;
+        content: import("../types/multimodal.js").Content[] | undefined;
         files: (string | Buffer<ArrayBufferLike>)[] | undefined;
         csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
         pdfFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;

package/dist/utils/pdfProcessor.d.ts

@@ -7,4 +7,13 @@ export declare class PDFProcessor {
     private static isValidPDF;
     private static extractBasicMetadata;
     static estimateTokens(pageCount: number, mode?: "text-only" | "visual"): number;
+    static convertPDFToImages(pdfBuffer: Buffer, options?: {
+        maxPages?: number;
+        scale?: number;
+        format?: "png" | "jpeg";
+        quality?: number;
+    }): Promise<Array<{
+        buffer: Buffer;
+        pageNumber: number;
+    }>>;
 }

package/dist/utils/pdfProcessor.js

@@ -1,4 +1,6 @@
 import { logger } from "./logger.js";
+import * as pdfjs from "pdfjs-dist/legacy/build/pdf.mjs";
+import { createCanvas } from "canvas";
 const PDF_PROVIDER_CONFIGS = {
     anthropic: {
         maxSizeMB: 5,
@@ -87,7 +89,7 @@ const PDF_PROVIDER_CONFIGS = {
     mistral: {
         maxSizeMB: 10,
         maxPages: 100,
-        supportsNative: true,
+        supportsNative: false,
         requiresCitations: false,
         apiType: "files-api",
     },
@@ -115,16 +117,14 @@ export class PDFProcessor {
         if (!this.isValidPDF(content)) {
             throw new Error("Invalid PDF file format. File must start with %PDF- header.");
         }
-        if (!config || !config.supportsNative) {
-            const supportedProviders = Object.keys(PDF_PROVIDER_CONFIGS)
-                .filter((p) => PDF_PROVIDER_CONFIGS[p].supportsNative)
-                .join(", ");
-            throw new Error(`PDF files are not currently supported with ${provider} provider.\n` +
-                `Supported providers: ${supportedProviders}\n` +
+        if (!config) {
+            const supportedProviders = Object.keys(PDF_PROVIDER_CONFIGS).join(", ");
+            throw new Error(`PDF files are not configured for ${provider} provider.\n` +
+                `Configured providers: ${supportedProviders}\n` +
                 `Current provider: ${provider}\n\n` +
                 `Options:\n` +
-                `1. Switch to a supported provider (--provider openai or --provider vertex)\n` +
-                `2. Convert your PDF to text manually`);
+                `1. Switch to a configured provider (--provider openai or --provider vertex)\n` +
+                `2. Contact support to add ${provider} PDF configuration`);
         }
         const sizeMB = content.length / (1024 * 1024);
         if (sizeMB > config.maxSizeMB) {
@@ -195,4 +195,62 @@ export class PDFProcessor {
             return Math.ceil((pageCount / 3) * 7000);
         }
     }
+    static async convertPDFToImages(pdfBuffer, options) {
+        const maxPages = options?.maxPages || 10;
+        const scale = options?.scale || 2.0;
+        const format = options?.format || "png";
+        const quality = options?.quality || 0.9;
+        let pdfDocument = null;
+        try {
+            const loadingTask = pdfjs.getDocument({
+                data: new Uint8Array(pdfBuffer),
+                useSystemFonts: true,
+                standardFontDataUrl: `https://cdn.jsdelivr.net/npm/pdfjs-dist@${pdfjs.version}/standard_fonts/`,
+            });
+            pdfDocument = await loadingTask.promise;
+            const numPages = Math.min(pdfDocument.numPages, maxPages);
+            const images = [];
+            logger.info(`[PDF→Image] Converting ${numPages} page(s) from PDF (total: ${pdfDocument.numPages})`);
+            for (let pageNum = 1; pageNum <= numPages; pageNum++) {
+                const page = await pdfDocument.getPage(pageNum);
+                const viewport = page.getViewport({ scale });
+                const canvas = createCanvas(viewport.width, viewport.height);
+                const context = canvas.getContext("2d");
+                await page.render({
+                    canvasContext: context,
+                    viewport,
+                    // @ts-expect-error - canvas type mismatch between node-canvas and pdfjs-dist
+                    canvas: canvas,
+                }).promise;
+                const imageBuffer = format === "png"
+                    ? canvas.toBuffer("image/png")
+                    : canvas.toBuffer("image/jpeg", { quality });
+                images.push({ buffer: imageBuffer, pageNumber: pageNum });
+                logger.debug(`[PDF→Image] ✅ Converted page ${pageNum}/${numPages} (${(imageBuffer.length / 1024).toFixed(1)}KB)`);
+            }
+            if (pdfDocument.numPages > maxPages) {
+                logger.warn(`[PDF→Image] PDF has ${pdfDocument.numPages} pages, converted only first ${maxPages} pages`);
+            }
+            logger.info(`[PDF→Image] ✅ Successfully converted ${images.length} page(s) to images`);
+            return images;
+        }
+        catch (error) {
+            logger.error(`[PDF→Image] ❌ Failed to convert PDF to images:`, error instanceof Error ? error.message : String(error));
+            throw new Error(`PDF to image conversion failed: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        finally {
+            // Ensure pdfDocument is destroyed regardless of success or failure
+            if (pdfDocument) {
+                try {
+                    pdfDocument.destroy();
+                    logger.debug("[PDF→Image] PDF document resources cleaned up");
+                }
+                catch (destroyError) {
+                    logger.warn("[PDF→Image] Error destroying PDF document:", destroyError instanceof Error
+                        ? destroyError.message
+                        : String(destroyError));
+                }
+            }
+        }
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "8.3.0",
+  "version": "8.4.1",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",
@@ -179,6 +179,7 @@
     "@opentelemetry/sdk-trace-node": "^2.1.0",
     "@opentelemetry/semantic-conventions": "^1.30.1",
     "ai": "4.3.16",
+    "canvas": "^3.2.0",
     "chalk": "^5.6.2",
     "csv-parser": "^3.2.0",
     "dotenv": "^16.6.1",
@@ -190,6 +191,7 @@
     "ollama-ai-provider": "^1.2.0",
     "ora": "^7.0.1",
     "p-limit": "^6.2.0",
+    "pdfjs-dist": "^5.4.296",
     "reconnecting-eventsource": "^1.6.4",
     "redis": "^5.8.2",
     "undici": "^7.5.0",
@@ -286,7 +288,8 @@
       "esbuild",
       "protobufjs",
       "puppeteer",
-      "sqlite3"
+      "sqlite3",
+      "canvas"
     ],
     "overrides": {
       "esbuild@<=0.24.2": ">=0.25.0",