@juspay/neurolink 12.14.21 → 12.15.0

package/dist/rag/document/MDocument.js

@@ -194,9 +194,14 @@ export class MDocument {
      * Generate embeddings for all chunks
      * @param provider - Embedding provider name
      * @param modelName - Embedding model name
+     * @param imageData - Optional array of image data (Buffer or base64) aligned with chunks,
+     *                    enabling multi-modal embeddings for providers that support them
+     * @param mimeType - MIME type of the images (default: "image/png"). Pass an
+     *                    array to declare a per-image MIME type aligned with
+     *                    `imageData`; a plain string applies to every image.
      * @returns This MDocument instance (for chaining)
      */
-    async embed(provider = "openai", modelName = "text-embedding-3-small") {
+    async embed(provider = "openai", modelName = "text-embedding-3-small", imageData, mimeType = "image/png") {
         if (this.state.chunks.length === 0) {
             logger.warn("[MDocument] No chunks to embed. Call chunk() first.");
             return this;
@@ -208,14 +213,21 @@ export class MDocument {
             chunkCount: this.state.chunks.length,
             provider,
             model: modelName,
+            hasImages: imageData !== undefined && imageData.length > 0,
         });
         const embeddingProvider = await ProviderFactory.createProvider(provider, modelName);
         if (typeof embeddingProvider.embed !== "function") {
             throw new Error(`Provider ${provider} does not support embeddings`);
         }
         this.state.embeddings = [];
-        for (const chunk of this.state.chunks) {
-            const embedding = await embeddingProvider.embed(chunk.text);
+        for (let i = 0; i < this.state.chunks.length; i++) {
+            const chunk = this.state.chunks[i];
+            const image = imageData?.[i];
+            const imageMimeType = typeof mimeType === "string" ? mimeType : (mimeType[i] ?? "image/png");
+            const input = image
+                ? { text: chunk.text, image, mimeType: imageMimeType }
+                : chunk.text;
+            const embedding = await embeddingProvider.embed(input);
             this.state.embeddings.push(embedding);
             chunk.embedding = embedding;
         }
@@ -227,6 +239,21 @@ export class MDocument {
         });
         return this;
     }
+    /**
+     * Generate multi-modal embeddings for all chunks using text + optional image data.
+     * Uses a provider that supports multi-modal embeddings (e.g. Bedrock Titan Image).
+     *
+     * @param provider - Embedding provider name
+     * @param modelName - Embedding model name
+     * @param imageData - Optional array of image data (Buffer or base64) aligned with chunks
+     * @param mimeType - MIME type of the images (default: "image/png"). Pass an
+     *                    array to declare a per-image MIME type aligned with
+     *                    `imageData`; a plain string applies to every image.
+     * @returns This MDocument instance (for chaining)
+     */
+    async embedMultiModal(provider = "bedrock", modelName = "amazon.titan-embed-image-v1", imageData, mimeType = "image/png") {
+        return this.embed(provider, modelName, imageData, mimeType);
+    }
     // ============================================================================
     // Accessor Methods
     // ============================================================================

package/dist/rag/document/imageLoader.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Image Loader for Multi-Modal RAG
+ *
+ * Loads image files (from disk or URLs) into ImageDocument objects
+ * that can be embedded and stored in a vector store alongside text chunks.
+ *
+ * @example
+ * ```typescript
+ * import { ImageLoader } from './imageLoader.js';
+ *
+ * const loader = new ImageLoader();
+ *
+ * // Load from file path
+ * const doc = await loader.load('./photo.jpg');
+ *
+ * // Load from URL
+ * const doc2 = await loader.load('https://example.com/image.png');
+ *
+ * // Load from buffer
+ * const doc3 = loader.loadFromBuffer(imageBuffer, 'image/jpeg');
+ * ```
+ */
+import type { ImageDocument, ImageLoaderOptions } from "../../types/index.js";
+/**
+ * ImageLoader for multi-modal RAG.
+ * Loads images from file paths, URLs, or buffers into ImageDocument objects.
+ */
+export declare class ImageLoader {
+    private options;
+    constructor(options?: ImageLoaderOptions);
+    /**
+     * Load an image from a file path or URL.
+     */
+    load(source: string, options?: Partial<ImageLoaderOptions>): Promise<ImageDocument>;
+    /**
+     * Load an image from a local file path.
+     */
+    private loadFromPath;
+    /**
+     * Load an image from a URL.
+     */
+    private loadFromURL;
+    /**
+     * Load an image directly from a Buffer.
+     */
+    loadFromBuffer(buffer: Buffer, mimeType: string, source?: string, options?: Partial<ImageLoaderOptions>): ImageDocument;
+    /**
+     * Check if a file path or extension is a supported image format.
+     */
+    static isImageFile(source: string): boolean;
+    /**
+     * Get the MIME type for a file extension.
+     */
+    static getMimeType(filePath: string): string | undefined;
+}

package/dist/rag/document/imageLoader.js

@@ -0,0 +1,183 @@
+/**
+ * Image Loader for Multi-Modal RAG
+ *
+ * Loads image files (from disk or URLs) into ImageDocument objects
+ * that can be embedded and stored in a vector store alongside text chunks.
+ *
+ * @example
+ * ```typescript
+ * import { ImageLoader } from './imageLoader.js';
+ *
+ * const loader = new ImageLoader();
+ *
+ * // Load from file path
+ * const doc = await loader.load('./photo.jpg');
+ *
+ * // Load from URL
+ * const doc2 = await loader.load('https://example.com/image.png');
+ *
+ * // Load from buffer
+ * const doc3 = loader.loadFromBuffer(imageBuffer, 'image/jpeg');
+ * ```
+ */
+import { readFileSync, existsSync, statSync } from "fs";
+import { extname } from "path";
+import { logger } from "../../utils/logger.js";
+import { safeDownload } from "../../utils/safeFetch.js";
+import { redactUrlForError } from "../../utils/logSanitize.js";
+import { ImageProcessor, VALID_IMAGE_EXTENSIONS, } from "../../utils/imageProcessor.js";
+import { EXTENSION_MIME_MAP } from "../../processors/config/index.js";
+/**
+ * Generate a text representation from the image source.
+ */
+function generateTextDescription(source) {
+    // Drop any query string or fragment before deriving the name. On a presigned
+    // URL the signature lives in the query, and the query is part of the final
+    // slash-separated segment — so `.pop()` alone keeps it, and this caption is
+    // not a transient log line: it becomes embedded, BM25-indexed chunk text that
+    // persists in the vector store and is read back into model context.
+    const withoutQueryOrFragment = source.split(/[?#]/)[0];
+    const filename = withoutQueryOrFragment.split("/").pop() ?? source;
+    // Remove extension and clean up
+    const name = filename
+        .replace(/\.[^.]+$/, "")
+        .replace(/[-_]/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    return `Image: ${name}`;
+}
+/**
+ * ImageLoader for multi-modal RAG.
+ * Loads images from file paths, URLs, or buffers into ImageDocument objects.
+ */
+export class ImageLoader {
+    options;
+    constructor(options) {
+        this.options = {
+            metadata: options?.metadata ?? {},
+            maxImageSize: options?.maxImageSize ?? 10 * 1024 * 1024, // 10MB
+            fetchTimeout: options?.fetchTimeout ?? 30_000,
+            headers: options?.headers ?? {},
+        };
+    }
+    /**
+     * Load an image from a file path or URL.
+     */
+    async load(source, options) {
+        const opts = { ...this.options, ...options };
+        if (source.startsWith("http://") || source.startsWith("https://")) {
+            return this.loadFromURL(source, opts);
+        }
+        return this.loadFromPath(source, opts);
+    }
+    /**
+     * Load an image from a local file path.
+     */
+    async loadFromPath(filePath, options) {
+        if (!existsSync(filePath)) {
+            throw new Error(`Image file not found: ${filePath}`);
+        }
+        const { size } = statSync(filePath);
+        if (size > options.maxImageSize) {
+            throw new Error(`Image size (${size} bytes) exceeds maximum (${options.maxImageSize} bytes)`);
+        }
+        const buffer = readFileSync(filePath);
+        const ext = extname(filePath).toLowerCase();
+        const mimeType = EXTENSION_MIME_MAP[ext] ?? ImageProcessor.detectImageType(buffer);
+        const dimensions = ImageProcessor.getImageDimensions(buffer);
+        logger.debug("[ImageLoader] Loaded image from path", {
+            path: filePath,
+            size: buffer.byteLength,
+            mimeType,
+            dimensions,
+        });
+        return {
+            id: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+            text: generateTextDescription(filePath),
+            image: buffer,
+            mimeType,
+            metadata: {
+                source: filePath,
+                width: dimensions?.width,
+                height: dimensions?.height,
+                format: ext.replace(".", ""),
+                size: buffer.byteLength,
+                ...options.metadata,
+            },
+        };
+    }
+    /**
+     * Load an image from a URL.
+     */
+    async loadFromURL(url, options) {
+        const buffer = await safeDownload(url, {
+            maxBytes: options.maxImageSize,
+            label: "RAG image",
+            timeoutMs: options.fetchTimeout,
+        });
+        const mimeType = ImageProcessor.detectImageType(buffer);
+        const dimensions = ImageProcessor.getImageDimensions(buffer);
+        logger.debug("[ImageLoader] Loaded image from URL", {
+            url: redactUrlForError(url),
+            size: buffer.byteLength,
+            mimeType,
+            dimensions,
+        });
+        return {
+            id: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+            text: generateTextDescription(url),
+            image: buffer,
+            mimeType,
+            metadata: {
+                // Redacted for the same reason as the caption above: `source` is
+                // persisted into the vector store and returned verbatim to the model
+                // as the RAG tool's citation, so a signed URL stored here outlives the
+                // request that fetched it.
+                source: redactUrlForError(url),
+                width: dimensions?.width,
+                height: dimensions?.height,
+                format: mimeType.split("/").at(1),
+                size: buffer.byteLength,
+                ...options.metadata,
+            },
+        };
+    }
+    /**
+     * Load an image directly from a Buffer.
+     */
+    loadFromBuffer(buffer, mimeType, source, options) {
+        const opts = { ...this.options, ...options };
+        if (buffer.byteLength > opts.maxImageSize) {
+            throw new Error(`Image size (${buffer.byteLength} bytes) exceeds maximum (${opts.maxImageSize} bytes)`);
+        }
+        const dimensions = ImageProcessor.getImageDimensions(buffer);
+        return {
+            id: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+            text: generateTextDescription(source ?? "inline-image"),
+            image: buffer,
+            mimeType,
+            metadata: {
+                source: source ?? "inline",
+                width: dimensions?.width,
+                height: dimensions?.height,
+                format: mimeType.split("/").at(1),
+                size: buffer.byteLength,
+                ...opts.metadata,
+            },
+        };
+    }
+    /**
+     * Check if a file path or extension is a supported image format.
+     */
+    static isImageFile(source) {
+        const ext = extname(source).toLowerCase().replace(/^\./, "");
+        return VALID_IMAGE_EXTENSIONS.includes(ext);
+    }
+    /**
+     * Get the MIME type for a file extension.
+     */
+    static getMimeType(filePath) {
+        const ext = extname(filePath).toLowerCase();
+        return EXTENSION_MIME_MAP[ext];
+    }
+}

package/dist/rag/index.d.ts

@@ -48,6 +48,7 @@ export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChun
 export * from "./chunkers/index.js";
 export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
 export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
+export { ImageLoader } from "./document/imageLoader.js";
 export * from "./errors/index.js";
 export { GraphRAG } from "./graphRag/index.js";
 export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor, MetadataExtractorFactory, MetadataExtractorRegistry, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";

package/dist/rag/index.js

@@ -52,6 +52,8 @@ export * from "./chunkers/index.js";
 export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
 // Document Processing
 export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
+// Image Loader for multi-modal RAG
+export { ImageLoader } from "./document/imageLoader.js";
 // Error handling
 export * from "./errors/index.js";
 // Graph RAG

package/dist/rag/pipeline/RAGPipeline.d.ts

@@ -25,7 +25,7 @@
  * console.log(response.answer, response.sources);
  * ```
  */
-import type { RAGPipelineConfig, IngestOptions, QueryOptions, RAGResponse, PipelineStats } from "../../types/index.js";
+import type { RAGPipelineConfig, IngestOptions, QueryOptions, RAGResponse, PipelineStats, MultiModalRAGConfig, MultiModalQuery, MultiModalSearchResult } from "../../types/index.js";
 import { MDocument } from "../document/MDocument.js";
 export declare class RAGPipeline {
     private id;
@@ -34,10 +34,21 @@ export declare class RAGPipeline {
     private bm25Index;
     private graphRAG;
     private embeddingProvider?;
+    private multiModalEmbeddingProvider?;
     private generationProvider?;
+    /**
+     * Vision provider for image captioning, when `multiModal.captionProvider` /
+     * `captionModel` name one. Undefined means captioning uses
+     * {@link generationProvider} — captioning has always run on that provider,
+     * and these two fields were declared but read nowhere, so a caller who set
+     * them got the pipeline's generation model with no indication otherwise.
+     */
+    private captionProvider?;
     private hybridSearch?;
     private documents;
     private allChunks;
+    private multiModalConfig?;
+    private imageLoader;
     constructor(config: RAGPipelineConfig);
     /**
      * Initialize the pipeline (lazy loading of providers)
@@ -73,6 +84,47 @@ export declare class RAGPipeline {
      * Clear all indexed data
      */
     clear(): Promise<void>;
+    /**
+     * Ingest images into the pipeline for multi-modal RAG.
+     * Loads images, generates embeddings via the configured multi-modal provider,
+     * and stores them alongside text chunks in the vector store.
+     *
+     * @param sources - Array of image file paths, URLs, or Buffer objects
+     * @param options - Ingestion options
+     */
+    ingestImages(sources: Array<string | Buffer>, options?: IngestOptions): Promise<{
+        imagesProcessed: number;
+        chunksCreated: number;
+    }>;
+    /**
+     * Query the pipeline with multi-modal input (text, image, or both).
+     *
+     * @param query - Text, image, or combined query
+     * @param options - Query options
+     * @returns Array of multi-modal search results
+     */
+    queryMultiModal(query: MultiModalQuery, options?: QueryOptions): Promise<MultiModalSearchResult[]>;
+    /**
+     * Get multi-modal pipeline statistics
+     */
+    getMultiModalStats(): {
+        totalImages: number;
+        totalTextChunks: number;
+        multiModalEnabled: boolean;
+    };
+    /**
+     * Generate a multi-modal embedding for text, image, or both.
+     * Uses the configured multi-modal embedding provider (e.g. Bedrock Titan Image).
+     */
+    private generateMultiModalEmbedding;
+    /**
+     * Generate a caption for an image using the generation provider's vision model.
+     */
+    private generateImageCaption;
+    /**
+     * Diversify multi-modal results to ensure a mix of text and image results.
+     */
+    private diversifyMultiModalResults;
     /**
      * Ensure pipeline is initialized
      */
@@ -102,4 +154,5 @@ export declare function createRAGPipeline(options: {
     generationModel?: string;
     enableHybrid?: boolean;
     enableGraph?: boolean;
+    multiModal?: MultiModalRAGConfig;
 }): RAGPipeline;