@juspay/neurolink 12.14.21 → 12.15.0

package/dist/rag/ragIntegration.js

@@ -3,8 +3,11 @@ import { extname, resolve } from "path";
 import { z } from "zod";
 import { SpanSerializer, SpanType, SpanStatus, getMetricsAggregator, } from "../observability/index.js";
 import { logger } from "../utils/logger.js";
+import { redactUrlForError } from "../utils/logSanitize.js";
 import { createChunker } from "./ChunkerFactory.js";
 import { createVectorQueryTool, InMemoryVectorStore, } from "./retrieval/vectorQueryTool.js";
+import { ImageLoader } from "./document/imageLoader.js";
+import { IMAGE_EXTENSIONS } from "../processors/config/index.js";
 import { withSpan } from "../telemetry/withSpan.js";
 import { tracers } from "../telemetry/tracers.js";
 /**
@@ -43,6 +46,8 @@ function detectStrategy(filePath) {
     const ext = extname(filePath).toLowerCase();
     return EXTENSION_TO_STRATEGY[ext] || "recursive";
 }
+/** Embedding dimension used by {@link generateSimpleEmbedding}. */
+const EMBEDDING_DIMENSION = 128;
 /**
  * Simple hash function for strings (FNV-1a variant).
  * Maps a word to a bucket index deterministically.
@@ -177,14 +182,62 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
     if (!files || files.length === 0) {
         throw new Error("RAG config requires at least one file path in 'files'");
     }
-    // 1. Load files
+    // 1. Load files — separate text and image files
     const fileContents = [];
+    const imageFiles = [];
     for (const filePath of files) {
+        // HTTP(S) sources are handled by ImageLoader directly and must not go
+        // through resolve()/existsSync(), which would mangle the URL into a
+        // bogus local path and always report it as missing.
+        if (/^https?:\/\//i.test(filePath)) {
+            // `new URL()` throws on a string the guarding regex still accepts —
+            // "https://" is the minimal case. Uncaught, it leaves this loop, is
+            // rethrown by prepareRAGTool, and disables RAG for the WHOLE call, so
+            // one bad entry discards every file already loaded. Skip the source
+            // instead, matching how an unreadable local file is handled below.
+            let ext;
+            try {
+                ext = extname(new URL(filePath).pathname).toLowerCase();
+            }
+            catch {
+                logger.warn(`[RAG] Malformed URL source, skipping: ${redactUrlForError(filePath)}`);
+                continue;
+            }
+            if (ext === ".svg") {
+                // IMAGE_EXTENSIONS includes .svg, but every consumer downstream of
+                // here treats an image as raster bytes: ImageLoader returns the file
+                // verbatim, captioning base64s it into a vision call, and the Bedrock
+                // embed path ships it as-is — Nova rejects the format outright and
+                // Titan accepts markup it cannot embed meaningfully. Skipping is
+                // honest; supporting it needs a rasterize-or-sanitize step in
+                // ImageLoader, not a classification change here.
+                logger.warn(`[RAG] SVG is not supported as a RAG image source, skipping: ${redactUrlForError(filePath)}`);
+                continue;
+            }
+            if (IMAGE_EXTENSIONS.includes(ext)) {
+                imageFiles.push({ path: filePath, ext });
+            }
+            else {
+                logger.warn(`[RAG] Unsupported URL source, skipping: ${redactUrlForError(filePath)}`);
+            }
+            continue;
+        }
         const resolvedPath = resolve(filePath);
         if (!existsSync(resolvedPath)) {
             logger.warn(`[RAG] File not found, skipping: ${resolvedPath}`);
             continue;
         }
+        // Detect image files by extension
+        const ext = extname(resolvedPath).toLowerCase();
+        if (ext === ".svg") {
+            // See the URL branch above for why SVG is excluded from the raster path.
+            logger.warn(`[RAG] SVG is not supported as a RAG image source, skipping: ${resolvedPath}`);
+            continue;
+        }
+        if (IMAGE_EXTENSIONS.includes(ext)) {
+            imageFiles.push({ path: resolvedPath, ext });
+            continue;
+        }
         try {
             const content = readFileSync(resolvedPath, "utf-8");
             const strategy = userStrategy || detectStrategy(resolvedPath);
@@ -195,11 +248,12 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
         }
     }
     // Auto-increase topK for multi-file scenarios to ensure coverage
-    // (computed after loading so it reflects only files that actually exist)
-    const topK = fileContents.length > 1
-        ? Math.max(userTopK, fileContents.length * 3)
-        : userTopK;
-    if (fileContents.length === 0) {
+    // (computed after loading so it reflects only files that actually exist).
+    // Image files count toward the file total so image-only RAG gets a topK
+    // that covers every source.
+    const totalSources = fileContents.length + imageFiles.length;
+    const topK = totalSources > 1 ? Math.max(userTopK, totalSources * 3) : userTopK;
+    if (fileContents.length === 0 && imageFiles.length === 0) {
         throw new Error("RAG: No files could be loaded. Check that file paths exist and are readable.");
     }
     logger.info(`[RAG] Loaded ${fileContents.length} files for indexing`);
@@ -230,9 +284,44 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
             });
         }
     }
-    logger.info(`[RAG] Created ${allChunks.length} chunks from ${fileContents.length} files`);
+    logger.info(`[RAG] Created ${allChunks.length} chunks from ${fileContents.length} text files`);
+    // 2b. Load image files (multi-modal RAG).
+    //
+    // Deliberately NOT embedded here. Images are vectorised below through the
+    // same `embedFn` as the text chunks, because index-wide a single embedding
+    // space is the whole precondition for similarity to mean anything. Embedding
+    // them here — as this path originally did, with a hash of the caption —
+    // produced a 128-dimension image vector sitting in an index whose text
+    // vectors came from a provider at 768 or 1536, which is not a worse ranking
+    // but an incomparable one.
+    const imageChunks = [];
+    if (imageFiles.length > 0) {
+        const imageLoader = new ImageLoader();
+        for (const { path: imgPath } of imageFiles) {
+            try {
+                const imageDoc = await imageLoader.load(imgPath);
+                // Use filename-based text representation for simple embedding
+                const imageText = imageDoc.text;
+                imageChunks.push({
+                    id: `img-chunk-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+                    text: imageText,
+                    metadata: {
+                        text: imageText,
+                        source: imgPath,
+                        hasImage: true,
+                        mimeType: imageDoc.mimeType,
+                        imageWidth: imageDoc.metadata.width,
+                        imageHeight: imageDoc.metadata.height,
+                    },
+                });
+            }
+            catch (error) {
+                logger.warn(`[RAG] Failed to load image: ${imgPath}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        logger.info(`[RAG] Loaded ${imageChunks.length} image files for multi-modal RAG`);
+    }
     // 3. Generate embeddings and store in vector store
-    const EMBEDDING_DIMENSION = 128;
     const vectorStore = new InMemoryVectorStore();
     const indexName = "rag-index";
     // When the caller configured an embedding provider/model, embed BOTH the
@@ -262,9 +351,19 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
             logger.warn("[RAG] Failed to create embedding provider; falling back to hash embeddings", { error: error instanceof Error ? error.message : String(error) });
         }
     }
+    // Text and image captions embed together, in one call and under one
+    // fallback. Splitting them into two try/catch blocks would let the text side
+    // succeed on the provider while the image side fell back to the hash, which
+    // is the mixed-space index this is trying to prevent.
     let chunkVectors;
+    let imageVectors;
     try {
-        chunkVectors = await Promise.all(allChunks.map((chunk) => embedFn(chunk.text)));
+        const allVectors = await Promise.all([
+            ...allChunks.map((chunk) => embedFn(chunk.text)),
+            ...imageChunks.map((chunk) => embedFn(chunk.text)),
+        ]);
+        chunkVectors = allVectors.slice(0, allChunks.length);
+        imageVectors = allVectors.slice(allChunks.length);
     }
     catch (error) {
         // One failed chunk must not leave a mixed-space index — flip the whole
@@ -272,17 +371,36 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
         logger.warn("[RAG] Provider embedding failed mid-index; falling back to hash embeddings for index and queries", { error: error instanceof Error ? error.message : String(error) });
         embedFn = (text) => Promise.resolve(generateSimpleEmbedding(text, EMBEDDING_DIMENSION));
         chunkVectors = allChunks.map((chunk) => generateSimpleEmbedding(chunk.text, EMBEDDING_DIMENSION));
+        imageVectors = imageChunks.map((chunk) => generateSimpleEmbedding(chunk.text, EMBEDDING_DIMENSION));
+    }
+    const items = [
+        ...allChunks.map((chunk, i) => ({
+            id: `rag-chunk-${i}`,
+            vector: chunkVectors[i],
+            metadata: {
+                text: chunk.text,
+                ...chunk.metadata,
+            },
+        })),
+        ...imageChunks.map((chunk, i) => ({
+            id: chunk.id,
+            vector: imageVectors[i],
+            metadata: chunk.metadata,
+        })),
+    ];
+    // The guard near the top of this function fires when no source was found
+    // at all. This one covers the other way to end up with an empty index:
+    // sources WERE found and every one of them failed to load — an image over
+    // the size ceiling, a corrupt file, an unreachable URL. Each of those is
+    // caught and warned individually, so without this the call returns
+    // `chunksIndexed: 0` alongside a positive `filesLoaded` and reports
+    // success, and the caller gets a silently empty knowledge base.
+    if (items.length === 0) {
+        throw new Error("RAG: every source failed to load, so nothing was indexed. " +
+            "Check the warnings above for the per-source reason.");
     }
-    const items = allChunks.map((chunk, i) => ({
-        id: `rag-chunk-${i}`,
-        vector: chunkVectors[i],
-        metadata: {
-            text: chunk.text,
-            ...chunk.metadata,
-        },
-    }));
     await vectorStore.upsert(indexName, items);
-    logger.info(`[RAG] Indexed ${items.length} chunks in vector store`);
+    logger.info(`[RAG] Indexed ${items.length} chunks (${allChunks.length} text + ${imageChunks.length} images) in vector store`);
     // 4. Create the search tool
     // Determine embedding provider/model for the query tool
     const provider = embeddingProvider || fallbackProvider || "vertex";
@@ -315,15 +433,16 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                 // Query through the same embedding space the index was built in —
                 // provider embeddings when configured (and healthy), else the hash.
                 const queryEmbedding = await embedFn(query).catch(() => generateSimpleEmbedding(query, EMBEDDING_DIMENSION));
-                // Fetch more candidates than needed so diversity can select across files
-                const fetchK = fileContents.length > 1 ? topK * 3 : topK;
+                // Fetch more candidates than needed so diversity can select across files and images
+                const loadedSources = fileContents.length + imageChunks.length;
+                const fetchK = loadedSources > 1 ? topK * 3 : topK;
                 const rawResults = await vectorStore.query({
                     indexName,
                     queryVector: queryEmbedding,
                     topK: fetchK,
                 });
-                // Apply source-file diversity for multi-file RAG
-                const results = fileContents.length > 1
+                // Apply source-file diversity for multi-file/multi-image RAG
+                const results = loadedSources > 1
                     ? diversifyResults(rawResults, topK)
                     : rawResults.slice(0, topK);
                 if (results.length === 0) {
@@ -335,7 +454,8 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                     };
                 }
                 const relevantContext = results
-                    .map((r, i) => `[${i + 1}] ${r.metadata?.text || r.text || ""}`)
+                    .map((r, i) => `[${i + 1}] ${r.metadata?.text || r.text || ""}` +
+                    (r.metadata?.hasImage ? " [image]" : ""))
                     .join("\n\n");
                 span.setAttribute("rag.results_count", results.length);
                 return {
@@ -345,6 +465,7 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
                         score: r.score,
                         source: r.metadata?.source,
                         text: (r.metadata?.text || r.text || "").slice(0, 200),
+                        hasImage: r.metadata?.hasImage ?? false,
                     })),
                     totalResults: results.length,
                 };
@@ -354,8 +475,8 @@ async function _prepareRAGToolInner(ragConfig, fallbackProvider) {
     return {
         tool: aiTool,
         toolName,
-        chunksIndexed: allChunks.length,
-        filesLoaded: fileContents.length,
+        chunksIndexed: allChunks.length + imageChunks.length,
+        filesLoaded: fileContents.length + imageFiles.length,
     };
 }
 /** @internal Exported for testing only */

package/dist/types/providers.d.ts

@@ -28,6 +28,18 @@ export type SupportedModelName = BedrockModels | DeepSeekModels | OpenAIModels |
  * Extract provider names from enum
  */
 export type ProviderName = (typeof AIProviderName)[keyof typeof AIProviderName];
+/**
+ * Multi-modal embedding input — accepts text, image, or both.
+ * Used by providers that support multi-modal embeddings (e.g. Bedrock Titan Image, Nova Multimodal).
+ */
+export type EmbedInput = {
+    /** Text content to embed */
+    text?: string;
+    /** Image data as Buffer or base64 string */
+    image?: Buffer | string;
+    /** MIME type of the image (e.g. "image/png", "image/jpeg") */
+    mimeType?: string;
+};
 /**
  * Provider status information
  */
@@ -741,7 +753,15 @@ export type AIProvider = {
     stream(optionsOrPrompt: StreamOptions | string, analysisSchema?: ValidationSchema): Promise<StreamResult>;
     generate(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
     gen(optionsOrPrompt: TextGenerationOptions | string, analysisSchema?: ValidationSchema): Promise<EnhancedGenerateResult | null>;
-    embed(text: string, modelName?: string): Promise<number[]>;
+    /**
+     * Generate an embedding vector for text or multi-modal input.
+     * Accepts either a plain string (text-only) or an EmbedInput object
+     * for multi-modal embeddings (text + image).
+     */
+    embed(input: string | EmbedInput, modelName?: string): Promise<number[]>;
+    /**
+     * Generate embedding vectors for multiple text inputs in batch.
+     */
     embedMany(texts: string[], modelName?: string): Promise<number[][]>;
     setupToolExecutor(sdk: {
         customTools: Map<string, unknown>;

package/dist/types/rag.d.ts

@@ -1,4 +1,5 @@
 import type { Tool } from "./tools.js";
+import type { EmbedInput } from "./providers.js";
 /**
  * RAG Type Definitions
  *
@@ -231,6 +232,8 @@ export type RAGPipelineConfig = {
     enableReranking?: boolean;
     /** Reranking model configuration */
     rerankingModel?: EmbeddingModelConfig;
+    /** Multi-modal RAG configuration (image + text embeddings) */
+    multiModal?: MultiModalRAGConfig;
 };
 /**
  * Ingestion options
@@ -419,6 +422,18 @@ export type VectorStore = {
         includeVectors?: boolean;
     }): Promise<VectorQueryResult[]>;
 };
+/**
+ * Structural extension of {@link VectorStore} for stores that can also write
+ * vectors via `upsert()` (e.g. {@link InMemoryVectorStore}). Custom stores
+ * that expose an `upsert` method satisfy this type without a cast.
+ */
+export type UpsertableVectorStore = VectorStore & {
+    upsert(indexName: string, items: Array<{
+        id: string;
+        vector: number[];
+        metadata?: Record<string, unknown>;
+    }>): Promise<void>;
+};
 /** Minimal shape of a query result, satisfied by both `pg` and `pglite`. */
 export type PgQueryResult<T = unknown> = {
     rows: T[];
@@ -1330,3 +1345,132 @@ export type DocumentState = {
 };
 /** Canonical RAG error code. */
 export type RAGErrorCode = "RAG_CHUNKING_ERROR" | "RAG_CHUNKING_INVALID_CONFIG" | "RAG_CHUNKING_STRATEGY_NOT_FOUND" | "RAG_CHUNKING_EMPTY_CONTENT" | "RAG_CHUNKING_SIZE_EXCEEDED" | "RAG_METADATA_EXTRACTION_ERROR" | "RAG_METADATA_EXTRACTION_TIMEOUT" | "RAG_METADATA_SCHEMA_INVALID" | "RAG_METADATA_EXTRACTOR_NOT_FOUND" | "RAG_EMBEDDING_ERROR" | "RAG_EMBEDDING_DIMENSION_MISMATCH" | "RAG_EMBEDDING_RATE_LIMIT" | "RAG_EMBEDDING_PROVIDER_ERROR" | "RAG_VECTOR_QUERY_ERROR" | "RAG_VECTOR_QUERY_TIMEOUT" | "RAG_VECTOR_STORE_UNAVAILABLE" | "RAG_VECTOR_STORE_CONNECTION_ERROR" | "RAG_VECTOR_INDEX_NOT_FOUND" | "RAG_RERANKER_ERROR" | "RAG_RERANKER_NOT_FOUND" | "RAG_RERANKER_API_ERROR" | "RAG_GRAPH_ERROR" | "RAG_GRAPH_TRAVERSAL_ERROR" | "RAG_GRAPH_NODE_NOT_FOUND" | "RAG_PIPELINE_ERROR" | "RAG_PIPELINE_STAGE_FAILED" | "RAG_PIPELINE_PARTIAL_FAILURE" | "RAG_CIRCUIT_BREAKER_OPEN" | "RAG_CIRCUIT_BREAKER_HALF_OPEN_LIMIT" | "RAG_OPERATION_TIMEOUT" | "RAG_RETRY_EXHAUSTED" | "RAG_INVALID_CONFIGURATION";
+/**
+ * Modality of an embedding model
+ */
+export type EmbeddingModality = "text" | "image" | "multimodal";
+/**
+ * Multi-modal embedding model configuration
+ */
+export type MultiModalEmbeddingConfig = {
+    /** Provider name (e.g. "bedrock") */
+    provider: string;
+    /** Model name (e.g. "amazon.titan-embed-image-v1") */
+    modelName: string;
+    /** What modalities this model supports */
+    modality: EmbeddingModality;
+    /** Embedding dimension (e.g. 1024 for Titan Image) */
+    dimensions?: number;
+};
+/**
+ * Image metadata attached to multi-modal chunks
+ */
+export type ImageChunkMetadata = {
+    /** Image width in pixels */
+    width?: number;
+    /** Image height in pixels */
+    height?: number;
+    /** Image MIME type */
+    mimeType?: string;
+    /** Image format (jpeg, png, webp, etc.) */
+    format?: string;
+    /** Original file path or URL */
+    source?: string;
+    /** Whether this chunk contains image data */
+    hasImage: boolean;
+};
+/**
+ * Extended chunk type for multi-modal RAG.
+ * Contains optional image data alongside text for image+text search.
+ */
+export type MultiModalChunk = Chunk & {
+    /** Image data as Buffer or base64 string (present for image chunks) */
+    image?: Buffer | string;
+    /** Image MIME type */
+    imageMimeType?: string;
+    /** Image-specific metadata */
+    imageMeta?: ImageChunkMetadata;
+};
+/**
+ * Multi-modal query input — can be text, image, or both.
+ * Same shape as EmbedInput.
+ */
+export type MultiModalQuery = EmbedInput;
+/**
+ * Multi-modal RAG pipeline configuration extension.
+ * Passed alongside RAGPipelineConfig to enable multi-modal features.
+ */
+export type MultiModalRAGConfig = {
+    /** Whether multi-modal RAG is enabled */
+    enabled: boolean;
+    /** Embedding model for multi-modal content */
+    embeddingModel: MultiModalEmbeddingConfig;
+    /** How to generate text from images for indexing: "caption" uses LLM vision, "filename" uses filename, "none" skips text */
+    imageTextStrategy?: "caption" | "filename" | "none";
+    /** Maximum image file size in bytes (default: 10MB) */
+    maxImageSize?: number;
+    /** Supported image MIME types */
+    supportedFormats?: string[];
+    /** Provider for generating image captions (used with imageTextStrategy: "caption") */
+    captionProvider?: string;
+    /** Model for generating image captions */
+    captionModel?: string;
+};
+/**
+ * Match type indicating how the result was found
+ */
+export type MultiModalMatchType = "text-to-text" | "text-to-image" | "image-to-image" | "image-to-text";
+/**
+ * Image search result from multi-modal retrieval
+ */
+export type MultiModalSearchResult = {
+    /** The chunk containing the match */
+    chunk: MultiModalChunk;
+    /** Similarity score (0-1) */
+    score: number;
+    /** How this match was found */
+    matchType: MultiModalMatchType;
+};
+/**
+ * Supported image MIME types for multi-modal RAG
+ */
+export declare const SUPPORTED_IMAGE_MIME_TYPES: readonly ["image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/tiff", "image/avif"];
+/**
+ * Image document loaded for RAG ingestion
+ */
+export type ImageDocument = {
+    /** Unique identifier for this image document */
+    id: string;
+    /** Text representation (filename, alt text, or caption) */
+    text: string;
+    /** Raw image data */
+    image: Buffer;
+    /** Image MIME type */
+    mimeType: string;
+    /** Metadata about the image */
+    metadata: {
+        /** Original file path or URL */
+        source: string;
+        /** Image width in pixels (if detectable) */
+        width?: number;
+        /** Image height in pixels (if detectable) */
+        height?: number;
+        /** Image format (jpeg, png, webp, etc.) */
+        format?: string;
+        /** File size in bytes */
+        size: number;
+    };
+};
+/**
+ * Options for loading images
+ */
+export type ImageLoaderOptions = {
+    /** Custom metadata to attach to the image document */
+    metadata?: Record<string, unknown>;
+    /** Maximum image file size in bytes (default: 10MB) */
+    maxImageSize?: number;
+    /** Timeout for URL fetches in milliseconds (default: 30000) */
+    fetchTimeout?: number;
+    /** Custom headers for URL fetches */
+    headers?: Record<string, string>;
+};

package/dist/types/rag.js

@@ -1 +1,12 @@
-export {};
+/**
+ * Supported image MIME types for multi-modal RAG
+ */
+export const SUPPORTED_IMAGE_MIME_TYPES = [
+    "image/jpeg",
+    "image/png",
+    "image/gif",
+    "image/webp",
+    "image/bmp",
+    "image/tiff",
+    "image/avif",
+];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.14.21",
+  "version": "12.15.0",
   "packageManager": "pnpm@10.15.1",
   "description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
   "author": {
@@ -92,6 +92,7 @@
     "test:media": "pnpm exec tsx test/continuous-test-suite-media-gen.ts",
     "test:media-registry-collisions": "pnpm exec tsx test/continuous-test-suite-media-registry-collisions.ts",
     "test:memory": "pnpm exec tsx test/continuous-test-suite-memory.ts",
+    "test:multimodal-rag": "pnpm exec tsx test/continuous-test-suite-multimodal-rag.ts",
     "test:proxy-sharing": "pnpm run build && pnpm exec tsx test/continuous-test-suite-proxy-sharing.ts",
     "test:openai-compat-streaming-retry": "pnpm exec tsx test/continuous-test-suite-openai-compat-streaming-retry.ts",
     "test:anthropic-streaming-retry": "pnpm exec tsx test/continuous-test-suite-anthropic-streaming-retry.ts",