@juspay/neurolink 12.14.21 → 12.15.1

package/dist/proxy/proxyAnalysis.js

@@ -1,3 +1,4 @@
+import { proxyTokenUsage } from "./proxyTokenUsage.js";
 import { createReadStream } from "node:fs";
 import { isDeepStrictEqual } from "node:util";
 import { isProxyAuxiliaryRequest } from "./proxyRequestKind.js";
@@ -411,8 +412,15 @@ function summarizeFinalRequests(finalRequests, terminalStreamErrors, attemptsByR
             request.outputTokens !== null ||
             request.cacheReadTokens !== null ||
             request.cacheCreationTokens !== null) {
+            const billingUsage = proxyTokenUsage({
+                inputIncludesCachedTokens: request.inputIncludesCachedTokens,
+                inputTokens: request.inputTokens ?? 0,
+                outputTokens: request.outputTokens ?? 0,
+                cacheCreationTokens: request.cacheCreationTokens ?? 0,
+                cacheReadTokens: request.cacheReadTokens ?? 0,
+            });
             requestsWithUsage += 1;
-            inputTokens += request.inputTokens ?? 0;
+            inputTokens += billingUsage.input;
             outputTokens += request.outputTokens ?? 0;
             cacheReadTokens += request.cacheReadTokens ?? 0;
             cacheCreationTokens += request.cacheCreationTokens ?? 0;
@@ -422,16 +430,7 @@ function summarizeFinalRequests(finalRequests, terminalStreamErrors, attemptsByR
                 // "openai-compatible" resolves to the cross-provider table search in
                 // pricing.ts, which finds the model wherever it lives — a far better
                 // guess than assuming Anthropic and pricing a GPT model at $0.
-                const cost = calculateCost(request.provider ?? "openai-compatible", request.model, {
-                    input: request.inputTokens ?? 0,
-                    output: request.outputTokens ?? 0,
-                    total: (request.inputTokens ?? 0) +
-                        (request.outputTokens ?? 0) +
-                        (request.cacheCreationTokens ?? 0) +
-                        (request.cacheReadTokens ?? 0),
-                    cacheCreationTokens: request.cacheCreationTokens ?? 0,
-                    cacheReadTokens: request.cacheReadTokens ?? 0,
-                });
+                const cost = calculateCost(request.provider ?? "openai-compatible", request.model, billingUsage);
                 // Ask the table directly rather than inferring from cost > 0: a real
                 // request with trivial usage can round to $0.000000 and is priced, not
                 // unpriced.
@@ -982,6 +981,10 @@ export async function analyzeProxyLogs(options) {
                 accountType: stringValue(record.accountType) ?? "unknown",
                 model: stringValue(record.model),
                 provider: stringValue(record.provider),
+                inputIncludesCachedTokens: typeof record.inputIncludesCachedTokens === "boolean"
+                    ? record.inputIncludesCachedTokens
+                    : typeof record.path === "string" &&
+                        record.path.includes("/backend-api/codex/"),
                 inputTokens: finiteNumber(record.inputTokens),
                 outputTokens: finiteNumber(record.outputTokens),
                 cacheReadTokens: finiteNumber(record.cacheReadTokens),

package/dist/proxy/proxyTokenUsage.d.ts

@@ -0,0 +1,7 @@
+import type { TokenUsage, UsageContext } from "../types/index.js";
+/** Convert proxy wire usage into disjoint billing buckets. Codex input already
+ * includes cache tokens; reasoning is included in output for both providers.
+ * OpenAI's documented formula subtracts both cached_tokens and cache_write_tokens:
+ * https://developers.openai.com/api/docs/guides/prompt-caching#monitor-cache-performance
+ */
+export declare function proxyTokenUsage(usage: UsageContext): TokenUsage;

package/dist/proxy/proxyTokenUsage.js

@@ -0,0 +1,21 @@
+/** Convert proxy wire usage into disjoint billing buckets. Codex input already
+ * includes cache tokens; reasoning is included in output for both providers.
+ * OpenAI's documented formula subtracts both cached_tokens and cache_write_tokens:
+ * https://developers.openai.com/api/docs/guides/prompt-caching#monitor-cache-performance
+ */
+export function proxyTokenUsage(usage) {
+    const cached = usage.cacheReadTokens + usage.cacheCreationTokens;
+    const input = usage.inputIncludesCachedTokens
+        ? Math.max(0, usage.inputTokens - cached)
+        : usage.inputTokens;
+    return {
+        input,
+        output: usage.outputTokens,
+        total: usage.inputIncludesCachedTokens
+            ? usage.inputTokens + usage.outputTokens
+            : input + cached + usage.outputTokens,
+        cacheReadTokens: usage.cacheReadTokens,
+        cacheCreationTokens: usage.cacheCreationTokens,
+        reasoning: usage.reasoningTokens,
+    };
+}

package/dist/proxy/proxyTracer.d.ts

@@ -57,6 +57,8 @@ declare class ProxyTracer {
     startStream(): Span;
     /** Record account selection outcome on the root span. */
     setAccountSelection(ctx: AccountSelectionContext): void;
+    /** Attribute fallback metrics to the account that actually served them. */
+    setServedAccount(account: string, accountType: string): void;
     /** Record token usage and cost on the root span. */
     setUsage(ctx: UsageContext): void;
     /**

package/dist/proxy/proxyTracer.js

@@ -16,6 +16,7 @@
 import { SpanStatusCode, SpanKind, ROOT_CONTEXT, isSpanContextValid, propagation, context, metrics, trace, } from "@opentelemetry/api";
 import { getTracer, setLangfuseContext, } from "../services/server/ai/observability/instrumentation.js";
 import { OtelBridge } from "../observability/otelBridge.js";
+import { proxyTokenUsage } from "./proxyTokenUsage.js";
 import { calculateCost } from "../utils/pricing.js";
 import { TelemetryService } from "../telemetry/telemetryService.js";
 import { logger } from "../utils/logger.js";
@@ -360,17 +361,22 @@ class ProxyTracer {
             // Non-fatal
         });
     }
+    /** Attribute fallback metrics to the account that actually served them. */
+    setServedAccount(account, accountType) {
+        this.accountEmail = account;
+        this.rootSpan.setAttributes({
+            "proxy.account.served": account,
+            "proxy.account.served_type": accountType,
+        });
+    }
     /** Record token usage and cost on the root span. */
     setUsage(ctx) {
         this.usage = ctx;
-        const totalTokens = ctx.inputTokens +
-            ctx.outputTokens +
-            ctx.cacheCreationTokens +
-            ctx.cacheReadTokens +
-            (ctx.reasoningTokens ?? 0);
-        // NeuroLink-format token attributes (from SpanAttributes)
+        const billingUsage = proxyTokenUsage(ctx);
+        const totalTokens = billingUsage.total;
+        // Custom token buckets are disjoint; standard GenAI input includes cache.
         this.rootSpan.setAttributes({
-            "ai.tokens.input": ctx.inputTokens,
+            "ai.tokens.input": billingUsage.input,
             "ai.tokens.output": ctx.outputTokens,
             "ai.tokens.total": totalTokens,
             "ai.tokens.cache_creation": ctx.cacheCreationTokens,
@@ -381,18 +387,12 @@ class ProxyTracer {
         }
         // GenAI semantic convention attributes (for Langfuse compatibility)
         this.rootSpan.setAttributes({
-            "gen_ai.usage.input_tokens": ctx.inputTokens,
+            "gen_ai.usage.input_tokens": totalTokens - ctx.outputTokens,
             "gen_ai.usage.output_tokens": ctx.outputTokens,
             "gen_ai.usage.total_tokens": totalTokens,
         });
         // Cost calculation via pricing.ts
-        const cost = calculateCost(this.billingProvider, this.model, {
-            input: ctx.inputTokens,
-            output: ctx.outputTokens,
-            total: totalTokens,
-            cacheCreationTokens: ctx.cacheCreationTokens,
-            cacheReadTokens: ctx.cacheReadTokens,
-        });
+        const cost = calculateCost(this.billingProvider, this.model, billingUsage);
         if (cost > 0) {
             this.rootSpan.setAttributes({
                 "ai.cost.total": cost,
@@ -665,7 +665,7 @@ class ProxyTracer {
                 model: this.model,
                 account: this.accountEmail ?? "unknown",
             };
-            m.tokensInput.add(this.usage.inputTokens, tokenLabels);
+            m.tokensInput.add(proxyTokenUsage(this.usage).input, tokenLabels);
             m.tokensOutput.add(this.usage.outputTokens, tokenLabels);
             m.tokensCacheRead.add(this.usage.cacheReadTokens, tokenLabels);
             m.tokensCacheCreation.add(this.usage.cacheCreationTokens, tokenLabels);
@@ -673,18 +673,8 @@ class ProxyTracer {
                 m.tokensReasoning.add(this.usage.reasoningTokens, tokenLabels);
             }
             // Cost
-            const totalTokens = this.usage.inputTokens +
-                this.usage.outputTokens +
-                this.usage.cacheCreationTokens +
-                this.usage.cacheReadTokens +
-                (this.usage.reasoningTokens ?? 0);
-            const cost = calculateCost(this.billingProvider, this.model, {
-                input: this.usage.inputTokens,
-                output: this.usage.outputTokens,
-                total: totalTokens,
-                cacheCreationTokens: this.usage.cacheCreationTokens,
-                cacheReadTokens: this.usage.cacheReadTokens,
-            });
+            const billingUsage = proxyTokenUsage(this.usage);
+            const cost = calculateCost(this.billingProvider, this.model, billingUsage);
             if (cost > 0) {
                 m.costTotal.add(cost, tokenLabels);
             }
@@ -711,19 +701,10 @@ class ProxyTracer {
         if (!this.usage) {
             return;
         }
-        const totalTokens = this.usage.inputTokens +
-            this.usage.outputTokens +
-            this.usage.cacheCreationTokens +
-            this.usage.cacheReadTokens +
-            (this.usage.reasoningTokens ?? 0);
+        const billingUsage = proxyTokenUsage(this.usage);
+        const totalTokens = billingUsage.total;
         const durationMs = Date.now() - this.startTime;
-        const cost = calculateCost(this.billingProvider, this.model, {
-            input: this.usage.inputTokens,
-            output: this.usage.outputTokens,
-            total: totalTokens,
-            cacheCreationTokens: this.usage.cacheCreationTokens,
-            cacheReadTokens: this.usage.cacheReadTokens,
-        });
+        const cost = calculateCost(this.billingProvider, this.model, billingUsage);
         TelemetryService.getInstance().recordAIRequest(this.billingProvider, this.model, totalTokens, durationMs, cost > 0 ? cost : undefined);
     }
     // -------------------------------------------------------------------------

package/dist/proxy/quotaHeaders.js

@@ -133,10 +133,14 @@ export function buildQuotaResponseHeaders(context, now = Date.now()) {
     }
     const quota = context.quota;
     if (quota && context.source !== "none") {
-        set("x-neurolink-quota-session-left-pct", utilizationToLeftPct(quota.sessionUsed));
+        set("x-neurolink-quota-session-left-pct", quota.sessionStatus === "unknown"
+            ? undefined
+            : utilizationToLeftPct(quota.sessionUsed));
         set("x-neurolink-quota-session-status", quota.sessionStatus);
         set("x-neurolink-quota-session-resets-in", secondsUntil(quota.sessionResetAt, now));
-        set("x-neurolink-quota-weekly-left-pct", utilizationToLeftPct(quota.weeklyUsed));
+        set("x-neurolink-quota-weekly-left-pct", quota.weeklyStatus === "unknown"
+            ? undefined
+            : utilizationToLeftPct(quota.weeklyUsed));
         set("x-neurolink-quota-weekly-status", quota.weeklyStatus);
         set("x-neurolink-quota-weekly-resets-in", secondsUntil(quota.weeklyResetAt, now));
         set("x-neurolink-quota-unified-status", quota.unifiedStatus);
@@ -173,7 +177,7 @@ export function summarizePoolHeadroom(entries, now = Date.now()) {
             continue;
         }
         available += 1;
-        if (entry.quota) {
+        if (entry.quota && entry.quota.sessionStatus !== "unknown") {
             const left = utilizationToLeftPct(entry.quota.sessionUsed);
             if (bestSessionLeftPct === undefined || left > bestSessionLeftPct) {
                 bestSessionLeftPct = left;

package/dist/rag/document/MDocument.d.ts

@@ -99,9 +99,27 @@ export declare class MDocument {
      * Generate embeddings for all chunks
      * @param provider - Embedding provider name
      * @param modelName - Embedding model name
+     * @param imageData - Optional array of image data (Buffer or base64) aligned with chunks,
+     *                    enabling multi-modal embeddings for providers that support them
+     * @param mimeType - MIME type of the images (default: "image/png"). Pass an
+     *                    array to declare a per-image MIME type aligned with
+     *                    `imageData`; a plain string applies to every image.
      * @returns This MDocument instance (for chaining)
      */
-    embed(provider?: string, modelName?: string): Promise<MDocument>;
+    embed(provider?: string, modelName?: string, imageData?: Array<Buffer | string>, mimeType?: string | string[]): Promise<MDocument>;
+    /**
+     * Generate multi-modal embeddings for all chunks using text + optional image data.
+     * Uses a provider that supports multi-modal embeddings (e.g. Bedrock Titan Image).
+     *
+     * @param provider - Embedding provider name
+     * @param modelName - Embedding model name
+     * @param imageData - Optional array of image data (Buffer or base64) aligned with chunks
+     * @param mimeType - MIME type of the images (default: "image/png"). Pass an
+     *                    array to declare a per-image MIME type aligned with
+     *                    `imageData`; a plain string applies to every image.
+     * @returns This MDocument instance (for chaining)
+     */
+    embedMultiModal(provider?: string, modelName?: string, imageData?: Array<Buffer | string>, mimeType?: string | string[]): Promise<MDocument>;
     /**
      * Get document ID
      */

package/dist/rag/document/MDocument.js

@@ -194,9 +194,14 @@ export class MDocument {
      * Generate embeddings for all chunks
      * @param provider - Embedding provider name
      * @param modelName - Embedding model name
+     * @param imageData - Optional array of image data (Buffer or base64) aligned with chunks,
+     *                    enabling multi-modal embeddings for providers that support them
+     * @param mimeType - MIME type of the images (default: "image/png"). Pass an
+     *                    array to declare a per-image MIME type aligned with
+     *                    `imageData`; a plain string applies to every image.
      * @returns This MDocument instance (for chaining)
      */
-    async embed(provider = "openai", modelName = "text-embedding-3-small") {
+    async embed(provider = "openai", modelName = "text-embedding-3-small", imageData, mimeType = "image/png") {
         if (this.state.chunks.length === 0) {
             logger.warn("[MDocument] No chunks to embed. Call chunk() first.");
             return this;
@@ -208,14 +213,21 @@ export class MDocument {
             chunkCount: this.state.chunks.length,
             provider,
             model: modelName,
+            hasImages: imageData !== undefined && imageData.length > 0,
         });
         const embeddingProvider = await ProviderFactory.createProvider(provider, modelName);
         if (typeof embeddingProvider.embed !== "function") {
             throw new Error(`Provider ${provider} does not support embeddings`);
         }
         this.state.embeddings = [];
-        for (const chunk of this.state.chunks) {
-            const embedding = await embeddingProvider.embed(chunk.text);
+        for (let i = 0; i < this.state.chunks.length; i++) {
+            const chunk = this.state.chunks[i];
+            const image = imageData?.[i];
+            const imageMimeType = typeof mimeType === "string" ? mimeType : (mimeType[i] ?? "image/png");
+            const input = image
+                ? { text: chunk.text, image, mimeType: imageMimeType }
+                : chunk.text;
+            const embedding = await embeddingProvider.embed(input);
             this.state.embeddings.push(embedding);
             chunk.embedding = embedding;
         }
@@ -227,6 +239,21 @@ export class MDocument {
         });
         return this;
     }
+    /**
+     * Generate multi-modal embeddings for all chunks using text + optional image data.
+     * Uses a provider that supports multi-modal embeddings (e.g. Bedrock Titan Image).
+     *
+     * @param provider - Embedding provider name
+     * @param modelName - Embedding model name
+     * @param imageData - Optional array of image data (Buffer or base64) aligned with chunks
+     * @param mimeType - MIME type of the images (default: "image/png"). Pass an
+     *                    array to declare a per-image MIME type aligned with
+     *                    `imageData`; a plain string applies to every image.
+     * @returns This MDocument instance (for chaining)
+     */
+    async embedMultiModal(provider = "bedrock", modelName = "amazon.titan-embed-image-v1", imageData, mimeType = "image/png") {
+        return this.embed(provider, modelName, imageData, mimeType);
+    }
     // ============================================================================
     // Accessor Methods
     // ============================================================================

package/dist/rag/document/imageLoader.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Image Loader for Multi-Modal RAG
+ *
+ * Loads image files (from disk or URLs) into ImageDocument objects
+ * that can be embedded and stored in a vector store alongside text chunks.
+ *
+ * @example
+ * ```typescript
+ * import { ImageLoader } from './imageLoader.js';
+ *
+ * const loader = new ImageLoader();
+ *
+ * // Load from file path
+ * const doc = await loader.load('./photo.jpg');
+ *
+ * // Load from URL
+ * const doc2 = await loader.load('https://example.com/image.png');
+ *
+ * // Load from buffer
+ * const doc3 = loader.loadFromBuffer(imageBuffer, 'image/jpeg');
+ * ```
+ */
+import type { ImageDocument, ImageLoaderOptions } from "../../types/index.js";
+/**
+ * ImageLoader for multi-modal RAG.
+ * Loads images from file paths, URLs, or buffers into ImageDocument objects.
+ */
+export declare class ImageLoader {
+    private options;
+    constructor(options?: ImageLoaderOptions);
+    /**
+     * Load an image from a file path or URL.
+     */
+    load(source: string, options?: Partial<ImageLoaderOptions>): Promise<ImageDocument>;
+    /**
+     * Load an image from a local file path.
+     */
+    private loadFromPath;
+    /**
+     * Load an image from a URL.
+     */
+    private loadFromURL;
+    /**
+     * Load an image directly from a Buffer.
+     */
+    loadFromBuffer(buffer: Buffer, mimeType: string, source?: string, options?: Partial<ImageLoaderOptions>): ImageDocument;
+    /**
+     * Check if a file path or extension is a supported image format.
+     */
+    static isImageFile(source: string): boolean;
+    /**
+     * Get the MIME type for a file extension.
+     */
+    static getMimeType(filePath: string): string | undefined;
+}

package/dist/rag/document/imageLoader.js

@@ -0,0 +1,183 @@
+/**
+ * Image Loader for Multi-Modal RAG
+ *
+ * Loads image files (from disk or URLs) into ImageDocument objects
+ * that can be embedded and stored in a vector store alongside text chunks.
+ *
+ * @example
+ * ```typescript
+ * import { ImageLoader } from './imageLoader.js';
+ *
+ * const loader = new ImageLoader();
+ *
+ * // Load from file path
+ * const doc = await loader.load('./photo.jpg');
+ *
+ * // Load from URL
+ * const doc2 = await loader.load('https://example.com/image.png');
+ *
+ * // Load from buffer
+ * const doc3 = loader.loadFromBuffer(imageBuffer, 'image/jpeg');
+ * ```
+ */
+import { readFileSync, existsSync, statSync } from "fs";
+import { extname } from "path";
+import { logger } from "../../utils/logger.js";
+import { safeDownload } from "../../utils/safeFetch.js";
+import { redactUrlForError } from "../../utils/logSanitize.js";
+import { ImageProcessor, VALID_IMAGE_EXTENSIONS, } from "../../utils/imageProcessor.js";
+import { EXTENSION_MIME_MAP } from "../../processors/config/index.js";
+/**
+ * Generate a text representation from the image source.
+ */
+function generateTextDescription(source) {
+    // Drop any query string or fragment before deriving the name. On a presigned
+    // URL the signature lives in the query, and the query is part of the final
+    // slash-separated segment — so `.pop()` alone keeps it, and this caption is
+    // not a transient log line: it becomes embedded, BM25-indexed chunk text that
+    // persists in the vector store and is read back into model context.
+    const withoutQueryOrFragment = source.split(/[?#]/)[0];
+    const filename = withoutQueryOrFragment.split("/").pop() ?? source;
+    // Remove extension and clean up
+    const name = filename
+        .replace(/\.[^.]+$/, "")
+        .replace(/[-_]/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    return `Image: ${name}`;
+}
+/**
+ * ImageLoader for multi-modal RAG.
+ * Loads images from file paths, URLs, or buffers into ImageDocument objects.
+ */
+export class ImageLoader {
+    options;
+    constructor(options) {
+        this.options = {
+            metadata: options?.metadata ?? {},
+            maxImageSize: options?.maxImageSize ?? 10 * 1024 * 1024, // 10MB
+            fetchTimeout: options?.fetchTimeout ?? 30_000,
+            headers: options?.headers ?? {},
+        };
+    }
+    /**
+     * Load an image from a file path or URL.
+     */
+    async load(source, options) {
+        const opts = { ...this.options, ...options };
+        if (source.startsWith("http://") || source.startsWith("https://")) {
+            return this.loadFromURL(source, opts);
+        }
+        return this.loadFromPath(source, opts);
+    }
+    /**
+     * Load an image from a local file path.
+     */
+    async loadFromPath(filePath, options) {
+        if (!existsSync(filePath)) {
+            throw new Error(`Image file not found: ${filePath}`);
+        }
+        const { size } = statSync(filePath);
+        if (size > options.maxImageSize) {
+            throw new Error(`Image size (${size} bytes) exceeds maximum (${options.maxImageSize} bytes)`);
+        }
+        const buffer = readFileSync(filePath);
+        const ext = extname(filePath).toLowerCase();
+        const mimeType = EXTENSION_MIME_MAP[ext] ?? ImageProcessor.detectImageType(buffer);
+        const dimensions = ImageProcessor.getImageDimensions(buffer);
+        logger.debug("[ImageLoader] Loaded image from path", {
+            path: filePath,
+            size: buffer.byteLength,
+            mimeType,
+            dimensions,
+        });
+        return {
+            id: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+            text: generateTextDescription(filePath),
+            image: buffer,
+            mimeType,
+            metadata: {
+                source: filePath,
+                width: dimensions?.width,
+                height: dimensions?.height,
+                format: ext.replace(".", ""),
+                size: buffer.byteLength,
+                ...options.metadata,
+            },
+        };
+    }
+    /**
+     * Load an image from a URL.
+     */
+    async loadFromURL(url, options) {
+        const buffer = await safeDownload(url, {
+            maxBytes: options.maxImageSize,
+            label: "RAG image",
+            timeoutMs: options.fetchTimeout,
+        });
+        const mimeType = ImageProcessor.detectImageType(buffer);
+        const dimensions = ImageProcessor.getImageDimensions(buffer);
+        logger.debug("[ImageLoader] Loaded image from URL", {
+            url: redactUrlForError(url),
+            size: buffer.byteLength,
+            mimeType,
+            dimensions,
+        });
+        return {
+            id: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+            text: generateTextDescription(url),
+            image: buffer,
+            mimeType,
+            metadata: {
+                // Redacted for the same reason as the caption above: `source` is
+                // persisted into the vector store and returned verbatim to the model
+                // as the RAG tool's citation, so a signed URL stored here outlives the
+                // request that fetched it.
+                source: redactUrlForError(url),
+                width: dimensions?.width,
+                height: dimensions?.height,
+                format: mimeType.split("/").at(1),
+                size: buffer.byteLength,
+                ...options.metadata,
+            },
+        };
+    }
+    /**
+     * Load an image directly from a Buffer.
+     */
+    loadFromBuffer(buffer, mimeType, source, options) {
+        const opts = { ...this.options, ...options };
+        if (buffer.byteLength > opts.maxImageSize) {
+            throw new Error(`Image size (${buffer.byteLength} bytes) exceeds maximum (${opts.maxImageSize} bytes)`);
+        }
+        const dimensions = ImageProcessor.getImageDimensions(buffer);
+        return {
+            id: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+            text: generateTextDescription(source ?? "inline-image"),
+            image: buffer,
+            mimeType,
+            metadata: {
+                source: source ?? "inline",
+                width: dimensions?.width,
+                height: dimensions?.height,
+                format: mimeType.split("/").at(1),
+                size: buffer.byteLength,
+                ...opts.metadata,
+            },
+        };
+    }
+    /**
+     * Check if a file path or extension is a supported image format.
+     */
+    static isImageFile(source) {
+        const ext = extname(source).toLowerCase().replace(/^\./, "");
+        return VALID_IMAGE_EXTENSIONS.includes(ext);
+    }
+    /**
+     * Get the MIME type for a file extension.
+     */
+    static getMimeType(filePath) {
+        const ext = extname(filePath).toLowerCase();
+        return EXTENSION_MIME_MAP[ext];
+    }
+}

package/dist/rag/index.d.ts

@@ -48,6 +48,7 @@ export { ChunkerRegistry as ChunkerRegistryV2, chunkerRegistry, getAvailableChun
 export * from "./chunkers/index.js";
 export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
 export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
+export { ImageLoader } from "./document/imageLoader.js";
 export * from "./errors/index.js";
 export { GraphRAG } from "./graphRag/index.js";
 export { createMetadataExtractor, extractMetadata, getAvailableExtractors, getAvailableExtractorTypes, getExtractor, getExtractorDefaultConfig, getExtractorMetadata, getRegisteredExtractorMetadata, LLMMetadataExtractor, MetadataExtractorFactory, MetadataExtractorRegistry, metadataExtractorFactory, metadataExtractorRegistry, } from "./metadata/index.js";

package/dist/rag/index.js

@@ -52,6 +52,8 @@ export * from "./chunkers/index.js";
 export { CharacterChunker, ChunkerRegistry, chunkText, HTMLChunker, JSONChunker, LaTeXChunker, MarkdownChunker, RecursiveChunker, SemanticChunker, SentenceChunker, TokenChunker, } from "./chunking/index.js";
 // Document Processing
 export { CSVLoader, HTMLLoader, JSONLoader, loadDocument, loadDocuments, MarkdownLoader, MDocument, PDFLoader, TextLoader, WebLoader, } from "./document/index.js";
+// Image Loader for multi-modal RAG
+export { ImageLoader } from "./document/imageLoader.js";
 // Error handling
 export * from "./errors/index.js";
 // Graph RAG

package/dist/rag/pipeline/RAGPipeline.d.ts

@@ -25,7 +25,7 @@
  * console.log(response.answer, response.sources);
  * ```
  */
-import type { RAGPipelineConfig, IngestOptions, QueryOptions, RAGResponse, PipelineStats } from "../../types/index.js";
+import type { RAGPipelineConfig, IngestOptions, QueryOptions, RAGResponse, PipelineStats, MultiModalRAGConfig, MultiModalQuery, MultiModalSearchResult } from "../../types/index.js";
 import { MDocument } from "../document/MDocument.js";
 export declare class RAGPipeline {
     private id;
@@ -34,10 +34,21 @@ export declare class RAGPipeline {
     private bm25Index;
     private graphRAG;
     private embeddingProvider?;
+    private multiModalEmbeddingProvider?;
     private generationProvider?;
+    /**
+     * Vision provider for image captioning, when `multiModal.captionProvider` /
+     * `captionModel` name one. Undefined means captioning uses
+     * {@link generationProvider} — captioning has always run on that provider,
+     * and these two fields were declared but read nowhere, so a caller who set
+     * them got the pipeline's generation model with no indication otherwise.
+     */
+    private captionProvider?;
     private hybridSearch?;
     private documents;
     private allChunks;
+    private multiModalConfig?;
+    private imageLoader;
     constructor(config: RAGPipelineConfig);
     /**
      * Initialize the pipeline (lazy loading of providers)
@@ -73,6 +84,47 @@ export declare class RAGPipeline {
      * Clear all indexed data
      */
     clear(): Promise<void>;
+    /**
+     * Ingest images into the pipeline for multi-modal RAG.
+     * Loads images, generates embeddings via the configured multi-modal provider,
+     * and stores them alongside text chunks in the vector store.
+     *
+     * @param sources - Array of image file paths, URLs, or Buffer objects
+     * @param options - Ingestion options
+     */
+    ingestImages(sources: Array<string | Buffer>, options?: IngestOptions): Promise<{
+        imagesProcessed: number;
+        chunksCreated: number;
+    }>;
+    /**
+     * Query the pipeline with multi-modal input (text, image, or both).
+     *
+     * @param query - Text, image, or combined query
+     * @param options - Query options
+     * @returns Array of multi-modal search results
+     */
+    queryMultiModal(query: MultiModalQuery, options?: QueryOptions): Promise<MultiModalSearchResult[]>;
+    /**
+     * Get multi-modal pipeline statistics
+     */
+    getMultiModalStats(): {
+        totalImages: number;
+        totalTextChunks: number;
+        multiModalEnabled: boolean;
+    };
+    /**
+     * Generate a multi-modal embedding for text, image, or both.
+     * Uses the configured multi-modal embedding provider (e.g. Bedrock Titan Image).
+     */
+    private generateMultiModalEmbedding;
+    /**
+     * Generate a caption for an image using the generation provider's vision model.
+     */
+    private generateImageCaption;
+    /**
+     * Diversify multi-modal results to ensure a mix of text and image results.
+     */
+    private diversifyMultiModalResults;
     /**
      * Ensure pipeline is initialized
      */
@@ -102,4 +154,5 @@ export declare function createRAGPipeline(options: {
     generationModel?: string;
     enableHybrid?: boolean;
     enableGraph?: boolean;
+    multiModal?: MultiModalRAGConfig;
 }): RAGPipeline;