@juspay/neurolink 9.95.2 → 9.95.3

package/dist/utils/fileDetector.js

@@ -1413,6 +1413,41 @@ export class FileDetector {
         const timeout = options?.timeout || FileDetector.DEFAULT_NETWORK_TIMEOUT;
         const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES;
         const retryDelay = options?.retryDelay ?? DEFAULT_RETRY_DELAY;
+        // #317: pre-flight HEAD to reject an oversized file BEFORE downloading any
+        // body. content-length is advisory (chunked responses omit it), so a
+        // missing/invalid header — or a server that refuses HEAD — falls through to
+        // the streaming byte guard below; only a genuine oversize rejection stops
+        // the GET from ever running.
+        //
+        // #323: skip the pre-flight entirely when this exact URL was recently seen
+        // (its Content-Type is still cached, whether from a prior loadFromURL GET
+        // or a MimeTypeStrategy HEAD) — issuing a fresh HEAD here would defeat the
+        // whole point of that cache. The streaming byte guard in the GET below
+        // still enforces maxSize even without a pre-flight, so this doesn't remove
+        // the oversize protection — it only skips the redundant round-trip for a
+        // URL we've already been talking to within the last 60s.
+        if (getCachedUrlContentType(url, Date.now()) === undefined) {
+            try {
+                const head = await request(url, {
+                    dispatcher: getGlobalDispatcher().compose(interceptors.redirect({ maxRedirections: 5 })),
+                    method: "HEAD",
+                    headersTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
+                    bodyTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
+                });
+                // Drain/close the (empty) HEAD body so the connection can be reused.
+                await head.body.dump();
+                const declaredLength = Number(head.headers["content-length"]);
+                if (Number.isFinite(declaredLength) && declaredLength > maxSize) {
+                    throw new Error(`File too large: ${formatFileSize(declaredLength)} (max: ${formatFileSize(maxSize)})`);
+                }
+            }
+            catch (error) {
+                if (error instanceof Error && /File too large/.test(error.message)) {
+                    throw error;
+                }
+                logger.debug(`[FileDetector] HEAD pre-flight skipped for ${url}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
         return withRetry(async () => {
             try {
                 const response = await request(url, {

package/dist/utils/messageBuilder.js

@@ -1026,6 +1026,23 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
             throw error;
         }
     }
+    // #309: enforce the provider's page/size ceilings across ALL PDFs, not just
+    // per-file. N files each just under the single-file limit can still blow past
+    // it in aggregate (e.g. three 40-page PDFs → 120 pages for a 100-page API).
+    const aggregateConfig = PDFProcessor.getProviderConfig(provider);
+    if (aggregateConfig && pdfFiles.length > 1) {
+        const totalPages = pdfFiles.reduce((sum, f) => sum + (f.pageCount ?? 0), 0);
+        const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
+        if (totalPages > aggregateConfig.maxPages) {
+            throw new Error(`[PDF] Combined page count across ${pdfFiles.length} PDFs (${totalPages}) exceeds the ` +
+                `${aggregateConfig.maxPages}-page limit for ${provider}. ` +
+                `Split the request or reduce the number of PDFs.`);
+        }
+        if (totalMB > aggregateConfig.maxSizeMB) {
+            throw new Error(`[PDF] Combined size across ${pdfFiles.length} PDFs (${totalMB.toFixed(2)}MB) exceeds the ` +
+                `${aggregateConfig.maxSizeMB}MB limit for ${provider}.`);
+        }
+    }
     return pdfFiles;
 }
 /**

package/dist/utils/pdfProcessor.d.ts

@@ -7,7 +7,7 @@
  *
  * The conversion uses pdf-to-img package (MuPDF-based) for high-quality conversion.
  */
-import type { FileProcessingResult, PDFProcessorOptions, PDFProviderConfig, PDFImageConversionOptions, PDFImageConversionResult } from "../types/index.js";
+import type { FileProcessingResult, PDFProcessorOptions, PDFProviderConfig, PDFImageConversionOptions, PDFImageConversionResult, PDFImagePage } from "../types/index.js";
 export declare class PDFProcessor {
     private static readonly PDF_SIGNATURE;
     static process(content: Buffer, options?: PDFProcessorOptions): Promise<FileProcessingResult>;
@@ -20,6 +20,15 @@ export declare class PDFProcessor {
     static getProviderConfig(provider: string): PDFProviderConfig | null;
     private static isValidPDF;
     private static extractBasicMetadata;
+    /**
+     * Accurate page count via pdf-parse (pdfjs) (#287). The header regex in
+     * `extractBasicMetadata` only sees plaintext `/Type /Page` markers and misses
+     * compressed/object-stream PDFs (and miscounts when a page dict spans a chunk
+     * boundary). This parses the document properly, bounded by a timeout so a
+     * pathological PDF can't block; returns null on timeout/failure so the caller
+     * falls back to the regex estimate.
+     */
+    private static getAccuratePageCount;
     static estimateTokens(pageCount: number, mode?: "text-only" | "visual"): number;
     /**
      * Estimate the largest page's rendered pixel count from the PDF's MediaBox
@@ -61,6 +70,20 @@ export declare class PDFProcessor {
      * ```
      */
     static convertToImages(pdfBuffer: Buffer, options?: PDFImageConversionOptions): Promise<PDFImageConversionResult>;
+    /**
+     * Shared input validation for the image-conversion paths. Returns the PDF
+     * size in MB (needed by the memory-estimate log). Throws on any invalid input
+     * with the same messages the batch path has always used.
+     */
+    private static validateImageConversionInput;
+    /**
+     * Streaming variant of {@link convertToImages} (#302): yields each page's
+     * base64 PNG as soon as it renders instead of buffering the whole document,
+     * and reports progress via `options.onProgress`. A page that fails to render
+     * is yielded with `error` set (per-page isolation, #294) rather than aborting
+     * the stream. `convertToImages` is the batch wrapper over this contract.
+     */
+    static convertToImagesStream(pdfBuffer: Buffer, options?: PDFImageConversionOptions): AsyncGenerator<PDFImagePage, void, void>;
     /**
      * Convert a PDF file path to an array of base64 PNG images
      *

package/dist/utils/pdfProcessor.js

@@ -150,6 +150,13 @@ export class PDFProcessor {
             throw new Error(`PDF size ${sizeMB.toFixed(2)}MB exceeds ${config.maxSizeMB}MB limit for ${provider}`);
         }
         const metadata = PDFProcessor.extractBasicMetadata(content);
+        // #287: prefer an accurate page count (pdf-parse/pdfjs) over the unreliable
+        // header regex for both limit enforcement and returned metadata; fall back
+        // to the regex estimate when the accurate probe times out or fails.
+        const accuratePages = await PDFProcessor.getAccuratePageCount(content);
+        if (accuratePages !== null) {
+            metadata.estimatedPages = accuratePages;
+        }
         if (metadata.estimatedPages && metadata.estimatedPages > config.maxPages) {
             const enforceLimits = options?.enforceLimits !== false;
             if (enforceLimits) {
@@ -185,6 +192,10 @@ export class PDFProcessor {
                 ...metadata,
                 provider,
                 apiType: config.apiType,
+                // #349: surface the provider's citations requirement so downstream
+                // provider adapters (e.g. Bedrock Converse document blocks) can act on
+                // it, instead of the config field being read nowhere.
+                requiresCitations: config.requiresCitations,
             },
         };
     }
@@ -220,6 +231,35 @@ export class PDFProcessor {
             filename: undefined,
         };
     }
+    /**
+     * Accurate page count via pdf-parse (pdfjs) (#287). The header regex in
+     * `extractBasicMetadata` only sees plaintext `/Type /Page` markers and misses
+     * compressed/object-stream PDFs (and miscounts when a page dict spans a chunk
+     * boundary). This parses the document properly, bounded by a timeout so a
+     * pathological PDF can't block; returns null on timeout/failure so the caller
+     * falls back to the regex estimate.
+     */
+    static async getAccuratePageCount(buffer) {
+        try {
+            const { PDFParse } = await import("pdf-parse");
+            const pdf = new PDFParse({ data: new Uint8Array(buffer) });
+            try {
+                const info = await Promise.race([
+                    pdf.getInfo(),
+                    new Promise((_, reject) => setTimeout(() => reject(new Error("page-count timeout")), PDF_LIMITS.PAGE_COUNT_TIMEOUT_MS)),
+                ]);
+                const total = info.total;
+                return typeof total === "number" && total > 0 ? total : null;
+            }
+            finally {
+                await pdf.destroy?.();
+            }
+        }
+        catch (error) {
+            logger.debug(`[PDF] Accurate page count unavailable; using regex estimate: ${error instanceof Error ? error.message : String(error)}`);
+            return null;
+        }
+    }
     static estimateTokens(pageCount, mode = "visual") {
         if (mode === "text-only") {
             return Math.ceil((pageCount / 3) * 1000);
@@ -291,48 +331,32 @@ export class PDFProcessor {
      */
     static async convertToImages(pdfBuffer, options) {
         const startTime = Date.now();
-        const { scale = 2, maxPages = PDF_LIMITS.DEFAULT_MAX_PAGES, format = "png", maxCanvasPixels = PDF_LIMITS.DEFAULT_MAX_CANVAS_PIXELS, password, } = options || {};
+        const { scale = PDF_LIMITS.DEFAULT_SCALE, maxPages = PDF_LIMITS.DEFAULT_MAX_PAGES, format = "png", maxCanvasPixels = PDF_LIMITS.DEFAULT_MAX_CANVAS_PIXELS, password, onProgress, } = options || {};
         const images = [];
         const warnings = [];
-        // ============================================================================
-        // INPUT VALIDATION (Security: Prevent malformed/malicious PDF processing)
-        // ============================================================================
-        // 0. Validate format is supported and case-sensitive
-        if (format !== "png") {
-            throw new Error(`Invalid format: "${format}". Only "png" format is currently supported.`);
-        }
-        // 0b. Validate the render scale. A non-finite or non-positive scale yields a
-        // degenerate viewport (blank/zero-dimension render), and an excessive scale
-        // can allocate hundreds of MB per page — reject both with a clear message.
-        if (!Number.isFinite(scale) || scale <= 0 || scale > PDF_LIMITS.MAX_SCALE) {
-            throw new Error(`Invalid scale: ${scale}. Scale must be a finite number greater than 0 and at most ${PDF_LIMITS.MAX_SCALE}.`);
-        }
-        // 0c. Validate the per-page pixel ceiling (#260). A non-finite or
-        // non-positive value would disable the guard or yield a NaN downscale.
-        if (!Number.isFinite(maxCanvasPixels) || maxCanvasPixels <= 0) {
-            throw new Error(`Invalid maxCanvasPixels: ${maxCanvasPixels}. Must be a finite number greater than 0.`);
-        }
-        // 1. Validate buffer is not empty or too small
-        if (!pdfBuffer || pdfBuffer.length < 5) {
-            throw new Error("Invalid PDF: Buffer is too small or empty. " +
-                "A valid PDF must be at least 5 bytes (PDF header).");
-        }
-        // 2. Validate PDF magic bytes (%PDF-)
-        if (!PDFProcessor.isValidPDF(pdfBuffer)) {
-            throw new Error("Invalid PDF: File must start with %PDF- header. " +
-                "The provided buffer does not appear to be a valid PDF file.");
-        }
-        // 3. Validate maximum buffer size to prevent memory exhaustion
-        const sizeMB = pdfBuffer.length / (1024 * 1024);
-        if (sizeMB > PDF_LIMITS.MAX_SIZE_MB) {
-            throw new Error(`PDF too large for image conversion: ${sizeMB.toFixed(2)}MB exceeds ${PDF_LIMITS.MAX_SIZE_MB}MB limit. ` +
-                "Consider splitting the PDF or using a provider with native PDF support.");
-        }
+        const pageErrors = [];
+        // Validation shared with convertToImagesStream (#302).
+        const sizeMB = PDFProcessor.validateImageConversionInput(pdfBuffer, {
+            format,
+            scale,
+            maxCanvasPixels,
+        });
         logger.debug("[PDF→Image] ✅ PDF validation passed", {
             bufferSize: pdfBuffer.length,
             sizeMB: sizeMB.toFixed(2),
             maxPages,
         });
+        // #297: surface an up-front memory estimate so an operator can spot a
+        // conversion that will allocate a lot before it runs. Uses the cheap regex
+        // page estimate (the accurate pdf-parse count runs in process(), not here).
+        const estimatedPages = Math.max(1, PDFProcessor.extractBasicMetadata(pdfBuffer).estimatedPages ?? 1);
+        const estimatedMemoryMB = PDFProcessor.estimateConversionMemoryUsage(pdfBuffer.length, estimatedPages, scale);
+        logger.info("[PDF→Image] Estimated memory usage before conversion", {
+            scale,
+            estimatedPages,
+            estimatedMemoryMB,
+            sizeMB: Number(sizeMB.toFixed(2)),
+        });
         try {
             // Dynamic import to avoid loading MuPDF binaries until needed
             const pdfToImgModule = await import("pdf-to-img");
@@ -369,35 +393,60 @@ export class PDFProcessor {
                 logger.warn(`[PDF→Image] ⚠️ ${msg}`);
                 warnings.push(msg);
             }
-            // Create PDF document iterator (password forwarded for encrypted PDFs, #258)
+            // Create PDF document (password forwarded for encrypted PDFs, #258).
+            // pdf-to-img resolves `.length` (numPages) synchronously here and exposes
+            // `.getPage(n)`, so we drive an indexed loop rather than the async
+            // iterator — that lets one bad page be isolated instead of aborting all.
             const document = await pdf(pdfBuffer, {
                 scale: effectiveScale,
                 ...(password ? { password } : {}),
             });
-            let pageIndex = 0;
-            // Iterate through pages and convert to base64
-            for await (const page of document) {
-                // Check if we've reached the max pages limit
-                if (maxPages !== undefined && pageIndex >= maxPages) {
-                    warnings.push(`Stopped at page ${pageIndex} (maxPages limit: ${maxPages})`);
+            const totalPages = document.length;
+            // #294: convert page-by-page with per-page isolation. A single page whose
+            // canvas render throws (e.g. a degenerate MediaBox) no longer discards
+            // every already-converted page — it is recorded in `errors` and the rest
+            // continue.
+            for (let pageNum = 1; pageNum <= totalPages; pageNum++) {
+                if (maxPages !== undefined && pageNum - 1 >= maxPages) {
+                    warnings.push(`Stopped at page ${pageNum - 1} (maxPages limit: ${maxPages})`);
                     break;
                 }
-                // Convert PNG buffer to base64
-                const base64Image = page.toString("base64");
-                images.push(base64Image);
-                pageIndex++;
-                logger.debug(`[PDF→Image] Converted page ${pageIndex}`, {
-                    imageSizeBytes: page.length,
-                    base64Length: base64Image.length,
-                });
+                try {
+                    const page = await document.getPage(pageNum);
+                    const base64Image = page.toString("base64");
+                    images.push(base64Image);
+                    logger.debug(`[PDF→Image] Converted page ${pageNum}`, {
+                        imageSizeBytes: page.length,
+                        base64Length: base64Image.length,
+                    });
+                    // #302: report progress after each successfully converted page.
+                    if (onProgress) {
+                        await onProgress({
+                            pagesConverted: images.length,
+                            totalPages,
+                            elapsedMs: Date.now() - startTime,
+                        });
+                    }
+                }
+                catch (pageError) {
+                    const msg = pageError instanceof Error ? pageError.message : String(pageError);
+                    logger.warn(`[PDF→Image] ⚠️ page ${pageNum} failed to render: ${msg}`);
+                    pageErrors.push({ page: pageNum, error: msg });
+                }
             }
-            // Check for empty PDF (0 pages)
+            // Empty PDF (0 pages) or every page failed → treat as a conversion
+            // failure (kept inside the try so the password/format mapping below still
+            // applies), otherwise return the pages that did convert.
             if (images.length === 0) {
+                if (pageErrors.length > 0) {
+                    throw new Error(`All ${pageErrors.length} page(s) failed to render. First error: ${pageErrors[0].error}`);
+                }
                 throw new Error("PDF has 0 pages. Cannot convert empty PDF to images.");
             }
             const conversionTimeMs = Date.now() - startTime;
             logger.info("[PDF→Image] ✅ PDF conversion completed", {
                 pageCount: images.length,
+                failedPages: pageErrors.length,
                 conversionTimeMs,
                 totalImageBytes: images.reduce((sum, img) => sum + img.length, 0),
             });
@@ -406,6 +455,7 @@ export class PDFProcessor {
                 pageCount: images.length,
                 conversionTimeMs,
                 warnings: warnings.length > 0 ? warnings : undefined,
+                errors: pageErrors.length > 0 ? pageErrors : undefined,
             };
         }
         catch (error) {
@@ -431,6 +481,95 @@ export class PDFProcessor {
             });
         }
     }
+    /**
+     * Shared input validation for the image-conversion paths. Returns the PDF
+     * size in MB (needed by the memory-estimate log). Throws on any invalid input
+     * with the same messages the batch path has always used.
+     */
+    static validateImageConversionInput(pdfBuffer, opts) {
+        if (opts.format !== "png") {
+            throw new Error(`Invalid format: "${opts.format}". Only "png" format is currently supported.`);
+        }
+        if (!Number.isFinite(opts.scale) ||
+            opts.scale < PDF_LIMITS.MIN_SCALE ||
+            opts.scale > PDF_LIMITS.MAX_SCALE) {
+            throw new Error(`Invalid scale: ${opts.scale}. Scale must be a finite number between ${PDF_LIMITS.MIN_SCALE} and ${PDF_LIMITS.MAX_SCALE}.`);
+        }
+        if (!Number.isFinite(opts.maxCanvasPixels) || opts.maxCanvasPixels <= 0) {
+            throw new Error(`Invalid maxCanvasPixels: ${opts.maxCanvasPixels}. Must be a finite number greater than 0.`);
+        }
+        if (!pdfBuffer || pdfBuffer.length < 5) {
+            throw new Error("Invalid PDF: Buffer is too small or empty. " +
+                "A valid PDF must be at least 5 bytes (PDF header).");
+        }
+        if (!PDFProcessor.isValidPDF(pdfBuffer)) {
+            throw new Error("Invalid PDF: File must start with %PDF- header. " +
+                "The provided buffer does not appear to be a valid PDF file.");
+        }
+        const sizeMB = pdfBuffer.length / (1024 * 1024);
+        if (sizeMB > PDF_LIMITS.MAX_SIZE_MB) {
+            throw new Error(`PDF too large for image conversion: ${sizeMB.toFixed(2)}MB exceeds ${PDF_LIMITS.MAX_SIZE_MB}MB limit. ` +
+                "Consider splitting the PDF or using a provider with native PDF support.");
+        }
+        return sizeMB;
+    }
+    /**
+     * Streaming variant of {@link convertToImages} (#302): yields each page's
+     * base64 PNG as soon as it renders instead of buffering the whole document,
+     * and reports progress via `options.onProgress`. A page that fails to render
+     * is yielded with `error` set (per-page isolation, #294) rather than aborting
+     * the stream. `convertToImages` is the batch wrapper over this contract.
+     */
+    static async *convertToImagesStream(pdfBuffer, options) {
+        const startTime = Date.now();
+        const { scale = PDF_LIMITS.DEFAULT_SCALE, maxPages = PDF_LIMITS.DEFAULT_MAX_PAGES, format = "png", maxCanvasPixels = PDF_LIMITS.DEFAULT_MAX_CANVAS_PIXELS, password, onProgress, } = options || {};
+        PDFProcessor.validateImageConversionInput(pdfBuffer, {
+            format,
+            scale,
+            maxCanvasPixels,
+        });
+        const pdfToImgModule = await import("pdf-to-img");
+        const pdf = pdfToImgModule.pdf;
+        // #260: uniform downscale so the largest page stays under maxCanvasPixels.
+        let effectiveScale = scale;
+        const largestPixels = PDFProcessor.largestPagePixels(pdfBuffer, scale);
+        if (largestPixels > maxCanvasPixels) {
+            effectiveScale = scale * Math.sqrt(maxCanvasPixels / largestPixels);
+        }
+        const document = await pdf(pdfBuffer, {
+            scale: effectiveScale,
+            ...(password ? { password } : {}),
+        });
+        const totalPages = document.length;
+        let converted = 0;
+        for (let pageNum = 1; pageNum <= totalPages; pageNum++) {
+            if (maxPages !== undefined && pageNum - 1 >= maxPages) {
+                break;
+            }
+            try {
+                const page = await document.getPage(pageNum);
+                const base64Image = page.toString("base64");
+                converted++;
+                if (onProgress) {
+                    await onProgress({
+                        pagesConverted: converted,
+                        totalPages,
+                        elapsedMs: Date.now() - startTime,
+                    });
+                }
+                yield {
+                    pageIndex: pageNum,
+                    image: base64Image,
+                    imageSizeBytes: page.length,
+                };
+            }
+            catch (pageError) {
+                const msg = pageError instanceof Error ? pageError.message : String(pageError);
+                logger.warn(`[PDF→Image] ⚠️ page ${pageNum} failed to render (stream): ${msg}`);
+                yield { pageIndex: pageNum, image: "", imageSizeBytes: 0, error: msg };
+            }
+        }
+    }
     /**
      * Convert a PDF file path to an array of base64 PNG images
      *
@@ -466,7 +605,7 @@ export class PDFProcessor {
      * @param scale - Scale factor
      * @returns Estimated memory usage in MB
      */
-    static estimateConversionMemoryUsage(pdfSizeBytes, pageCount, scale = 2) {
+    static estimateConversionMemoryUsage(pdfSizeBytes, pageCount, scale = PDF_LIMITS.DEFAULT_SCALE) {
         // Rough estimation:
         // - Each page at scale 2 produces ~1-3MB PNG
         // - MuPDF needs ~2x PDF size for processing

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "9.95.2",
+  "version": "9.95.3",
   "packageManager": "pnpm@10.15.1",
   "description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
   "author": {