@juspay/neurolink 9.95.2 → 10.0.0

package/dist/utils/pdfProcessor.js

@@ -150,6 +150,13 @@ export class PDFProcessor {
             throw new Error(`PDF size ${sizeMB.toFixed(2)}MB exceeds ${config.maxSizeMB}MB limit for ${provider}`);
         }
         const metadata = PDFProcessor.extractBasicMetadata(content);
+        // #287: prefer an accurate page count (pdf-parse/pdfjs) over the unreliable
+        // header regex for both limit enforcement and returned metadata; fall back
+        // to the regex estimate when the accurate probe times out or fails.
+        const accuratePages = await PDFProcessor.getAccuratePageCount(content);
+        if (accuratePages !== null) {
+            metadata.estimatedPages = accuratePages;
+        }
         if (metadata.estimatedPages && metadata.estimatedPages > config.maxPages) {
             const enforceLimits = options?.enforceLimits !== false;
             if (enforceLimits) {
@@ -185,6 +192,10 @@ export class PDFProcessor {
                 ...metadata,
                 provider,
                 apiType: config.apiType,
+                // #349: surface the provider's citations requirement so downstream
+                // provider adapters (e.g. Bedrock Converse document blocks) can act on
+                // it, instead of the config field being read nowhere.
+                requiresCitations: config.requiresCitations,
             },
         };
     }
@@ -220,6 +231,53 @@ export class PDFProcessor {
             filename: undefined,
         };
     }
+    /**
+     * Accurate page count via pdf-parse (pdfjs) (#287). The header regex in
+     * `extractBasicMetadata` only sees plaintext `/Type /Page` markers and misses
+     * compressed/object-stream PDFs (and miscounts when a page dict spans a chunk
+     * boundary). This parses the document properly, bounded by a timeout so a
+     * pathological PDF can't block; returns null on timeout/failure so the caller
+     * falls back to the regex estimate.
+     */
+    static async getAccuratePageCount(buffer) {
+        try {
+            const { PDFParse } = await import("pdf-parse");
+            const pdf = new PDFParse({ data: new Uint8Array(buffer) });
+            // Captured outside the try so the `finally` below can always clear it
+            // — otherwise a `getInfo()` that wins the race leaves this timer
+            // running until PAGE_COUNT_TIMEOUT_MS fires for nothing. `.unref()`
+            // so it can never itself hold the process open in the meantime.
+            let timeoutHandle;
+            try {
+                const info = await Promise.race([
+                    pdf.getInfo(),
+                    new Promise((_, reject) => {
+                        timeoutHandle = setTimeout(() => reject(new Error("page-count timeout")), PDF_LIMITS.PAGE_COUNT_TIMEOUT_MS);
+                        timeoutHandle.unref?.();
+                    }),
+                ]);
+                const total = info.total;
+                return typeof total === "number" && total > 0 ? total : null;
+            }
+            finally {
+                if (timeoutHandle !== undefined) {
+                    clearTimeout(timeoutHandle);
+                }
+                try {
+                    await pdf.destroy?.();
+                }
+                catch {
+                    // A throwing destroy() must not discard an otherwise-valid page
+                    // count returned above (matches fileReferenceRegistry.ts's
+                    // pdf.destroy() cleanup pattern) — swallow it.
+                }
+            }
+        }
+        catch (error) {
+            logger.debug(`[PDF] Accurate page count unavailable; using regex estimate: ${error instanceof Error ? error.message : String(error)}`);
+            return null;
+        }
+    }
     static estimateTokens(pageCount, mode = "visual") {
         if (mode === "text-only") {
             return Math.ceil((pageCount / 3) * 1000);
@@ -291,48 +349,32 @@ export class PDFProcessor {
      */
     static async convertToImages(pdfBuffer, options) {
         const startTime = Date.now();
-        const { scale = 2, maxPages = PDF_LIMITS.DEFAULT_MAX_PAGES, format = "png", maxCanvasPixels = PDF_LIMITS.DEFAULT_MAX_CANVAS_PIXELS, password, } = options || {};
+        const { scale = PDF_LIMITS.DEFAULT_SCALE, maxPages = PDF_LIMITS.DEFAULT_MAX_PAGES, format = "png", maxCanvasPixels = PDF_LIMITS.DEFAULT_MAX_CANVAS_PIXELS, password, onProgress, } = options || {};
         const images = [];
         const warnings = [];
-        // ============================================================================
-        // INPUT VALIDATION (Security: Prevent malformed/malicious PDF processing)
-        // ============================================================================
-        // 0. Validate format is supported and case-sensitive
-        if (format !== "png") {
-            throw new Error(`Invalid format: "${format}". Only "png" format is currently supported.`);
-        }
-        // 0b. Validate the render scale. A non-finite or non-positive scale yields a
-        // degenerate viewport (blank/zero-dimension render), and an excessive scale
-        // can allocate hundreds of MB per page — reject both with a clear message.
-        if (!Number.isFinite(scale) || scale <= 0 || scale > PDF_LIMITS.MAX_SCALE) {
-            throw new Error(`Invalid scale: ${scale}. Scale must be a finite number greater than 0 and at most ${PDF_LIMITS.MAX_SCALE}.`);
-        }
-        // 0c. Validate the per-page pixel ceiling (#260). A non-finite or
-        // non-positive value would disable the guard or yield a NaN downscale.
-        if (!Number.isFinite(maxCanvasPixels) || maxCanvasPixels <= 0) {
-            throw new Error(`Invalid maxCanvasPixels: ${maxCanvasPixels}. Must be a finite number greater than 0.`);
-        }
-        // 1. Validate buffer is not empty or too small
-        if (!pdfBuffer || pdfBuffer.length < 5) {
-            throw new Error("Invalid PDF: Buffer is too small or empty. " +
-                "A valid PDF must be at least 5 bytes (PDF header).");
-        }
-        // 2. Validate PDF magic bytes (%PDF-)
-        if (!PDFProcessor.isValidPDF(pdfBuffer)) {
-            throw new Error("Invalid PDF: File must start with %PDF- header. " +
-                "The provided buffer does not appear to be a valid PDF file.");
-        }
-        // 3. Validate maximum buffer size to prevent memory exhaustion
-        const sizeMB = pdfBuffer.length / (1024 * 1024);
-        if (sizeMB > PDF_LIMITS.MAX_SIZE_MB) {
-            throw new Error(`PDF too large for image conversion: ${sizeMB.toFixed(2)}MB exceeds ${PDF_LIMITS.MAX_SIZE_MB}MB limit. ` +
-                "Consider splitting the PDF or using a provider with native PDF support.");
-        }
+        const pageErrors = [];
+        // Validation shared with convertToImagesStream (#302).
+        const sizeMB = PDFProcessor.validateImageConversionInput(pdfBuffer, {
+            format,
+            scale,
+            maxCanvasPixels,
+        });
         logger.debug("[PDF→Image] ✅ PDF validation passed", {
             bufferSize: pdfBuffer.length,
             sizeMB: sizeMB.toFixed(2),
             maxPages,
         });
+        // #297: surface an up-front memory estimate so an operator can spot a
+        // conversion that will allocate a lot before it runs. Uses the cheap regex
+        // page estimate (the accurate pdf-parse count runs in process(), not here).
+        const estimatedPages = Math.max(1, PDFProcessor.extractBasicMetadata(pdfBuffer).estimatedPages ?? 1);
+        const estimatedMemoryMB = PDFProcessor.estimateConversionMemoryUsage(pdfBuffer.length, estimatedPages, scale);
+        logger.info("[PDF→Image] Estimated memory usage before conversion", {
+            scale,
+            estimatedPages,
+            estimatedMemoryMB,
+            sizeMB: Number(sizeMB.toFixed(2)),
+        });
         try {
             // Dynamic import to avoid loading MuPDF binaries until needed
             const pdfToImgModule = await import("pdf-to-img");
@@ -369,35 +411,60 @@ export class PDFProcessor {
                 logger.warn(`[PDF→Image] ⚠️ ${msg}`);
                 warnings.push(msg);
             }
-            // Create PDF document iterator (password forwarded for encrypted PDFs, #258)
+            // Create PDF document (password forwarded for encrypted PDFs, #258).
+            // pdf-to-img resolves `.length` (numPages) synchronously here and exposes
+            // `.getPage(n)`, so we drive an indexed loop rather than the async
+            // iterator — that lets one bad page be isolated instead of aborting all.
             const document = await pdf(pdfBuffer, {
                 scale: effectiveScale,
                 ...(password ? { password } : {}),
             });
-            let pageIndex = 0;
-            // Iterate through pages and convert to base64
-            for await (const page of document) {
-                // Check if we've reached the max pages limit
-                if (maxPages !== undefined && pageIndex >= maxPages) {
-                    warnings.push(`Stopped at page ${pageIndex} (maxPages limit: ${maxPages})`);
+            const totalPages = document.length;
+            // #294: convert page-by-page with per-page isolation. A single page whose
+            // canvas render throws (e.g. a degenerate MediaBox) no longer discards
+            // every already-converted page — it is recorded in `errors` and the rest
+            // continue.
+            for (let pageNum = 1; pageNum <= totalPages; pageNum++) {
+                if (maxPages !== undefined && pageNum - 1 >= maxPages) {
+                    warnings.push(`Stopped at page ${pageNum - 1} (maxPages limit: ${maxPages})`);
                     break;
                 }
-                // Convert PNG buffer to base64
-                const base64Image = page.toString("base64");
-                images.push(base64Image);
-                pageIndex++;
-                logger.debug(`[PDF→Image] Converted page ${pageIndex}`, {
-                    imageSizeBytes: page.length,
-                    base64Length: base64Image.length,
-                });
+                try {
+                    const page = await document.getPage(pageNum);
+                    const base64Image = page.toString("base64");
+                    images.push(base64Image);
+                    logger.debug(`[PDF→Image] Converted page ${pageNum}`, {
+                        imageSizeBytes: page.length,
+                        base64Length: base64Image.length,
+                    });
+                    // #302: report progress after each successfully converted page.
+                    if (onProgress) {
+                        await onProgress({
+                            pagesConverted: images.length,
+                            totalPages,
+                            elapsedMs: Date.now() - startTime,
+                        });
+                    }
+                }
+                catch (pageError) {
+                    const msg = pageError instanceof Error ? pageError.message : String(pageError);
+                    logger.warn(`[PDF→Image] ⚠️ page ${pageNum} failed to render: ${msg}`);
+                    pageErrors.push({ page: pageNum, error: msg });
+                }
             }
-            // Check for empty PDF (0 pages)
+            // Empty PDF (0 pages) or every page failed → treat as a conversion
+            // failure (kept inside the try so the password/format mapping below still
+            // applies), otherwise return the pages that did convert.
             if (images.length === 0) {
+                if (pageErrors.length > 0) {
+                    throw new Error(`All ${pageErrors.length} page(s) failed to render. First error: ${pageErrors[0].error}`);
+                }
                 throw new Error("PDF has 0 pages. Cannot convert empty PDF to images.");
             }
             const conversionTimeMs = Date.now() - startTime;
             logger.info("[PDF→Image] ✅ PDF conversion completed", {
                 pageCount: images.length,
+                failedPages: pageErrors.length,
                 conversionTimeMs,
                 totalImageBytes: images.reduce((sum, img) => sum + img.length, 0),
             });
@@ -406,6 +473,7 @@ export class PDFProcessor {
                 pageCount: images.length,
                 conversionTimeMs,
                 warnings: warnings.length > 0 ? warnings : undefined,
+                errors: pageErrors.length > 0 ? pageErrors : undefined,
             };
         }
         catch (error) {
@@ -431,6 +499,101 @@ export class PDFProcessor {
             });
         }
     }
+    /**
+     * Shared input validation for the image-conversion paths. Returns the PDF
+     * size in MB (needed by the memory-estimate log). Throws on any invalid input
+     * with the same messages the batch path has always used.
+     */
+    static validateImageConversionInput(pdfBuffer, opts) {
+        if (opts.format !== "png") {
+            throw new Error(`Invalid format: "${opts.format}". Only "png" format is currently supported.`);
+        }
+        if (!Number.isFinite(opts.scale) ||
+            opts.scale < PDF_LIMITS.MIN_SCALE ||
+            opts.scale > PDF_LIMITS.MAX_SCALE) {
+            throw new Error(`Invalid scale: ${opts.scale}. Scale must be a finite number between ${PDF_LIMITS.MIN_SCALE} and ${PDF_LIMITS.MAX_SCALE}.`);
+        }
+        if (!Number.isFinite(opts.maxCanvasPixels) || opts.maxCanvasPixels <= 0) {
+            throw new Error(`Invalid maxCanvasPixels: ${opts.maxCanvasPixels}. Must be a finite number greater than 0.`);
+        }
+        if (!pdfBuffer || pdfBuffer.length < 5) {
+            throw new Error("Invalid PDF: Buffer is too small or empty. " +
+                "A valid PDF must be at least 5 bytes (PDF header).");
+        }
+        if (!PDFProcessor.isValidPDF(pdfBuffer)) {
+            throw new Error("Invalid PDF: File must start with %PDF- header. " +
+                "The provided buffer does not appear to be a valid PDF file.");
+        }
+        const sizeMB = pdfBuffer.length / (1024 * 1024);
+        if (sizeMB > PDF_LIMITS.MAX_SIZE_MB) {
+            throw new Error(`PDF too large for image conversion: ${sizeMB.toFixed(2)}MB exceeds ${PDF_LIMITS.MAX_SIZE_MB}MB limit. ` +
+                "Consider splitting the PDF or using a provider with native PDF support.");
+        }
+        return sizeMB;
+    }
+    /**
+     * Streaming variant of {@link convertToImages} (#302): yields each page's
+     * base64 PNG as soon as it renders instead of buffering the whole document,
+     * and reports progress via `options.onProgress`. A page that fails to render
+     * is yielded with `error` set (per-page isolation, #294) rather than aborting
+     * the stream.
+     *
+     * NOT a wrapper of/over {@link convertToImages}, despite the similar
+     * contract — the two are independent, parallel implementations (each does
+     * its own `pdf-to-img` import, downscale calculation, and page loop)
+     * rather than one delegating to the other. Keep behavior changes (page
+     * isolation, downscale, password handling) in sync across both by hand.
+     */
+    static async *convertToImagesStream(pdfBuffer, options) {
+        const startTime = Date.now();
+        const { scale = PDF_LIMITS.DEFAULT_SCALE, maxPages = PDF_LIMITS.DEFAULT_MAX_PAGES, format = "png", maxCanvasPixels = PDF_LIMITS.DEFAULT_MAX_CANVAS_PIXELS, password, onProgress, } = options || {};
+        PDFProcessor.validateImageConversionInput(pdfBuffer, {
+            format,
+            scale,
+            maxCanvasPixels,
+        });
+        const pdfToImgModule = await import("pdf-to-img");
+        const pdf = pdfToImgModule.pdf;
+        // #260: uniform downscale so the largest page stays under maxCanvasPixels.
+        let effectiveScale = scale;
+        const largestPixels = PDFProcessor.largestPagePixels(pdfBuffer, scale);
+        if (largestPixels > maxCanvasPixels) {
+            effectiveScale = scale * Math.sqrt(maxCanvasPixels / largestPixels);
+        }
+        const document = await pdf(pdfBuffer, {
+            scale: effectiveScale,
+            ...(password ? { password } : {}),
+        });
+        const totalPages = document.length;
+        let converted = 0;
+        for (let pageNum = 1; pageNum <= totalPages; pageNum++) {
+            if (maxPages !== undefined && pageNum - 1 >= maxPages) {
+                break;
+            }
+            try {
+                const page = await document.getPage(pageNum);
+                const base64Image = page.toString("base64");
+                converted++;
+                if (onProgress) {
+                    await onProgress({
+                        pagesConverted: converted,
+                        totalPages,
+                        elapsedMs: Date.now() - startTime,
+                    });
+                }
+                yield {
+                    pageIndex: pageNum,
+                    image: base64Image,
+                    imageSizeBytes: page.length,
+                };
+            }
+            catch (pageError) {
+                const msg = pageError instanceof Error ? pageError.message : String(pageError);
+                logger.warn(`[PDF→Image] ⚠️ page ${pageNum} failed to render (stream): ${msg}`);
+                yield { pageIndex: pageNum, image: "", imageSizeBytes: 0, error: msg };
+            }
+        }
+    }
     /**
      * Convert a PDF file path to an array of base64 PNG images
      *
@@ -466,7 +629,7 @@ export class PDFProcessor {
      * @param scale - Scale factor
      * @returns Estimated memory usage in MB
      */
-    static estimateConversionMemoryUsage(pdfSizeBytes, pageCount, scale = 2) {
+    static estimateConversionMemoryUsage(pdfSizeBytes, pageCount, scale = PDF_LIMITS.DEFAULT_SCALE) {
         // Rough estimation:
         // - Each page at scale 2 produces ~1-3MB PNG
         // - MuPDF needs ~2x PDF size for processing

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "9.95.2",
+  "version": "10.0.0",
   "packageManager": "pnpm@10.15.1",
   "description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
   "author": {