@juspay/neurolink 9.95.2 → 10.0.0

package/dist/utils/imageProcessor.js

@@ -20,6 +20,31 @@ export const MIN_IMAGE_SIZE = 12;
  * buffer that carries a format's magic bytes but is smaller than this is
  * truncated/corrupt and would fail (or render blank) downstream — reject early
  * with a clear message instead.
+ *
+ * Each threshold approximates the smallest byte count that format's
+ * container can structurally be while still holding the mandatory
+ * header/chunk skeleton for a minimal (e.g. 1×1 pixel) image — not an exact
+ * spec minimum, but close enough that anything smaller is unambiguously
+ * truncated:
+ *  - **PNG (67)**: 8-byte signature + IHDR chunk (4 len + 4 type + 13 data +
+ *    4 CRC = 25) + a minimal IDAT chunk carrying one zlib-compressed
+ *    scanline (~22 bytes) + IEND chunk (4 len + 4 type + 0 data + 4 CRC =
+ *    12) — matches the commonly-cited size of the smallest possible valid
+ *    PNG (a 1×1 transparent pixel).
+ *  - **JPEG (125)**: SOI + APP0/JFIF marker (18) + at least one DQT
+ *    quantization table (~69 for one 8-bit luma table) + SOF0 + DHT + SOS
+ *    headers + a minimal entropy-coded scan + EOI — the smallest legal
+ *    baseline-JPEG skeleton for a 1×1 image.
+ *  - **GIF (43)**: 6-byte GIF87a/89a header + 7-byte Logical Screen
+ *    Descriptor + a 2-color (6-byte) color table + 10-byte Image
+ *    Descriptor + minimal LZW-coded image sub-blocks + 1-byte trailer.
+ *  - **WEBP (20)**: RIFF container header ("RIFF" + size + "WEBP" = 12
+ *    bytes) + the first chunk's fourCC + size (8 bytes) — enough to confirm
+ *    a well-formed RIFF/WEBP container and its first chunk, before any
+ *    actual VP8/VP8L bitstream payload.
+ *  - **AVIF (100)**: ISOBMFF `ftyp` box plus the mandatory `meta` box
+ *    skeleton (`hdlr`/`pitm`/`iloc`/`iinf` sub-boxes) — roughly the minimum
+ *    to hold AVIF's required box structure before any AV1 image item data.
  */
 export const MIN_VALID_IMAGE_SIZE = {
     "image/png": 67,
@@ -932,7 +957,9 @@ export const imageUtils = {
         const cache = getImageCache();
         const cached = cache.get(url);
         if (cached) {
-            logger.debug("Using cached image for URL", { url: url.substring(0, 50) });
+            logger.debug("Using cached image for URL", {
+                url: redactUrlForError(url),
+            });
             return cached.dataUri;
         }
         // Apply rate limiting before download

package/dist/utils/logSanitize.d.ts

@@ -110,6 +110,103 @@ export declare function redactUrlCredentials(url: string): string;
  * @param url - The URL (or URL-shaped string) to redact.
  */
 export declare function redactUrlForError(url: string): string;
+/**
+ * Query-parameter names that commonly carry authentication/signature
+ * secrets in presigned URLs, matched case-insensitively: generic
+ * `token`/`signature`, AWS SigV4 (`X-Amz-Signature`, `X-Amz-Credential`,
+ * `X-Amz-Security-Token`, and the legacy `AWSAccessKeyId`/`Signature`/
+ * `Expires` query-auth params), Azure SAS (`sig`/`se`/`sp`/`sr`/`sv`), and
+ * GCS V4 signed URLs (`X-Goog-Signature`, `X-Goog-Credential`,
+ * `X-Goog-Algorithm`, `X-Goog-Date`, `X-Goog-Expires`,
+ * `X-Goog-SignedHeaders`).
+ *
+ * Unlike {@link redactUrlForError} (which drops the *entire* query string
+ * for a one-off error message), this is used where a URL is normalized
+ * into a long-lived in-memory cache key — content-varying query params
+ * must survive so distinct resources still get distinct keys, but a
+ * presigned URL's secret must never persist as a Map key for the process
+ * lifetime.
+ */
+export declare const SENSITIVE_URL_QUERY_PARAM_DENYLIST: readonly string[];
+/**
+ * Strip common tracking/analytics query params (`utm_*`, `fbclid`, `gclid`,
+ * `_ga`) from `parsed` in place, so callers that build a cache key from a
+ * URL don't miss a cache hit solely because two otherwise-identical URLs
+ * carry different tracking noise.
+ *
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` —
+ * both build a long-lived in-memory cache key from a URL and should
+ * normalize tracking params the same way. Call this before
+ * {@link stripSensitiveUrlParamsForCacheKey} so both normalizations apply
+ * consistently regardless of caller.
+ *
+ * @param parsed - A `URL` instance to strip tracking params from in place.
+ */
+export declare function stripTrackingParams(parsed: URL): void;
+/**
+ * Build a secret-free, but still auth-context-distinct, cache key from a
+ * URL that may carry presigned-URL secrets in its query string.
+ *
+ * Naively stripping {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} params before
+ * using a URL as a long-lived cache key (an earlier version of this code did
+ * exactly that) creates a correctness bug, not just a hygiene one: two
+ * *different* presigned URLs for the same underlying object — e.g. one
+ * whose signature has since expired or been revoked, and a fresh one —
+ * strip down to the *same* key and collide. A cache populated under the
+ * first would silently serve its bytes back for the second, without ever
+ * revalidating against the origin.
+ *
+ * Fix: still remove the sensitive params from `parsed` in place (so the
+ * plaintext key never retains a raw secret — the original goal), but fold a
+ * short, non-reversible hash of the *removed* key=value pairs into the
+ * returned suffix whenever any were present, so distinct auth contexts keep
+ * distinct cache keys. Two requests presenting the exact same secret value
+ * still collide (correct — that's genuinely the same authorized request);
+ * two different secret values for the same object no longer do.
+ *
+ * Callers combine the return value with the now-stripped `parsed.toString()`
+ * to form the final cache key (see `ImageCache.normalizeUrl` and
+ * `fileDetector.cacheKeyForUrl`), rather than this function returning the
+ * full key itself, so each caller can apply its own additional
+ * normalization (e.g. tracking-param removal) in between.
+ *
+ * Matching against {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} is
+ * case-insensitive (both the denylist lookup and the value folded into the
+ * hash lower-case the param name), so the original casing of a sensitive
+ * param's name is not preserved anywhere in the returned key. That's fine
+ * here — this produces a cache key, not a user-facing URL, and the same
+ * lower-cased name always hashes to the same suffix for the same value
+ * regardless of how the caller happened to case it (`Token=` vs `token=`).
+ * Non-sensitive params left on `parsed` keep their original casing untouched.
+ *
+ * @param parsed - A `URL` instance to strip sensitive params from in place.
+ * @returns "" when no sensitive params were present; otherwise a short hex
+ *   suffix — prefixed with `\0` (a byte `URL#toString()` never emits, since
+ *   every URL component is percent-encoded, so it can't collide with real
+ *   URL content) — to append to the caller's cache key.
+ */
+export declare function stripSensitiveUrlParamsForCacheKey(parsed: URL): string;
+/**
+ * Build a secret-free, auth-context-distinct cache key from a URL, for any
+ * caller that needs a long-lived in-memory Map key derived from a URL.
+ *
+ * Centralizes the exact sequence both existing callers used independently:
+ * parse via `new URL()`, strip tracking noise ({@link stripTrackingParams}),
+ * then strip and hash-suffix sensitive auth params
+ * ({@link stripSensitiveUrlParamsForCacheKey}) so distinct presigned URLs for
+ * the same object don't collide (see that function's doc comment for why a
+ * naive strip-only approach is a correctness bug, not just a hygiene one).
+ *
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` — both
+ * build a cache key from a URL and must normalize it identically, or the same
+ * URL could hash to two different keys depending on which cache looked it up.
+ *
+ * Falls back to the raw `url` string unchanged when it isn't a parseable
+ * absolute URL (mirrors both callers' pre-existing catch behavior).
+ *
+ * @param url - The URL (or URL-shaped string) to normalize into a cache key.
+ */
+export declare function normalizeUrlForCache(url: string): string;
 /**
  * Scrub any embedded absolute URLs out of free-form text — typically the
  * `.message` of an underlying network/DNS/TLS error thrown by `fetch`,

package/dist/utils/logSanitize.js

@@ -15,6 +15,7 @@
  *     jina_ (Jina), fish- (Fish Audio)
  *   - Generic key=value pairs: api_key=…, access_token: …, secret_key=…
  */
+import { createHash } from "crypto";
 import { basename, resolve as resolvePath } from "path";
 const TOKEN_PREFIXES = [
     "sk",
@@ -227,6 +228,168 @@ export function redactUrlForError(url) {
         return truncate(redactUrlCredentials(withoutQueryOrFragment));
     }
 }
+/**
+ * Query-parameter names that commonly carry authentication/signature
+ * secrets in presigned URLs, matched case-insensitively: generic
+ * `token`/`signature`, AWS SigV4 (`X-Amz-Signature`, `X-Amz-Credential`,
+ * `X-Amz-Security-Token`, and the legacy `AWSAccessKeyId`/`Signature`/
+ * `Expires` query-auth params), Azure SAS (`sig`/`se`/`sp`/`sr`/`sv`), and
+ * GCS V4 signed URLs (`X-Goog-Signature`, `X-Goog-Credential`,
+ * `X-Goog-Algorithm`, `X-Goog-Date`, `X-Goog-Expires`,
+ * `X-Goog-SignedHeaders`).
+ *
+ * Unlike {@link redactUrlForError} (which drops the *entire* query string
+ * for a one-off error message), this is used where a URL is normalized
+ * into a long-lived in-memory cache key — content-varying query params
+ * must survive so distinct resources still get distinct keys, but a
+ * presigned URL's secret must never persist as a Map key for the process
+ * lifetime.
+ */
+export const SENSITIVE_URL_QUERY_PARAM_DENYLIST = [
+    "token",
+    "signature",
+    "x-amz-signature",
+    "x-amz-credential",
+    "x-amz-security-token",
+    "awsaccesskeyid",
+    "expires",
+    "sig",
+    "se",
+    "sp",
+    "sr",
+    "sv",
+    "x-goog-signature",
+    "x-goog-credential",
+    "x-goog-algorithm",
+    "x-goog-date",
+    "x-goog-expires",
+    "x-goog-signedheaders",
+];
+/** Common tracking/analytics query params that don't affect the fetched
+ * content, so two URLs differing only by these should map to the same
+ * cache key rather than missing each other. */
+const TRACKING_URL_QUERY_PARAMS = [
+    "utm_source",
+    "utm_medium",
+    "utm_campaign",
+    "utm_term",
+    "utm_content",
+    "fbclid",
+    "gclid",
+    "_ga",
+];
+/**
+ * Strip common tracking/analytics query params (`utm_*`, `fbclid`, `gclid`,
+ * `_ga`) from `parsed` in place, so callers that build a cache key from a
+ * URL don't miss a cache hit solely because two otherwise-identical URLs
+ * carry different tracking noise.
+ *
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` —
+ * both build a long-lived in-memory cache key from a URL and should
+ * normalize tracking params the same way. Call this before
+ * {@link stripSensitiveUrlParamsForCacheKey} so both normalizations apply
+ * consistently regardless of caller.
+ *
+ * @param parsed - A `URL` instance to strip tracking params from in place.
+ */
+export function stripTrackingParams(parsed) {
+    TRACKING_URL_QUERY_PARAMS.forEach((param) => parsed.searchParams.delete(param));
+}
+/**
+ * Build a secret-free, but still auth-context-distinct, cache key from a
+ * URL that may carry presigned-URL secrets in its query string.
+ *
+ * Naively stripping {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} params before
+ * using a URL as a long-lived cache key (an earlier version of this code did
+ * exactly that) creates a correctness bug, not just a hygiene one: two
+ * *different* presigned URLs for the same underlying object — e.g. one
+ * whose signature has since expired or been revoked, and a fresh one —
+ * strip down to the *same* key and collide. A cache populated under the
+ * first would silently serve its bytes back for the second, without ever
+ * revalidating against the origin.
+ *
+ * Fix: still remove the sensitive params from `parsed` in place (so the
+ * plaintext key never retains a raw secret — the original goal), but fold a
+ * short, non-reversible hash of the *removed* key=value pairs into the
+ * returned suffix whenever any were present, so distinct auth contexts keep
+ * distinct cache keys. Two requests presenting the exact same secret value
+ * still collide (correct — that's genuinely the same authorized request);
+ * two different secret values for the same object no longer do.
+ *
+ * Callers combine the return value with the now-stripped `parsed.toString()`
+ * to form the final cache key (see `ImageCache.normalizeUrl` and
+ * `fileDetector.cacheKeyForUrl`), rather than this function returning the
+ * full key itself, so each caller can apply its own additional
+ * normalization (e.g. tracking-param removal) in between.
+ *
+ * Matching against {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} is
+ * case-insensitive (both the denylist lookup and the value folded into the
+ * hash lower-case the param name), so the original casing of a sensitive
+ * param's name is not preserved anywhere in the returned key. That's fine
+ * here — this produces a cache key, not a user-facing URL, and the same
+ * lower-cased name always hashes to the same suffix for the same value
+ * regardless of how the caller happened to case it (`Token=` vs `token=`).
+ * Non-sensitive params left on `parsed` keep their original casing untouched.
+ *
+ * @param parsed - A `URL` instance to strip sensitive params from in place.
+ * @returns "" when no sensitive params were present; otherwise a short hex
+ *   suffix — prefixed with `\0` (a byte `URL#toString()` never emits, since
+ *   every URL component is percent-encoded, so it can't collide with real
+ *   URL content) — to append to the caller's cache key.
+ */
+export function stripSensitiveUrlParamsForCacheKey(parsed) {
+    const strippedPairs = [];
+    const keysToDelete = [];
+    for (const [key, value] of parsed.searchParams.entries()) {
+        if (SENSITIVE_URL_QUERY_PARAM_DENYLIST.includes(key.toLowerCase())) {
+            strippedPairs.push(`${key.toLowerCase()}=${value}`);
+            keysToDelete.push(key);
+        }
+    }
+    if (strippedPairs.length === 0) {
+        return "";
+    }
+    keysToDelete.forEach((key) => parsed.searchParams.delete(key));
+    // Sort so key order in the original URL doesn't affect the hash — the
+    // same *set* of secret values must always hash to the same suffix.
+    strippedPairs.sort();
+    const hash = createHash("sha256")
+        .update(strippedPairs.join("&"))
+        .digest("hex")
+        .slice(0, 16);
+    return `\0auth=${hash}`;
+}
+/**
+ * Build a secret-free, auth-context-distinct cache key from a URL, for any
+ * caller that needs a long-lived in-memory Map key derived from a URL.
+ *
+ * Centralizes the exact sequence both existing callers used independently:
+ * parse via `new URL()`, strip tracking noise ({@link stripTrackingParams}),
+ * then strip and hash-suffix sensitive auth params
+ * ({@link stripSensitiveUrlParamsForCacheKey}) so distinct presigned URLs for
+ * the same object don't collide (see that function's doc comment for why a
+ * naive strip-only approach is a correctness bug, not just a hygiene one).
+ *
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` — both
+ * build a cache key from a URL and must normalize it identically, or the same
+ * URL could hash to two different keys depending on which cache looked it up.
+ *
+ * Falls back to the raw `url` string unchanged when it isn't a parseable
+ * absolute URL (mirrors both callers' pre-existing catch behavior).
+ *
+ * @param url - The URL (or URL-shaped string) to normalize into a cache key.
+ */
+export function normalizeUrlForCache(url) {
+    try {
+        const parsed = new URL(url);
+        stripTrackingParams(parsed);
+        const authSuffix = stripSensitiveUrlParamsForCacheKey(parsed);
+        return parsed.toString() + authSuffix;
+    }
+    catch {
+        return url;
+    }
+}
 /**
  * Scrub any embedded absolute URLs out of free-form text — typically the
  * `.message` of an underlying network/DNS/TLS error thrown by `fetch`,

package/dist/utils/messageBuilder.js

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync, statSync } from "fs";
 import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
+import { basename } from "path";
 import { getGlobalDispatcher, interceptors, request } from "undici";
 import { MultimodalLogger, ProviderImageAdapter, } from "../adapters/providerImageAdapter.js";
 import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
@@ -1026,6 +1027,41 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
             throw error;
         }
     }
+    // #309: enforce the provider's page/size ceilings across ALL PDFs, not just
+    // per-file. N files each just under the single-file limit can still blow past
+    // it in aggregate (e.g. three 40-page PDFs → 120 pages for a 100-page API).
+    const aggregateConfig = PDFProcessor.getProviderConfig(provider);
+    if (aggregateConfig && pdfFiles.length > 1) {
+        // A null pageCount (accurate count unavailable — see
+        // PDFProcessor.getAccuratePageCount) is treated as 0 in the sum below,
+        // which can undercount the aggregate and let a combined request over
+        // the provider's page limit slip through silently. Enforcement still
+        // runs against the known sum — a PDF with an unknown count must not
+        // fail the request outright — but the gap itself must not be silent.
+        const unknownPageCountFiles = pdfFiles.filter((f) => f.pageCount === null || f.pageCount === undefined);
+        const totalPages = pdfFiles.reduce((sum, f) => sum + (f.pageCount ?? 0), 0);
+        const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
+        if (unknownPageCountFiles.length > 0) {
+            // Filenames are caller-controlled and may be full paths carrying
+            // PII/internal directory segments — log only the basename.
+            const unknownFileNames = unknownPageCountFiles
+                .map((f) => (f.filename ? basename(f.filename) : "<unnamed file>"))
+                .join(", ");
+            logger.warn(`[PDF] Aggregate page-limit check across ${pdfFiles.length} PDFs could only be ` +
+                `partially verified: ${unknownPageCountFiles.length} file(s) have an unknown ` +
+                `page count (${unknownFileNames}), so the known total (${totalPages}) may ` +
+                `undercount the true combined page count.`);
+        }
+        if (totalPages > aggregateConfig.maxPages) {
+            throw new Error(`[PDF] Combined page count across ${pdfFiles.length} PDFs (${totalPages}) exceeds the ` +
+                `${aggregateConfig.maxPages}-page limit for ${provider}. ` +
+                `Split the request or reduce the number of PDFs.`);
+        }
+        if (totalMB > aggregateConfig.maxSizeMB) {
+            throw new Error(`[PDF] Combined size across ${pdfFiles.length} PDFs (${totalMB.toFixed(2)}MB) exceeds the ` +
+                `${aggregateConfig.maxSizeMB}MB limit for ${provider}.`);
+        }
+    }
     return pdfFiles;
 }
 /**

package/dist/utils/pdfProcessor.d.ts

@@ -7,7 +7,7 @@
  *
  * The conversion uses pdf-to-img package (MuPDF-based) for high-quality conversion.
  */
-import type { FileProcessingResult, PDFProcessorOptions, PDFProviderConfig, PDFImageConversionOptions, PDFImageConversionResult } from "../types/index.js";
+import type { FileProcessingResult, PDFProcessorOptions, PDFProviderConfig, PDFImageConversionOptions, PDFImageConversionResult, PDFImagePage } from "../types/index.js";
 export declare class PDFProcessor {
     private static readonly PDF_SIGNATURE;
     static process(content: Buffer, options?: PDFProcessorOptions): Promise<FileProcessingResult>;
@@ -20,6 +20,15 @@ export declare class PDFProcessor {
     static getProviderConfig(provider: string): PDFProviderConfig | null;
     private static isValidPDF;
     private static extractBasicMetadata;
+    /**
+     * Accurate page count via pdf-parse (pdfjs) (#287). The header regex in
+     * `extractBasicMetadata` only sees plaintext `/Type /Page` markers and misses
+     * compressed/object-stream PDFs (and miscounts when a page dict spans a chunk
+     * boundary). This parses the document properly, bounded by a timeout so a
+     * pathological PDF can't block; returns null on timeout/failure so the caller
+     * falls back to the regex estimate.
+     */
+    private static getAccuratePageCount;
     static estimateTokens(pageCount: number, mode?: "text-only" | "visual"): number;
     /**
      * Estimate the largest page's rendered pixel count from the PDF's MediaBox
@@ -61,6 +70,26 @@ export declare class PDFProcessor {
      * ```
      */
     static convertToImages(pdfBuffer: Buffer, options?: PDFImageConversionOptions): Promise<PDFImageConversionResult>;
+    /**
+     * Shared input validation for the image-conversion paths. Returns the PDF
+     * size in MB (needed by the memory-estimate log). Throws on any invalid input
+     * with the same messages the batch path has always used.
+     */
+    private static validateImageConversionInput;
+    /**
+     * Streaming variant of {@link convertToImages} (#302): yields each page's
+     * base64 PNG as soon as it renders instead of buffering the whole document,
+     * and reports progress via `options.onProgress`. A page that fails to render
+     * is yielded with `error` set (per-page isolation, #294) rather than aborting
+     * the stream.
+     *
+     * NOT a wrapper of/over {@link convertToImages}, despite the similar
+     * contract — the two are independent, parallel implementations (each does
+     * its own `pdf-to-img` import, downscale calculation, and page loop)
+     * rather than one delegating to the other. Keep behavior changes (page
+     * isolation, downscale, password handling) in sync across both by hand.
+     */
+    static convertToImagesStream(pdfBuffer: Buffer, options?: PDFImageConversionOptions): AsyncGenerator<PDFImagePage, void, void>;
     /**
      * Convert a PDF file path to an array of base64 PNG images
      *