@juspay/neurolink 12.28.1 → 12.28.3

package/dist/types/file.d.ts

@@ -32,6 +32,45 @@ export type VisionImageConversion = {
     /** True when the bytes were re-encoded; false when they were left alone. */
     readonly converted: boolean;
 };
+/**
+ * Knobs for one vision-compatibility pass.
+ *
+ * `autoOrient` exists so a caller that also wants EXIF orientation applied
+ * gets it inside the same decode. Rotating first and transcoding second means
+ * two full decode/re-encode cycles, and for the lossy formats in both sets
+ * (TIFF, which sharp writes JPEG-compressed by default) the intermediate
+ * re-encode costs a generation of image quality that the model then reads.
+ *
+ * `outputFormat` picks the transcode target; see {@link VisionImageOutputFormat}
+ * for the default and why it stays PNG unless a caller opts into JPEG.
+ */
+export type VisionTranscodeOptions = {
+    readonly autoOrient?: boolean;
+    readonly outputFormat?: VisionImageOutputFormat;
+};
+/**
+ * What a bounded header probe could establish about an image's EXIF
+ * orientation tag, without decoding the image or reading all of its bytes.
+ *
+ * `"inconclusive"` is not a failure — it is the honest answer whenever the
+ * prefix ran out before the parse reached a verdict, or the container is one
+ * this probe does not parse. Callers must treat it as "find out the expensive
+ * way", never as "no tag".
+ */
+export type ExifOrientationProbe = "absent" | "present" | "inconclusive";
+/**
+ * Outcome of an EXIF-orientation pass over one image.
+ *
+ * See `adapters/imageFormatSupport.ts` — `normalized` is false both when the
+ * image carried no orientation tag (or an already-upright `1`) and when the
+ * pass could not run (sharp unavailable, decode failure), so callers must not
+ * treat it as a success flag; either way `buffer` is safe to send as-is.
+ */
+export type ImageOrientationNormalization = {
+    readonly buffer: Buffer;
+    /** True when the bytes were re-encoded to apply and strip EXIF orientation. */
+    readonly normalized: boolean;
+};
 /**
  * Outcome of an audio-compatibility pass over one file.
  *

package/dist/utils/messageBuilder.js

@@ -1,5 +1,5 @@
 import { existsSync, readFileSync, statSync } from "fs";
-import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
+import { open as openAsync, readFile as readFileAsync, stat as statAsync, } from "fs/promises";
 import pLimit from "p-limit";
 import { request } from "undici";
 import { redirectFollowingDispatcher } from "./redirectDispatcher.js";
@@ -10,7 +10,7 @@ import { PDF_LIMITS } from "../core/constants.js";
 import { enforceAggregateFileBudget, FILE_READ_BUDGET_PERCENT, } from "../context/fileTokenBudget.js";
 import { isCSVContent, isTextContent, SIZE_TIER_THRESHOLDS, } from "../types/index.js";
 import { tracers, ATTR, withSpan } from "../telemetry/index.js";
-import { needsVisionTranscode, toVisionCompatibleImage, } from "../adapters/imageFormatSupport.js";
+import { EXIF_PROBE_PREFIX_BYTES, mayCarryExifOrientation, needsVisionTranscode, normalizeImageOrientation, probeExifOrientation, toVisionCompatibleImage, } from "../adapters/imageFormatSupport.js";
 import { FILE_TYPE_REGISTRY, lookupByExtension, } from "../processors/config/fileTypeRegistry.js";
 import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js";
 import { FileDetector } from "./fileDetector.js";
@@ -2007,12 +2007,17 @@ async function processImageToBase64(image, index, outputFormat) {
         ImageProcessor.validateBufferSize(image, `image input at index ${index}`);
         imageData = image.toString("base64");
     }
-    // Last line of defence for vision-format compatibility. `normalizeVisionImageFormats`
-    // handles `input.images` eagerly so the providers that read that array
-    // directly (Google AI Studio, Bedrock) see converted bytes, but images that
-    // arrive as URLs are downloaded further downstream and only become bytes
-    // here. A no-op for the universal formats, so the common path is unaffected.
-    if (needsVisionTranscode(mimeType)) {
+    // Last line of defence for vision-format compatibility AND orientation.
+    // `normalizeVisionImageFormats` handles `input.images` eagerly so the
+    // providers that read that array directly (Google AI Studio, Bedrock) see
+    // converted/oriented bytes, but images that arrive as URLs are downloaded
+    // further downstream and only become bytes here. A no-op for PNG/GIF (which
+    // cannot carry an orientation tag) already sent as-is, so the common
+    // no-EXIF case is unaffected.
+    const needsOrientationCheck = mayCarryExifOrientation(mimeType);
+    const needsTranscode = needsVisionTranscode(mimeType);
+    const context = `image input at index ${index}`;
+    if (needsTranscode) {
         // Guard the decoded bytes, not just the Buffer input. The buffer branch
         // above is already checked, but a data: URI reaches here having only been
         // regex-matched — so an oversized one was handed straight to sharp/ffmpeg,
@@ -2020,15 +2025,41 @@ async function processImageToBase64(image, index, outputFormat) {
         //
         // Sized before the decode rather than after: checking the Buffer would mean
         // allocating the very thing the limit exists to refuse.
-        const context = `image input at index ${index}`;
         ImageProcessor.validateSize(base64DecodedByteLength(imageData), context);
         const rawImage = Buffer.from(imageData, "base64");
-        const compatible = await toVisionCompatibleImage(rawImage, mimeType, outputFormat);
+        // Both passes need the same decode, so a format in both sets (HEIC,
+        // HEIF, TIFF, AVIF) gets one pipeline rather than two — see
+        // `toVisionCompatibleImage`'s `autoOrient`.
+        const compatible = await toVisionCompatibleImage(rawImage, mimeType, {
+            autoOrient: needsOrientationCheck,
+            outputFormat,
+        });
         if (compatible.converted) {
             imageData = compatible.buffer.toString("base64");
             mimeType = compatible.mimeType;
         }
     }
+    else if (needsOrientationCheck) {
+        // Ask a bounded header prefix whether this image needs touching at all
+        // before paying for the size guard or a full decode. PNG/JPEG/WebP were
+        // previously gated on `needsTranscode` alone — false for all three — so
+        // this MIME type reached the provider with no local size check
+        // whatsoever. A large image with no orientation tag is never going to be
+        // modified by this pass, and must keep reaching the provider exactly as
+        // it did before rather than being rejected locally for a limit that
+        // exists to bound a decode nothing here needs.
+        if (probeBase64ExifOrientation(imageData, mimeType) !== "absent") {
+            ImageProcessor.validateSize(base64DecodedByteLength(imageData), context);
+            const rawImage = Buffer.from(imageData, "base64");
+            const oriented = await normalizeImageOrientation(rawImage, mimeType);
+            // Re-encoding changes byte length. An oversized result falls back to
+            // the already-validated original, matching normalizeVisionImageFormats.
+            if (oriented.normalized &&
+                withinConversionLimit(oriented.buffer, `oriented ${context}`)) {
+                imageData = oriented.buffer.toString("base64");
+            }
+        }
+    }
     return { imageData, mimeType };
 }
 /**
@@ -2058,14 +2089,46 @@ export async function normalizeVisionImageFormats(input, outputFormat) {
             ? entry.data
             : entry;
         const source = await readImageSourceForConversion(payload);
-        if (!source || !needsVisionTranscode(source.mimeType)) {
+        if (!source ||
+            (!needsVisionTranscode(source.mimeType) &&
+                !mayCarryExifOrientation(source.mimeType))) {
             continue;
         }
-        const compatible = await toVisionCompatibleImage(source.buffer, source.mimeType, outputFormat);
-        if (!compatible.converted) {
+        let buffer = source.buffer;
+        let mimeType = source.mimeType;
+        let changed = false;
+        const orientable = mayCarryExifOrientation(mimeType);
+        if (needsVisionTranscode(mimeType)) {
+            // A format in both sets (HEIC, HEIF, TIFF, AVIF) is oriented inside the
+            // transcode's own decode instead of ahead of it. Two sequential passes
+            // cost two decodes and two encodes, and the intermediate re-encode is
+            // lossy for some of these formats — so the model would read an image a
+            // generation worse than the one it could have been sent.
+            const compatible = await toVisionCompatibleImage(buffer, mimeType, {
+                autoOrient: orientable,
+                outputFormat,
+            });
+            if (compatible.converted) {
+                buffer = compatible.buffer;
+                mimeType = compatible.mimeType;
+                changed = true;
+            }
+        }
+        else if (orientable) {
+            const oriented = await normalizeImageOrientation(buffer, mimeType);
+            // Re-encoding changes byte length, so the size guard that gated the
+            // read above must run again on the result — a re-encode that grew past
+            // the limit must not silently replace an under-limit original.
+            if (oriented.normalized &&
+                withinConversionLimit(oriented.buffer, `oriented image at index ${index}`)) {
+                buffer = oriented.buffer;
+                changed = true;
+            }
+        }
+        if (!changed) {
             continue;
         }
-        const dataUri = `data:${compatible.mimeType};base64,${compatible.buffer.toString("base64")}`;
+        const dataUri = `data:${mimeType};base64,${buffer.toString("base64")}`;
         images[index] = isWrapped
             ? { ...entry, data: dataUri }
             : dataUri;
@@ -2112,6 +2175,34 @@ function base64DecodedByteLength(base64) {
     }
     return Math.max(0, Math.floor(significant / 4) * 3 - padding);
 }
+/**
+ * Base64 characters sufficient to decode at least
+ * {@link EXIF_PROBE_PREFIX_BYTES} bytes, rounded up to a 4-character group so
+ * the slice below always starts at the beginning of the encoded stream.
+ */
+const EXIF_PROBE_BASE64_CHARS = Math.ceil(EXIF_PROBE_PREFIX_BYTES / 3) * 4;
+/**
+ * `probeExifOrientation`, but for a still-encoded data-URI payload: decodes
+ * only its leading {@link EXIF_PROBE_BASE64_CHARS} characters rather than the
+ * whole string, the same bounded-prefix trade `probeFileExifOrientation`
+ * makes for a local file. This is what lets `processImageToBase64` ask "does
+ * this image need touching at all?" before it pays for the size guard or a
+ * full decode — see the call site for why that ordering is the point.
+ *
+ * Reports `"inconclusive"` on any decode failure rather than propagating it:
+ * the caller's full-size path handles that verdict correctly already.
+ */
+function probeBase64ExifOrientation(base64, mimeType) {
+    try {
+        const window = base64.length <= EXIF_PROBE_BASE64_CHARS
+            ? base64
+            : base64.slice(0, EXIF_PROBE_BASE64_CHARS);
+        return probeExifOrientation(Buffer.from(window, "base64"), mimeType);
+    }
+    catch {
+        return "inconclusive";
+    }
+}
 /**
  * Whether a payload is small enough to hand to an image decoder, reported
  * rather than thrown. See {@link readImageSourceForConversion} for why this
@@ -2131,6 +2222,42 @@ function withinConversionByteLimit(bytes, context) {
 function withinConversionLimit(buffer, context) {
     return withinConversionByteLimit(buffer.length, context);
 }
+/**
+ * Ask a file's own header bytes whether it carries an EXIF orientation,
+ * reading at most {@link EXIF_PROBE_PREFIX_BYTES} to find out.
+ *
+ * This is what keeps the orientation pass off the JPEG hot path. Without it,
+ * widening the conversion gate to cover orientation means every local `.jpg`
+ * and `.webp` is stat'd, read into memory in full and handed to a decoder,
+ * purely to discover that the overwhelming majority carry no tag — new I/O
+ * on the single most common image input there is.
+ *
+ * Reports `"inconclusive"` on any read failure rather than propagating it:
+ * the full read that follows will hit the same problem a few lines later,
+ * where the existing handler already turns it into a warning and a skipped
+ * conversion. Failing here instead would only duplicate that.
+ */
+async function probeFileExifOrientation(path, mimeType, size) {
+    // The caller has already stat'd, so a small file costs a small buffer
+    // rather than the window's full width.
+    const window = Math.min(Math.max(size, 0), EXIF_PROBE_PREFIX_BYTES);
+    if (window === 0) {
+        return "inconclusive";
+    }
+    let handle;
+    try {
+        handle = await openAsync(path, "r");
+        const prefix = Buffer.alloc(window);
+        const { bytesRead } = await handle.read(prefix, 0, prefix.length, 0);
+        return probeExifOrientation(prefix.subarray(0, bytesRead), mimeType);
+    }
+    catch {
+        return "inconclusive";
+    }
+    finally {
+        await handle?.close().catch(() => undefined);
+    }
+}
 async function readImageSourceForConversion(payload) {
     // Both in-memory shapes are size-checked before they can reach a decoder,
     // the same way the file-path branch below is. `withinConversionLimit`
@@ -2162,16 +2289,27 @@ async function readImageSourceForConversion(payload) {
         return undefined;
     }
     const mimeType = getMimeTypeFromExtension(payload);
-    if (!needsVisionTranscode(mimeType)) {
+    const transcodable = needsVisionTranscode(mimeType);
+    if (!transcodable && !mayCarryExifOrientation(mimeType)) {
         return undefined;
     }
     try {
         // Preflight the size before reading. Conversion replaces the entry with a
         // data URI, which bypasses processImageToBase64's buffer-size guard — so
         // without this a large local HEIC/TIFF/BMP was read fully into memory and
-        // then re-encoded, with no limit applied at either step.
+        // then re-encoded, with no limit applied at either step. The same applies
+        // to a local JPEG read only for its EXIF tag: still bounded before the read.
         const { size } = await statAsync(payload);
         ImageProcessor.validateSize(size, `image at ${safeBasename(payload)}`);
+        // A file whose only reason to be here is the orientation question gets
+        // that question answered from its header, and is not read at all when the
+        // answer is no. A transcodable format is exempt: its bytes are needed
+        // whatever the orientation turns out to be, so a probe would be pure
+        // overhead.
+        if (!transcodable &&
+            (await probeFileExifOrientation(payload, mimeType, size)) === "absent") {
+            return undefined;
+        }
         const buffer = await readFileAsync(payload);
         ImageProcessor.validateBufferSize(buffer, `image at ${safeBasename(payload)}`);
         return { buffer, mimeType };