@juspay/neurolink 12.28.0 → 12.28.2

package/dist/types/file.d.ts

@@ -32,6 +32,45 @@ export type VisionImageConversion = {
     /** True when the bytes were re-encoded; false when they were left alone. */
     readonly converted: boolean;
 };
+/**
+ * Knobs for one vision-compatibility pass.
+ *
+ * `autoOrient` exists so a caller that also wants EXIF orientation applied
+ * gets it inside the same decode. Rotating first and transcoding second means
+ * two full decode/re-encode cycles, and for the lossy formats in both sets
+ * (TIFF, which sharp writes JPEG-compressed by default) the intermediate
+ * re-encode costs a generation of image quality that the model then reads.
+ *
+ * `outputFormat` picks the transcode target; see {@link VisionImageOutputFormat}
+ * for the default and why it stays PNG unless a caller opts into JPEG.
+ */
+export type VisionTranscodeOptions = {
+    readonly autoOrient?: boolean;
+    readonly outputFormat?: VisionImageOutputFormat;
+};
+/**
+ * What a bounded header probe could establish about an image's EXIF
+ * orientation tag, without decoding the image or reading all of its bytes.
+ *
+ * `"inconclusive"` is not a failure — it is the honest answer whenever the
+ * prefix ran out before the parse reached a verdict, or the container is one
+ * this probe does not parse. Callers must treat it as "find out the expensive
+ * way", never as "no tag".
+ */
+export type ExifOrientationProbe = "absent" | "present" | "inconclusive";
+/**
+ * Outcome of an EXIF-orientation pass over one image.
+ *
+ * See `adapters/imageFormatSupport.ts` — `normalized` is false both when the
+ * image carried no orientation tag (or an already-upright `1`) and when the
+ * pass could not run (sharp unavailable, decode failure), so callers must not
+ * treat it as a success flag; either way `buffer` is safe to send as-is.
+ */
+export type ImageOrientationNormalization = {
+    readonly buffer: Buffer;
+    /** True when the bytes were re-encoded to apply and strip EXIF orientation. */
+    readonly normalized: boolean;
+};
 /**
  * Outcome of an audio-compatibility pass over one file.
  *

package/dist/utils/messageBuilder.js

@@ -1,5 +1,5 @@
 import { existsSync, readFileSync, statSync } from "fs";
-import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
+import { open as openAsync, readFile as readFileAsync, stat as statAsync, } from "fs/promises";
 import pLimit from "p-limit";
 import { request } from "undici";
 import { redirectFollowingDispatcher } from "./redirectDispatcher.js";
@@ -8,9 +8,9 @@ import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../c
 import { getAvailableInputTokens } from "../constants/contextWindows.js";
 import { PDF_LIMITS } from "../core/constants.js";
 import { enforceAggregateFileBudget, FILE_READ_BUDGET_PERCENT, } from "../context/fileTokenBudget.js";
-import { isCSVContent, SIZE_TIER_THRESHOLDS } from "../types/index.js";
+import { isCSVContent, isTextContent, SIZE_TIER_THRESHOLDS, } from "../types/index.js";
 import { tracers, ATTR, withSpan } from "../telemetry/index.js";
-import { needsVisionTranscode, toVisionCompatibleImage, } from "../adapters/imageFormatSupport.js";
+import { EXIF_PROBE_PREFIX_BYTES, mayCarryExifOrientation, needsVisionTranscode, normalizeImageOrientation, probeExifOrientation, toVisionCompatibleImage, } from "../adapters/imageFormatSupport.js";
 import { FILE_TYPE_REGISTRY, lookupByExtension, } from "../processors/config/fileTypeRegistry.js";
 import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js";
 import { FileDetector } from "./fileDetector.js";
@@ -1405,6 +1405,20 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
         supportsNativeVideo(provider, model);
     // If no images, PDFs, audio or video, use standard message building and convert to MultimodalChatMessage[]
     if (!hasImages && !hasPDFs && !hasNativeAudio && !hasNativeVideo) {
+        // A `content: [{type: "text", ...}]` item never reaches
+        // convertContentToProviderFormat on this branch (that only runs for the
+        // multimodal path below), and buildMessagesArray only ever reads
+        // inp.text/options.prompt — so without this fold, every TextContent item
+        // in `content[]` was silently dropped whenever the request carried no
+        // image, PDF, native audio or native video. Fold them into inp.text first
+        // (same append-and-preserve-order shape as the CSV fold immediately
+        // below), so ordinary text-only requests behave the same as the
+        // multimodal path, which already reads its (first) text item out of
+        // `content[]`.
+        const textContentItems = inp.content?.filter(isTextContent) ?? [];
+        if (textContentItems.length > 0) {
+            inp.text = appendTextContentToText(textContentItems, inp.text ?? "");
+        }
         // #289: CSV content[] items don't need vision, so they never reach the
         // multimodal converter below — process them into the prompt text here
         // (otherwise a `content: [{type:"csv"}]`-only request silently drops it).
@@ -1562,6 +1576,23 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
         throw error;
     }
 }
+/**
+ * Fold TextContent `content[]` items into the prompt text, in the order they
+ * appear, on the text-only path (no image/PDF/native-audio, so
+ * convertContentToProviderFormat's own text handling never runs). Mirrors
+ * appendCsvContentToText's append-after-base-text shape; unlike CSV this
+ * needs no async decoding, just string concatenation.
+ */
+function appendTextContentToText(textItems, baseText) {
+    let text = baseText;
+    for (const item of textItems) {
+        if (!item.text) {
+            continue;
+        }
+        text += `${text ? "\n\n" : ""}${item.text}`;
+    }
+    return text;
+}
 /**
  * Timeout for detecting/parsing an in-memory CSV `content[]` buffer (#325
  * review, round 2). Bounds a stalled detector rather than blocking the
@@ -1976,12 +2007,17 @@ async function processImageToBase64(image, index, outputFormat) {
         ImageProcessor.validateBufferSize(image, `image input at index ${index}`);
         imageData = image.toString("base64");
     }
-    // Last line of defence for vision-format compatibility. `normalizeVisionImageFormats`
-    // handles `input.images` eagerly so the providers that read that array
-    // directly (Google AI Studio, Bedrock) see converted bytes, but images that
-    // arrive as URLs are downloaded further downstream and only become bytes
-    // here. A no-op for the universal formats, so the common path is unaffected.
-    if (needsVisionTranscode(mimeType)) {
+    // Last line of defence for vision-format compatibility AND orientation.
+    // `normalizeVisionImageFormats` handles `input.images` eagerly so the
+    // providers that read that array directly (Google AI Studio, Bedrock) see
+    // converted/oriented bytes, but images that arrive as URLs are downloaded
+    // further downstream and only become bytes here. A no-op for PNG/GIF (which
+    // cannot carry an orientation tag) already sent as-is, so the common
+    // no-EXIF case is unaffected.
+    const needsOrientationCheck = mayCarryExifOrientation(mimeType);
+    const needsTranscode = needsVisionTranscode(mimeType);
+    const context = `image input at index ${index}`;
+    if (needsTranscode) {
         // Guard the decoded bytes, not just the Buffer input. The buffer branch
         // above is already checked, but a data: URI reaches here having only been
         // regex-matched — so an oversized one was handed straight to sharp/ffmpeg,
@@ -1989,15 +2025,41 @@ async function processImageToBase64(image, index, outputFormat) {
         //
         // Sized before the decode rather than after: checking the Buffer would mean
         // allocating the very thing the limit exists to refuse.
-        const context = `image input at index ${index}`;
         ImageProcessor.validateSize(base64DecodedByteLength(imageData), context);
         const rawImage = Buffer.from(imageData, "base64");
-        const compatible = await toVisionCompatibleImage(rawImage, mimeType, outputFormat);
+        // Both passes need the same decode, so a format in both sets (HEIC,
+        // HEIF, TIFF, AVIF) gets one pipeline rather than two — see
+        // `toVisionCompatibleImage`'s `autoOrient`.
+        const compatible = await toVisionCompatibleImage(rawImage, mimeType, {
+            autoOrient: needsOrientationCheck,
+            outputFormat,
+        });
         if (compatible.converted) {
             imageData = compatible.buffer.toString("base64");
             mimeType = compatible.mimeType;
         }
     }
+    else if (needsOrientationCheck) {
+        // Ask a bounded header prefix whether this image needs touching at all
+        // before paying for the size guard or a full decode. PNG/JPEG/WebP were
+        // previously gated on `needsTranscode` alone — false for all three — so
+        // this MIME type reached the provider with no local size check
+        // whatsoever. A large image with no orientation tag is never going to be
+        // modified by this pass, and must keep reaching the provider exactly as
+        // it did before rather than being rejected locally for a limit that
+        // exists to bound a decode nothing here needs.
+        if (probeBase64ExifOrientation(imageData, mimeType) !== "absent") {
+            ImageProcessor.validateSize(base64DecodedByteLength(imageData), context);
+            const rawImage = Buffer.from(imageData, "base64");
+            const oriented = await normalizeImageOrientation(rawImage, mimeType);
+            // Re-encoding changes byte length. An oversized result falls back to
+            // the already-validated original, matching normalizeVisionImageFormats.
+            if (oriented.normalized &&
+                withinConversionLimit(oriented.buffer, `oriented ${context}`)) {
+                imageData = oriented.buffer.toString("base64");
+            }
+        }
+    }
     return { imageData, mimeType };
 }
 /**
@@ -2027,14 +2089,46 @@ export async function normalizeVisionImageFormats(input, outputFormat) {
             ? entry.data
             : entry;
         const source = await readImageSourceForConversion(payload);
-        if (!source || !needsVisionTranscode(source.mimeType)) {
+        if (!source ||
+            (!needsVisionTranscode(source.mimeType) &&
+                !mayCarryExifOrientation(source.mimeType))) {
             continue;
         }
-        const compatible = await toVisionCompatibleImage(source.buffer, source.mimeType, outputFormat);
-        if (!compatible.converted) {
+        let buffer = source.buffer;
+        let mimeType = source.mimeType;
+        let changed = false;
+        const orientable = mayCarryExifOrientation(mimeType);
+        if (needsVisionTranscode(mimeType)) {
+            // A format in both sets (HEIC, HEIF, TIFF, AVIF) is oriented inside the
+            // transcode's own decode instead of ahead of it. Two sequential passes
+            // cost two decodes and two encodes, and the intermediate re-encode is
+            // lossy for some of these formats — so the model would read an image a
+            // generation worse than the one it could have been sent.
+            const compatible = await toVisionCompatibleImage(buffer, mimeType, {
+                autoOrient: orientable,
+                outputFormat,
+            });
+            if (compatible.converted) {
+                buffer = compatible.buffer;
+                mimeType = compatible.mimeType;
+                changed = true;
+            }
+        }
+        else if (orientable) {
+            const oriented = await normalizeImageOrientation(buffer, mimeType);
+            // Re-encoding changes byte length, so the size guard that gated the
+            // read above must run again on the result — a re-encode that grew past
+            // the limit must not silently replace an under-limit original.
+            if (oriented.normalized &&
+                withinConversionLimit(oriented.buffer, `oriented image at index ${index}`)) {
+                buffer = oriented.buffer;
+                changed = true;
+            }
+        }
+        if (!changed) {
             continue;
         }
-        const dataUri = `data:${compatible.mimeType};base64,${compatible.buffer.toString("base64")}`;
+        const dataUri = `data:${mimeType};base64,${buffer.toString("base64")}`;
         images[index] = isWrapped
             ? { ...entry, data: dataUri }
             : dataUri;
@@ -2081,6 +2175,34 @@ function base64DecodedByteLength(base64) {
     }
     return Math.max(0, Math.floor(significant / 4) * 3 - padding);
 }
+/**
+ * Base64 characters sufficient to decode at least
+ * {@link EXIF_PROBE_PREFIX_BYTES} bytes, rounded up to a 4-character group so
+ * the slice below always starts at the beginning of the encoded stream.
+ */
+const EXIF_PROBE_BASE64_CHARS = Math.ceil(EXIF_PROBE_PREFIX_BYTES / 3) * 4;
+/**
+ * `probeExifOrientation`, but for a still-encoded data-URI payload: decodes
+ * only its leading {@link EXIF_PROBE_BASE64_CHARS} characters rather than the
+ * whole string, the same bounded-prefix trade `probeFileExifOrientation`
+ * makes for a local file. This is what lets `processImageToBase64` ask "does
+ * this image need touching at all?" before it pays for the size guard or a
+ * full decode — see the call site for why that ordering is the point.
+ *
+ * Reports `"inconclusive"` on any decode failure rather than propagating it:
+ * the caller's full-size path handles that verdict correctly already.
+ */
+function probeBase64ExifOrientation(base64, mimeType) {
+    try {
+        const window = base64.length <= EXIF_PROBE_BASE64_CHARS
+            ? base64
+            : base64.slice(0, EXIF_PROBE_BASE64_CHARS);
+        return probeExifOrientation(Buffer.from(window, "base64"), mimeType);
+    }
+    catch {
+        return "inconclusive";
+    }
+}
 /**
  * Whether a payload is small enough to hand to an image decoder, reported
  * rather than thrown. See {@link readImageSourceForConversion} for why this
@@ -2100,6 +2222,42 @@ function withinConversionByteLimit(bytes, context) {
 function withinConversionLimit(buffer, context) {
     return withinConversionByteLimit(buffer.length, context);
 }
+/**
+ * Ask a file's own header bytes whether it carries an EXIF orientation,
+ * reading at most {@link EXIF_PROBE_PREFIX_BYTES} to find out.
+ *
+ * This is what keeps the orientation pass off the JPEG hot path. Without it,
+ * widening the conversion gate to cover orientation means every local `.jpg`
+ * and `.webp` is stat'd, read into memory in full and handed to a decoder,
+ * purely to discover that the overwhelming majority carry no tag — new I/O
+ * on the single most common image input there is.
+ *
+ * Reports `"inconclusive"` on any read failure rather than propagating it:
+ * the full read that follows will hit the same problem a few lines later,
+ * where the existing handler already turns it into a warning and a skipped
+ * conversion. Failing here instead would only duplicate that.
+ */
+async function probeFileExifOrientation(path, mimeType, size) {
+    // The caller has already stat'd, so a small file costs a small buffer
+    // rather than the window's full width.
+    const window = Math.min(Math.max(size, 0), EXIF_PROBE_PREFIX_BYTES);
+    if (window === 0) {
+        return "inconclusive";
+    }
+    let handle;
+    try {
+        handle = await openAsync(path, "r");
+        const prefix = Buffer.alloc(window);
+        const { bytesRead } = await handle.read(prefix, 0, prefix.length, 0);
+        return probeExifOrientation(prefix.subarray(0, bytesRead), mimeType);
+    }
+    catch {
+        return "inconclusive";
+    }
+    finally {
+        await handle?.close().catch(() => undefined);
+    }
+}
 async function readImageSourceForConversion(payload) {
     // Both in-memory shapes are size-checked before they can reach a decoder,
     // the same way the file-path branch below is. `withinConversionLimit`
@@ -2131,16 +2289,27 @@ async function readImageSourceForConversion(payload) {
         return undefined;
     }
     const mimeType = getMimeTypeFromExtension(payload);
-    if (!needsVisionTranscode(mimeType)) {
+    const transcodable = needsVisionTranscode(mimeType);
+    if (!transcodable && !mayCarryExifOrientation(mimeType)) {
         return undefined;
     }
     try {
         // Preflight the size before reading. Conversion replaces the entry with a
         // data URI, which bypasses processImageToBase64's buffer-size guard — so
         // without this a large local HEIC/TIFF/BMP was read fully into memory and
-        // then re-encoded, with no limit applied at either step.
+        // then re-encoded, with no limit applied at either step. The same applies
+        // to a local JPEG read only for its EXIF tag: still bounded before the read.
         const { size } = await statAsync(payload);
         ImageProcessor.validateSize(size, `image at ${safeBasename(payload)}`);
+        // A file whose only reason to be here is the orientation question gets
+        // that question answered from its header, and is not read at all when the
+        // answer is no. A transcodable format is exempt: its bytes are needed
+        // whatever the orientation turns out to be, so a probe would be pure
+        // overhead.
+        if (!transcodable &&
+            (await probeFileExifOrientation(payload, mimeType, size)) === "absent") {
+            return undefined;
+        }
         const buffer = await readFileAsync(payload);
         ImageProcessor.validateBufferSize(buffer, `image at ${safeBasename(payload)}`);
         return { buffer, mimeType };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.28.0",
+  "version": "12.28.2",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -96,6 +96,7 @@
     "test:media-registry-collisions": "pnpm exec tsx test/continuous-test-suite-media-registry-collisions.ts",
     "test:memory": "pnpm exec tsx test/continuous-test-suite-memory.ts",
     "test:multimodal-rag": "pnpm exec tsx test/continuous-test-suite-multimodal-rag.ts",
+    "test:multimodal-text-content-fold": "pnpm exec tsx test/continuous-test-suite-multimodal-text-content-fold.ts",
     "test:proxy-sharing": "pnpm run build && pnpm exec tsx test/continuous-test-suite-proxy-sharing.ts",
     "test:openai-compat-streaming-retry": "pnpm exec tsx test/continuous-test-suite-openai-compat-streaming-retry.ts",
     "test:anthropic-streaming-retry": "pnpm exec tsx test/continuous-test-suite-anthropic-streaming-retry.ts",
@@ -183,7 +184,7 @@
     "// CI tier — fast, no live AI calls, safe for every commit (test:unit; also see the separate provider-safety-net CI job, which runs build + test:providers-mocked + test:provider-structure + test:error-classifier-contract on every PR)": "",
     "test:tool-routing": "pnpm exec tsx test/continuous-test-suite-tool-routing.ts",
     "test:tool-routing-semantic": "pnpm exec tsx test/continuous-test-suite-tool-routing-semantic.ts",
-    "test:unit": "pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:spans && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:classifier-router && pnpm run test:tool-routing-semantic && pnpm run test:mcp-result-cache && pnpm run test:mcp-direct-name-repair && pnpm run test:mcp-breaker-resolved-errors && pnpm run test:model-not-found-retryable && pnpm run test:archive:security && pnpm run test:office:security && pnpm run test:vector-chroma && pnpm run test:vector-pgvector && pnpm run test:vector-pinecone && pnpm run test:provider-wiring && pnpm run test:docs-mcp",
+    "test:unit": "pnpm run test:bugfixes && pnpm run test:mcp:infra && pnpm run test:mcp:spans && pnpm run test:tool-routing && pnpm run test:tool-routing-cli && pnpm run test:tool-dedup && pnpm run test:model-pool && pnpm run test:classifier-router && pnpm run test:tool-routing-semantic && pnpm run test:mcp-result-cache && pnpm run test:mcp-direct-name-repair && pnpm run test:mcp-breaker-resolved-errors && pnpm run test:model-not-found-retryable && pnpm run test:archive:security && pnpm run test:office:security && pnpm run test:image-exif && pnpm run test:vector-chroma && pnpm run test:vector-pgvector && pnpm run test:vector-pinecone && pnpm run test:provider-wiring && pnpm run test:docs-mcp",
     "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit; test:matrix, a different suite covering the full provider capability matrix, runs nightly via .github/workflows/live-matrix.yml — test:providers itself is still only wired into test:live, not any GitHub Actions workflow)": "",
     "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic",
     "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run (not wired into any GitHub Actions workflow as of this comment; run manually or add to live-matrix.yml if nightly coverage is needed)": "",
@@ -250,6 +251,7 @@
     "test:archive:security": "pnpm exec tsx test/continuous-test-suite-archive-security.ts",
     "test:office:security": "pnpm exec tsx test/continuous-test-suite-office-security.ts",
     "test:vision-image-format": "pnpm exec tsx test/continuous-test-suite-vision-image-format.ts",
+    "test:image-exif": "pnpm exec tsx test/continuous-test-suite-image-exif.ts",
     "test:model-pool": "pnpm exec tsx test/continuous-test-suite-model-pool.ts",
     "test:agents": "pnpm exec tsx test/continuous-test-suite-agents.ts",
     "test:classifier-router": "pnpm exec tsx test/continuous-test-suite-classifier-router.ts",