@juspay/neurolink 10.10.12 → 10.11.0

package/dist/lib/providers/googleVertex/client.js

@@ -5,16 +5,18 @@ import path from "path";
 import os from "os";
 import { AIProviderName, ErrorCategory, ErrorSeverity, } from "../../constants/enums.js";
 import { BaseProvider } from "../../core/baseProvider.js";
+import { unwrapImagePayload } from "../../adapters/imageFormatSupport.js";
+import { getMimeTypeForExtension } from "../../processors/config/mimeConstants.js";
 import { DEFAULT_GEMINI_STREAM_TIMEOUT_MS, DEFAULT_MAX_STEPS, DEFAULT_TOOL_EXECUTION_TIMEOUT_MS, DEFAULT_TOOL_MAX_RETRIES, GLOBAL_LOCATION_MODELS, IMAGE_GENERATION_MODELS, TOOL_STORAGE_TIMEOUT_MS, } from "../../core/constants.js";
 import { ModelConfigurationManager } from "../../core/modelConfiguration.js";
 import { isSchemaComplexityError } from "../../core/modules/structuredOutputPolicy.js";
-import { stringifyContentSafe } from "../../utils/logSanitize.js";
+import { redactUrlForError, stringifyContentSafe, } from "../../utils/logSanitize.js";
 import { createProxyFetch } from "../../proxy/proxyFetch.js";
 import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, RateLimitError, } from "../../types/index.js";
 import { ERROR_CODES, NeuroLinkError } from "../../utils/errorHandling.js";
 import { applyVertexAnthropicCacheBreakpoints } from "../../utils/anthropicCacheBreakpoints.js";
 import { FileDetector } from "../../utils/fileDetector.js";
-import { mergeMediaFileAliases, processUnifiedFilesArray, } from "../../utils/messageBuilder.js";
+import { mergeMediaFileAliases, normalizeVisionImageFormats, processUnifiedFilesArray, } from "../../utils/messageBuilder.js";
 import { logger } from "../../utils/logger.js";
 import { GEMINI_ELISION_NOTE, planGeminiLoopReclaim, previewGeminiToolResponseText, } from "../../context/geminiLoopGuard.js";
 import { ANTHROPIC_ELISION_NOTE, planAnthropicLoopReclaim, previewAnthropicToolResultText, } from "../../context/anthropicLoopGuard.js";
@@ -1035,16 +1037,19 @@ export class GoogleVertexProvider extends BaseProvider {
         if (options.input) {
             mergeMediaFileAliases(options.input);
         }
-        if (!options.input?.files?.length) {
-            return;
-        }
-        try {
-            // Mutates options.input.text / .images / .pdfFiles in place.
-            await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
-        }
-        catch (fileError) {
-            logger.warn(`[GoogleVertex] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
+        if (options.input?.files?.length) {
+            try {
+                // Mutates options.input.text / .images / .pdfFiles in place.
+                await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
+            }
+            catch (fileError) {
+                logger.warn(`[GoogleVertex] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
+            }
         }
+        // Runs even without input.files: a caller can populate input.images
+        // directly, and this native path never reaches the shared multimodal
+        // builder that would otherwise normalize the formats.
+        await normalizeVisionImageFormats(options.input);
     }
     async executeStream(options, _analysisSchema) {
         // ALL models now use native SDKs - no more @ai-sdk/google-vertex dependency
@@ -1401,23 +1406,31 @@ export class GoogleVertexProvider extends BaseProvider {
         // Add images as inlineData parts if present
         if (multimodalInput?.images && multimodalInput.images.length > 0) {
             logger.debug(`[GoogleVertex] Processing ${multimodalInput.images.length} image(s) for native stream`);
-            for (const image of multimodalInput.images) {
+            for (const rawImage of multimodalInput.images) {
+                // `input.images` accepts `{ data, altText }` as a documented public
+                // shape, but this loop only ever handled Buffer | string — a wrapper
+                // fell through to the "assume raw bytes" branch and base64-encoded the
+                // OBJECT, sending the literal "[object Object]" to Vertex. Unwrap once,
+                // here, so every branch below sees the payload it expects.
+                const image = unwrapImagePayload(rawImage);
                 let imageBuffer;
                 let mimeType = "image/jpeg"; // Default
                 if (typeof image === "string") {
                     if (fs.existsSync(image)) {
                         imageBuffer = fs.readFileSync(image);
                         // Detect mime type from extension
-                        const ext = path.extname(image).toLowerCase();
-                        if (ext === ".png") {
-                            mimeType = "image/png";
-                        }
-                        else if (ext === ".gif") {
-                            mimeType = "image/gif";
-                        }
-                        else if (ext === ".webp") {
-                            mimeType = "image/webp";
-                        }
+                        // Registry lookup rather than a png/gif/webp switch defaulting to
+                        // JPEG: that default labelled a .heic/.bmp/.tiff/.avif reference
+                        // image as image/jpeg, which is simply untrue and is what Vertex
+                        // then rejected. The registry answers "application/octet-stream"
+                        // for a missing or unregistered extension though, and Vertex
+                        // rejects a non-image media type just as firmly — so fall back to
+                        // the bytes, which are already in hand, rather than sending either
+                        // a guess or a non-image type.
+                        const byExtension = getMimeTypeForExtension(image);
+                        mimeType = byExtension.startsWith("image/")
+                            ? byExtension
+                            : this.detectImageType(imageBuffer);
                     }
                     else if (image.startsWith("data:")) {
                         // Handle data URL
@@ -1438,7 +1451,9 @@ export class GoogleVertexProvider extends BaseProvider {
                         try {
                             const response = await fetch(image);
                             if (!response.ok) {
-                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: image });
+                                // The URL may be presigned or carry credentials in its query
+                                // string, and wrapper URLs now reach this branch too.
+                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: redactUrlForError(image) });
                                 continue;
                             }
                             const arrayBuffer = await response.arrayBuffer();
@@ -1449,7 +1464,7 @@ export class GoogleVertexProvider extends BaseProvider {
                             }
                         }
                         catch (fetchError) {
-                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: image });
+                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: redactUrlForError(image) });
                             continue;
                         }
                     }
@@ -2396,23 +2411,31 @@ export class GoogleVertexProvider extends BaseProvider {
         // Add images as inlineData parts if present
         if (multimodalInput?.images && multimodalInput.images.length > 0) {
             logger.debug(`[GoogleVertex] Processing ${multimodalInput.images.length} image(s) for native generate`);
-            for (const image of multimodalInput.images) {
+            for (const rawImage of multimodalInput.images) {
+                // `input.images` accepts `{ data, altText }` as a documented public
+                // shape, but this loop only ever handled Buffer | string — a wrapper
+                // fell through to the "assume raw bytes" branch and base64-encoded the
+                // OBJECT, sending the literal "[object Object]" to Vertex. Unwrap once,
+                // here, so every branch below sees the payload it expects.
+                const image = unwrapImagePayload(rawImage);
                 let imageBuffer;
                 let mimeType = "image/jpeg"; // Default
                 if (typeof image === "string") {
                     if (fs.existsSync(image)) {
                         imageBuffer = fs.readFileSync(image);
                         // Detect mime type from extension
-                        const ext = path.extname(image).toLowerCase();
-                        if (ext === ".png") {
-                            mimeType = "image/png";
-                        }
-                        else if (ext === ".gif") {
-                            mimeType = "image/gif";
-                        }
-                        else if (ext === ".webp") {
-                            mimeType = "image/webp";
-                        }
+                        // Registry lookup rather than a png/gif/webp switch defaulting to
+                        // JPEG: that default labelled a .heic/.bmp/.tiff/.avif reference
+                        // image as image/jpeg, which is simply untrue and is what Vertex
+                        // then rejected. The registry answers "application/octet-stream"
+                        // for a missing or unregistered extension though, and Vertex
+                        // rejects a non-image media type just as firmly — so fall back to
+                        // the bytes, which are already in hand, rather than sending either
+                        // a guess or a non-image type.
+                        const byExtension = getMimeTypeForExtension(image);
+                        mimeType = byExtension.startsWith("image/")
+                            ? byExtension
+                            : this.detectImageType(imageBuffer);
                     }
                     else if (image.startsWith("data:")) {
                         // Handle data URL
@@ -2433,7 +2456,9 @@ export class GoogleVertexProvider extends BaseProvider {
                         try {
                             const response = await fetch(image);
                             if (!response.ok) {
-                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: image });
+                                // The URL may be presigned or carry credentials in its query
+                                // string, and wrapper URLs now reach this branch too.
+                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: redactUrlForError(image) });
                                 continue;
                             }
                             const arrayBuffer = await response.arrayBuffer();
@@ -2444,7 +2469,7 @@ export class GoogleVertexProvider extends BaseProvider {
                             }
                         }
                         catch (fetchError) {
-                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: image });
+                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: redactUrlForError(image) });
                             continue;
                         }
                     }
@@ -3497,23 +3522,31 @@ export class GoogleVertexProvider extends BaseProvider {
         // Add images as image parts if present
         if (multimodalInput?.images && multimodalInput.images.length > 0) {
             logger.debug(`[GoogleVertex] Processing ${multimodalInput.images.length} image(s) for native Anthropic stream`);
-            for (const image of multimodalInput.images) {
+            for (const rawImage of multimodalInput.images) {
+                // `input.images` accepts `{ data, altText }` as a documented public
+                // shape, but this loop only ever handled Buffer | string — a wrapper
+                // fell through to the "assume raw bytes" branch and base64-encoded the
+                // OBJECT, sending the literal "[object Object]" to Vertex. Unwrap once,
+                // here, so every branch below sees the payload it expects.
+                const image = unwrapImagePayload(rawImage);
                 let imageBuffer;
                 let mimeType = "image/jpeg"; // Default
                 if (typeof image === "string") {
                     if (fs.existsSync(image)) {
                         imageBuffer = fs.readFileSync(image);
                         // Detect mime type from extension
-                        const ext = path.extname(image).toLowerCase();
-                        if (ext === ".png") {
-                            mimeType = "image/png";
-                        }
-                        else if (ext === ".gif") {
-                            mimeType = "image/gif";
-                        }
-                        else if (ext === ".webp") {
-                            mimeType = "image/webp";
-                        }
+                        // Registry lookup rather than a png/gif/webp switch defaulting to
+                        // JPEG: that default labelled a .heic/.bmp/.tiff/.avif reference
+                        // image as image/jpeg, which is simply untrue and is what Vertex
+                        // then rejected. The registry answers "application/octet-stream"
+                        // for a missing or unregistered extension though, and Vertex
+                        // rejects a non-image media type just as firmly — so fall back to
+                        // the bytes, which are already in hand, rather than sending either
+                        // a guess or a non-image type.
+                        const byExtension = getMimeTypeForExtension(image);
+                        mimeType = byExtension.startsWith("image/")
+                            ? byExtension
+                            : this.detectImageType(imageBuffer);
                     }
                     else if (image.startsWith("data:")) {
                         // Handle data URL
@@ -3534,7 +3567,9 @@ export class GoogleVertexProvider extends BaseProvider {
                         try {
                             const response = await fetch(image);
                             if (!response.ok) {
-                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: image });
+                                // The URL may be presigned or carry credentials in its query
+                                // string, and wrapper URLs now reach this branch too.
+                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: redactUrlForError(image) });
                                 continue;
                             }
                             const arrayBuffer = await response.arrayBuffer();
@@ -3545,7 +3580,7 @@ export class GoogleVertexProvider extends BaseProvider {
                             }
                         }
                         catch (fetchError) {
-                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: image });
+                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: redactUrlForError(image) });
                             continue;
                         }
                     }
@@ -4832,23 +4867,31 @@ export class GoogleVertexProvider extends BaseProvider {
         // Add images as image parts if present
         if (multimodalInput?.images && multimodalInput.images.length > 0) {
             logger.debug(`[GoogleVertex] Processing ${multimodalInput.images.length} image(s) for native Anthropic generate`);
-            for (const image of multimodalInput.images) {
+            for (const rawImage of multimodalInput.images) {
+                // `input.images` accepts `{ data, altText }` as a documented public
+                // shape, but this loop only ever handled Buffer | string — a wrapper
+                // fell through to the "assume raw bytes" branch and base64-encoded the
+                // OBJECT, sending the literal "[object Object]" to Vertex. Unwrap once,
+                // here, so every branch below sees the payload it expects.
+                const image = unwrapImagePayload(rawImage);
                 let imageBuffer;
                 let mimeType = "image/jpeg"; // Default
                 if (typeof image === "string") {
                     if (fs.existsSync(image)) {
                         imageBuffer = fs.readFileSync(image);
                         // Detect mime type from extension
-                        const ext = path.extname(image).toLowerCase();
-                        if (ext === ".png") {
-                            mimeType = "image/png";
-                        }
-                        else if (ext === ".gif") {
-                            mimeType = "image/gif";
-                        }
-                        else if (ext === ".webp") {
-                            mimeType = "image/webp";
-                        }
+                        // Registry lookup rather than a png/gif/webp switch defaulting to
+                        // JPEG: that default labelled a .heic/.bmp/.tiff/.avif reference
+                        // image as image/jpeg, which is simply untrue and is what Vertex
+                        // then rejected. The registry answers "application/octet-stream"
+                        // for a missing or unregistered extension though, and Vertex
+                        // rejects a non-image media type just as firmly — so fall back to
+                        // the bytes, which are already in hand, rather than sending either
+                        // a guess or a non-image type.
+                        const byExtension = getMimeTypeForExtension(image);
+                        mimeType = byExtension.startsWith("image/")
+                            ? byExtension
+                            : this.detectImageType(imageBuffer);
                     }
                     else if (image.startsWith("data:")) {
                         // Handle data URL
@@ -4869,7 +4912,9 @@ export class GoogleVertexProvider extends BaseProvider {
                         try {
                             const response = await fetch(image);
                             if (!response.ok) {
-                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: image });
+                                // The URL may be presigned or carry credentials in its query
+                                // string, and wrapper URLs now reach this branch too.
+                                logger.warn(`[GoogleVertex] Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: redactUrlForError(image) });
                                 continue;
                             }
                             const arrayBuffer = await response.arrayBuffer();
@@ -4880,7 +4925,7 @@ export class GoogleVertexProvider extends BaseProvider {
                             }
                         }
                         catch (fetchError) {
-                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: image });
+                            logger.warn(`[GoogleVertex] Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: redactUrlForError(image) });
                             continue;
                         }
                     }
@@ -7043,6 +7088,14 @@ export class GoogleVertexProvider extends BaseProvider {
      * @returns A promise that resolves to the generation result, including the image data.
      */
     async executeImageGeneration(options) {
+        // Image-to-image generation takes reference images through the same
+        // `input.images` array, and both generate() and stream() route here BEFORE
+        // reaching preprocessNativeFileInput's normalization — so a reference photo
+        // in HEIC/BMP/AVIF would arrive at the API untranscoded. Normalizing at the
+        // top of this method covers every route in, rather than relying on each of
+        // the several call sites to remember. Idempotent, so an already-normalized
+        // request pays nothing.
+        await normalizeVisionImageFormats(options.input);
         const prompt = options.prompt || options.input?.text || "";
         const pdfFiles = options.input?.pdfFiles || [];
         const inputImages = options.input?.images || [];
@@ -7159,7 +7212,11 @@ export class GoogleVertexProvider extends BaseProvider {
             // This handles the case where PDFs are converted to images for models that don't support native PDF
             if (hasImageInput) {
                 for (let i = 0; i < inputImages.length; i++) {
-                    const image = inputImages[i];
+                    // A fifth image loop that predates the four above and has the same
+                    // blind spot: an ImageWithAltText wrapper matches neither the Buffer
+                    // nor the string branch, so a wrapped reference image was silently
+                    // skipped — including one this method had just transcoded.
+                    const image = unwrapImagePayload(inputImages[i]);
                     let imageBase64;
                     let mimeType;
                     if (Buffer.isBuffer(image)) {
@@ -7208,7 +7265,7 @@ export class GoogleVertexProvider extends BaseProvider {
                             try {
                                 const response = await fetch(image);
                                 if (!response.ok) {
-                                    logger.warn(`Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: image, index: i });
+                                    logger.warn(`Image fetch failed: ${response.status} ${response.statusText}, skipping`, { url: redactUrlForError(image), index: i });
                                     continue;
                                 }
                                 const arrayBuffer = await response.arrayBuffer();
@@ -7221,7 +7278,7 @@ export class GoogleVertexProvider extends BaseProvider {
                                         : this.detectImageType(fetchedBuffer);
                             }
                             catch (fetchError) {
-                                logger.warn(`Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: image, index: i });
+                                logger.warn(`Image URL fetch threw, skipping: ${fetchError instanceof Error ? fetchError.message : String(fetchError)}`, { url: redactUrlForError(image), index: i });
                                 continue;
                             }
                         }

package/dist/lib/types/file.d.ts

@@ -9,6 +9,47 @@ export type FileType = "csv" | "image" | "pdf" | "audio" | "video" | "archive" |
  * Office document types
  */
 export type OfficeDocumentType = "docx" | "pptx" | "xlsx";
+/**
+ * Outcome of a vision-compatibility pass over one image.
+ *
+ * See `adapters/imageFormatSupport.ts` — `converted` is false both when the
+ * source format was already universally accepted and when no decoder could
+ * read it, so callers must not treat it as a success flag.
+ */
+export type VisionImageConversion = {
+    readonly buffer: Buffer;
+    readonly mimeType: string;
+    /** True when the bytes were re-encoded; false when they were left alone. */
+    readonly converted: boolean;
+};
+/**
+ * Broad category a file format belongs to, as a human would name it.
+ *
+ * Distinct from {@link FileType}, which is the *routing* type the detector
+ * emits. The two deliberately differ where one processor handles several
+ * formats: an .odt has `modality: "document"` but `fileType: "docx"`, and a
+ * .svg has `modality: "image"` but `fileType: "svg"` because it is sanitised as
+ * markup rather than sent to a vision API.
+ */
+export type FileModality = "image" | "audio" | "video" | "document" | "data" | "archive";
+/**
+ * One format in the canonical file-type registry.
+ *
+ * `extensions[0]` and `mimeTypes[0]` are canonical; the remaining entries are
+ * aliases accepted on input. See `processors/config/fileTypeRegistry.ts`.
+ */
+export type FileFormatEntry = {
+    /** Human-readable format name, used in registry-conflict errors. */
+    readonly label: string;
+    /** Extensions with leading dots, lowercase; first is canonical. */
+    readonly extensions: readonly string[];
+    /** MIME types, lowercase; first is canonical. */
+    readonly mimeTypes: readonly string[];
+    /** Routing type the detector emits for this format. */
+    readonly fileType: FileType;
+    /** Category a human would put this format in. */
+    readonly modality: FileModality;
+};
 /**
  * File with metadata — allows callers to pass filename alongside a Buffer.
  *