@juspay/neurolink 12.14.20 → 12.15.0

package/dist/utils/fileDetector.js

@@ -284,7 +284,27 @@ export class FileDetector {
                 [ATTR.FILE_SIZE_BYTES]: inputSizeBytes,
             },
         }, async (span) => {
-            const detection = await FileDetector.detect(input, options);
+            // A URL used to be detected before it was loaded: MimeTypeStrategy
+            // issued a HEAD for Content-Type, then loadFromURL issued a GET for
+            // the body. Fetch it once and reuse the GET headers for the MIME
+            // strategy. A caller's explicit mimetypeHint remains authoritative,
+            // while a server header still comes after magic bytes.
+            const isUrlInput = typeof input === "string" &&
+                (input.startsWith("http://") || input.startsWith("https://"));
+            const loadedUrl = isUrlInput
+                ? await FileDetector.loadUrl(input, options)
+                : undefined;
+            const detection = loadedUrl
+                ? {
+                    ...(await FileDetector.detect(loadedUrl.content, {
+                        ...options,
+                        filenameHint: options?.filenameHint ||
+                            FileDetector.deriveInputFilename(input),
+                    }, loadedUrl.contentType, options?.filenameHint ||
+                        FileDetector.deriveInputFilename(input))),
+                    source: "url",
+                }
+                : await FileDetector.detect(input, options);
             span.setAttribute(ATTR.FILE_CATEGORY, detection.type);
             span.setAttribute(ATTR.FILE_MIMETYPE, detection.mimeType || "unknown");
             span.setAttribute(ATTR.FILE_CONFIDENCE, detection.metadata.confidence);
@@ -292,7 +312,8 @@ export class FileDetector {
             // FD-018: Comprehensive fallback parsing for extension-less files
             if (options?.allowedTypes &&
                 !options.allowedTypes.includes(detection.type)) {
-                const content = await FileDetector.loadContent(input, detection, options);
+                const content = loadedUrl?.content ??
+                    (await FileDetector.loadContent(input, detection, options));
                 const errors = [];
                 for (const allowedType of options.allowedTypes) {
                     try {
@@ -322,7 +343,8 @@ export class FileDetector {
                 FileDetector.setFileResultSpanAttributes(span, result, inputFilename, detection.type);
                 return result;
             }
-            const content = await FileDetector.loadContent(input, detection, options);
+            const content = loadedUrl?.content ??
+                (await FileDetector.loadContent(input, detection, options));
             const csvOptions = options?.csvOptions;
             const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider, options?.videoOptions);
             FileDetector.setFileResultSpanAttributes(span, result, inputFilename, detection.type);
@@ -619,7 +641,7 @@ export class FileDetector {
      * Detect file type using multi-strategy approach
      * Stops at first strategy with confidence >= threshold (default: 80%)
      */
-    static async detect(input, options) {
+    static async detect(input, options, responseMimeType, extensionInput) {
         // Short-circuit on a trustworthy caller-provided mimetype hint. This is
         // the eager-path counterpart to FileReferenceRegistry.register()'s hint
         // handling — necessary for tiny files (<= TINY_MAX) that skip the lazy
@@ -648,13 +670,15 @@ export class FileDetector {
         const confidenceThreshold = options?.confidenceThreshold ?? 80;
         const strategies = [
             new MagicBytesStrategy(),
-            new MimeTypeStrategy(),
+            new MimeTypeStrategy(responseMimeType),
             new ExtensionStrategy(),
             new ContentHeuristicStrategy(),
         ];
         let best = null;
         for (const strategy of strategies) {
-            const result = await strategy.detect(input);
+            const result = await strategy.detect(strategy instanceof ExtensionStrategy && extensionInput
+                ? extensionInput
+                : input);
             if (!best || result.metadata.confidence > best.metadata.confidence) {
                 best = result;
             }
@@ -1502,52 +1526,14 @@ export class FileDetector {
      * Load file from URL with automatic retry on transient network errors
      */
     static async loadFromURL(url, options) {
+        return (await FileDetector.loadUrl(url, options)).content;
+    }
+    /**
+     * Fetch a URL once and retain its response MIME type for detection.
+     */
+    static async loadUrl(url, options) {
         const maxSize = options?.maxSize || 200 * 1024 * 1024; // 200MB default (matches Curator memory-safety cap)
         const timeout = options?.timeout || FileDetector.DEFAULT_NETWORK_TIMEOUT;
-        // #317: pre-flight HEAD to reject an oversized file BEFORE downloading any
-        // body. content-length is advisory (chunked responses omit it), so a
-        // missing/invalid header — or a server that refuses HEAD — falls through to
-        // the streaming byte guard below; only a genuine oversize rejection stops
-        // the GET from ever running.
-        //
-        // #323: skip the pre-flight entirely when this exact URL was recently seen
-        // (its Content-Type is still cached, whether from a prior loadFromURL GET
-        // or a MimeTypeStrategy HEAD) — issuing a fresh HEAD here would defeat the
-        // whole point of that cache. The streaming byte guard in the GET below
-        // still enforces maxSize even without a pre-flight, so this doesn't remove
-        // the oversize protection — it only skips the redundant round-trip for a
-        // URL we've already been talking to within the last 60s.
-        if (getCachedUrlContentType(url, Date.now()) === undefined) {
-            try {
-                const head = await request(url, {
-                    dispatcher: redirectFollowingDispatcher(5),
-                    method: "HEAD",
-                    headersTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
-                    bodyTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
-                });
-                // Drain/close the (empty) HEAD body so the connection can be reused.
-                await head.body.dump();
-                // Only trust `content-length` on a genuine 2xx response. A non-2xx
-                // HEAD (redirect the dispatcher didn't follow, 403/404/405 "HEAD not
-                // allowed", 5xx, …) can still carry a stale/irrelevant
-                // `content-length` header — enforcing size off of that would reject
-                // (or silently pass) based on the wrong body. Treat any non-2xx HEAD
-                // as if the header were missing and fall through to the streaming
-                // GET guard below, which enforces maxSize independently either way.
-                if (head.statusCode >= 200 && head.statusCode < 300) {
-                    const declaredLength = Number(head.headers["content-length"]);
-                    if (Number.isFinite(declaredLength) && declaredLength > maxSize) {
-                        throw new Error(`File too large: ${formatFileSize(declaredLength)} (max: ${formatFileSize(maxSize)})`);
-                    }
-                }
-            }
-            catch (error) {
-                if (error instanceof Error && /File too large/.test(error.message)) {
-                    throw error;
-                }
-                logger.debug(`[FileDetector] HEAD pre-flight skipped for ${redactUrlForError(url)}: ${sanitizeErrorCause(error).message}`);
-            }
-        }
         return withRetry(async () => {
             try {
                 const response = await request(url, {
@@ -1561,9 +1547,21 @@ export class FileDetector {
                     // not be echoed into a thrown error.
                     throw new Error(`HTTP ${response.statusCode} fetching ${redactUrlForError(url)}`);
                 }
+                const contentType = response.headers["content-type"] || "";
+                const declaredLength = Number(response.headers["content-length"]);
+                if (Number.isFinite(declaredLength) && declaredLength > maxSize) {
+                    // `request()` resolves after response headers arrive. Close the
+                    // unread stream before throwing so a declared oversize response
+                    // never enters the body-reading loop or holds its connection.
+                    // Undici emits an abort error asynchronously when the stream is
+                    // destroyed, so consume that expected event before closing it.
+                    response.body.once("error", () => { });
+                    response.body.destroy();
+                    throw new Error(`File too large: ${formatFileSize(declaredLength)} (max: ${formatFileSize(maxSize)})`);
+                }
                 // #323: cache the Content-Type from this GET so a subsequent detection
                 // of the same URL needs no HEAD.
-                setCachedUrlContentType(url, response.headers["content-type"] || "", Date.now());
+                setCachedUrlContentType(url, contentType, Date.now());
                 const chunks = [];
                 let totalSize = 0;
                 for await (const chunk of response.body) {
@@ -1573,7 +1571,7 @@ export class FileDetector {
                     }
                     chunks.push(chunk);
                 }
-                return Buffer.concat(chunks);
+                return { content: Buffer.concat(chunks), contentType };
             }
             catch (error) {
                 // Node/undici DNS, TLS, and connect-timeout errors embed the full
@@ -2270,7 +2268,22 @@ class MagicBytesStrategy {
  * Detects file type from HTTP Content-Type headers
  */
 class MimeTypeStrategy {
+    responseMimeType;
+    constructor(responseMimeType) {
+        this.responseMimeType = responseMimeType;
+    }
     async detect(input) {
+        if (this.responseMimeType !== undefined) {
+            const contentType = this.responseMimeType;
+            const type = this.mimeToFileType(contentType);
+            return {
+                type,
+                mimeType: contentType.split(";")[0].trim(),
+                extension: null,
+                source: "url",
+                metadata: { confidence: type !== "unknown" ? 85 : 0 },
+            };
+        }
         if (typeof input !== "string" || !this.isURL(input)) {
             return this.unknown();
         }

package/dist/utils/messageBuilder.js

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync, statSync } from "fs";
 import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
+import pLimit from "p-limit";
 import { request } from "undici";
 import { redirectFollowingDispatcher } from "./redirectDispatcher.js";
 import { MultimodalLogger, ProviderImageAdapter, } from "../adapters/providerImageAdapter.js";
@@ -570,6 +571,21 @@ const PREPROCESSED_FILES = new WeakMap();
  * page cache the read usually hits — detection has just read the same bytes.
  */
 const FILE_READ_TIMEOUT_MS = 30_000;
+/**
+ * How many files may be detected and processed at once.
+ *
+ * A cap rather than unbounded `Promise.all`, and the cap is the point: this
+ * path admits files up to 100 MB, so N files in flight means N decoded buffers
+ * resident at once plus whatever each processor allocates on top. Turning a
+ * sequential loop into an unbounded fan-out trades a latency problem for a
+ * memory one, and on a large batch that is the worse of the two.
+ *
+ * Four is chosen to keep the worst case bounded rather than to saturate a
+ * disk: the win here is overlapping I/O wait, which most of it is, and the
+ * marginal gain past a handful of concurrent reads is small next to the
+ * marginal cost in resident bytes.
+ */
+const FILE_PROCESSING_CONCURRENCY = 4;
 /**
  * Read a file input's bytes, or null when they cannot be had.
  *
@@ -835,6 +851,57 @@ function markFileProcessed(input, entry) {
     processed.add(entry);
     PREPROCESSED_FILES.set(input, processed);
 }
+/**
+ * Detect and process one entry of the unified `files` array.
+ *
+ * Extracted so the same call can be made from the concurrent pass and from the
+ * sequential fall-through, rather than duplicated between them.
+ */
+async function detectFileForUnifiedArray(file, options, provider, genericFileMaxSize) {
+    const rawFileInput = isFileWithMetadata(file) ? file.buffer : file;
+    // Forward the caller's mimetype hint (Slack/Curator-style
+    // extension-less buffers) so the eager path classifies correctly
+    // for tiny files — the lazy registry path has its own hint wiring.
+    const fileMimetypeHint = isFileWithMetadata(file) ? file.mimetype : undefined;
+    // The name has to travel the same way, and for the same reason: the
+    // line above unwraps the object to its buffer, so by the time
+    // detection resolves an extension there is no name left to read one
+    // from. Without this a `.tar` supplied as bytes-plus-name is
+    // unidentifiable — its "ustar" marker sits at byte 257, not at
+    // offset 0 — and reports "Could not extract content" for an archive
+    // that extracts perfectly when handed its filename.
+    const fileFilenameHint = isFileWithMetadata(file) ? file.filename : undefined;
+    return FileDetector.detectAndProcess(rawFileInput, {
+        maxSize: genericFileMaxSize,
+        allowedTypes: [
+            "csv",
+            "image",
+            "pdf",
+            "svg",
+            "video",
+            "audio",
+            "archive",
+            "xlsx",
+            "docx",
+            "pptx",
+            "text",
+            "unknown",
+        ],
+        csvOptions: options.csvOptions,
+        // #478: videos arrive through this unified `files` path, so this is
+        // where the CLI's frame/quality/format request has to be handed on.
+        videoOptions: options.videoOptions
+            ? {
+                frames: options.videoOptions.frames,
+                quality: options.videoOptions.quality,
+                format: options.videoOptions.format,
+            }
+            : undefined,
+        provider: provider,
+        mimetypeHint: fileMimetypeHint,
+        filenameHint: fileFilenameHint,
+    });
+}
 export async function processUnifiedFilesArray(options, maxSize, provider) {
     options.input ??= {};
     if (!options.input.files || options.input.files.length === 0) {
@@ -882,15 +949,45 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
         inp2.text = inp2.text || "";
         let includedCount = 0;
         const fileRegistry = options.fileRegistry;
+        // Only detection runs concurrently, and only for files that will not
+        // take the lazy-registration branch.
+        //
+        // Everything else in this loop is order-dependent in a way that is not
+        // obvious from its shape. `appendDetectedFileResult` appends to
+        // `inp2.text`, `inp2.images` and `inp2.pdfFiles`, so running it out of
+        // order would reorder the prompt and the attachments against the files
+        // the caller supplied. `tryRegisterFileReference` mutates the shared
+        // registry and assigns reference ids from it. Both therefore stay in the
+        // sequential pass below; what moves off the critical path is
+        // `FileDetector.detectAndProcess`, which is where the read and the parse
+        // actually happen and which touches nothing shared.
+        //
+        // Lazy candidates are decided here — the predicate is synchronous — but
+        // are not detected up front, because registration usually means the
+        // bytes are never processed at all. Pre-detecting them would do exactly
+        // the work the lazy path exists to avoid. The rare fall-through, where
+        // registration is attempted and declines, detects inline below.
+        const isLazyCandidate = files.map((file) => Boolean(fileRegistry) &&
+            getFileSize(file) > SIZE_TIER_THRESHOLDS.TINY_MAX &&
+            !isEagerMultimodalFile(file));
+        const genericFileMaxSize = Math.max(maxSize, 100 * 1024 * 1024);
+        const detectLimit = pLimit(FILE_PROCESSING_CONCURRENCY);
+        // allSettled, not all: with `all` the rejection that surfaces is the one
+        // that happened FIRST IN TIME, so which file is blamed for a batch
+        // failure would depend on disk scheduling. The sequential loop reported
+        // the first failure BY INDEX, and the ordered walk below preserves that.
+        // It also means a second failure cannot become an unhandled rejection
+        // after the first has already thrown.
+        const detected = await Promise.allSettled(files.map((file, fileIdx) => isLazyCandidate[fileIdx]
+            ? Promise.resolve(undefined)
+            : detectLimit(() => detectFileForUnifiedArray(file, options, provider, genericFileMaxSize))));
         for (let fileIdx = 0; fileIdx < files.length; fileIdx++) {
             const file = files[fileIdx];
             const filename = extractFilename(file, fileIdx);
             try {
                 // ─── Lazy file registration path ──────────────────────────────
                 const fileSize = fileRegistry ? getFileSize(file) : 0;
-                if (fileRegistry &&
-                    fileSize > SIZE_TIER_THRESHOLDS.TINY_MAX &&
-                    !isEagerMultimodalFile(file)) {
+                if (fileRegistry && isLazyCandidate[fileIdx]) {
                     const registered = await tryRegisterFileReference(file, fileSize, fileRegistry, fileIdx);
                     if (registered) {
                         logger.info(`[NEUROLINK] File lazily registered: ${filename} (${fileSize} bytes) — deferred processing`);
@@ -900,54 +997,16 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
                     }
                 }
                 // ─── Full processing path (current behavior) ──────────────────
-                const genericFileMaxSize = Math.max(maxSize, 100 * 1024 * 1024);
-                const rawFileInput = isFileWithMetadata(file) ? file.buffer : file;
-                // Forward the caller's mimetype hint (Slack/Curator-style
-                // extension-less buffers) so the eager path classifies correctly
-                // for tiny files — the lazy registry path has its own hint wiring.
-                const fileMimetypeHint = isFileWithMetadata(file)
-                    ? file.mimetype
-                    : undefined;
-                // The name has to travel the same way, and for the same reason: the
-                // line above unwraps the object to its buffer, so by the time
-                // detection resolves an extension there is no name left to read one
-                // from. Without this a `.tar` supplied as bytes-plus-name is
-                // unidentifiable — its "ustar" marker sits at byte 257, not at
-                // offset 0 — and reports "Could not extract content" for an archive
-                // that extracts perfectly when handed its filename.
-                const fileFilenameHint = isFileWithMetadata(file)
-                    ? file.filename
-                    : undefined;
-                const result = await FileDetector.detectAndProcess(rawFileInput, {
-                    maxSize: genericFileMaxSize,
-                    allowedTypes: [
-                        "csv",
-                        "image",
-                        "pdf",
-                        "svg",
-                        "video",
-                        "audio",
-                        "archive",
-                        "xlsx",
-                        "docx",
-                        "pptx",
-                        "text",
-                        "unknown",
-                    ],
-                    csvOptions: options.csvOptions,
-                    // #478: videos arrive through this unified `files` path, so this is
-                    // where the CLI's frame/quality/format request has to be handed on.
-                    videoOptions: options.videoOptions
-                        ? {
-                            frames: options.videoOptions.frames,
-                            quality: options.videoOptions.quality,
-                            format: options.videoOptions.format,
-                        }
-                        : undefined,
-                    provider: provider,
-                    mimetypeHint: fileMimetypeHint,
-                    filenameHint: fileFilenameHint,
-                });
+                // Normally already resolved by the concurrent pass above. The inline
+                // call is the fall-through: this file was a lazy candidate, so it was
+                // deliberately not detected up front, and registration then declined
+                // it. Detection has to happen somewhere, and here it is sequential.
+                const outcome = detected[fileIdx];
+                if (outcome.status === "rejected") {
+                    throw outcome.reason;
+                }
+                const result = outcome.value ??
+                    (await detectFileForUnifiedArray(file, options, provider, genericFileMaxSize));
                 await appendDetectedFileResult(result, file, options);
                 includedCount++;
                 // Log what content type was added to the message
@@ -1023,31 +1082,22 @@ async function processExplicitCsvFiles(options) {
     }
     logger.info(`[CSV] Processing ${options.input.csvFiles.length} explicit CSV file(s)`);
     options.input.text = options.input.text || "";
-    for (let i = 0; i < options.input.csvFiles.length; i++) {
-        const csvFile = options.input.csvFiles[i];
-        try {
-            const result = await FileDetector.detectAndProcess(csvFile, {
-                allowedTypes: ["csv"],
-                csvOptions: options.csvOptions,
-            });
-            const filename = extractFilename(csvFile, i);
-            const filePath = typeof csvFile === "string" ? csvFile : filename;
-            let csvSection = `\n\n## CSV Data from "${filename}":\n`;
-            if (result.metadata) {
-                const metadataText = formatCSVMetadata(result.metadata);
-                if (metadataText) {
-                    csvSection += metadataText + `\n\n`;
-                }
-            }
-            // Put the actual CSV content BEFORE the tool instructions —
-            // buildCSVToolInstructions references "the CSV data shown above"
-            // and the trailing position keeps that reference accurate.
-            csvSection += result.content;
-            csvSection += buildCSVToolInstructions(filePath);
-            options.input.text += csvSection;
-            logger.info(`[CSV] ✅ Processed: ${filename}`);
-        }
-        catch (error) {
+    const csvFiles = options.input.csvFiles;
+    // Detection is the expensive half — a read plus a parse per file, and
+    // previously each one waited for the last. It is also the only half that is
+    // independent per file, so it is the only half that runs concurrently: the
+    // sections below are appended to a single `text` string, so they stay in a
+    // sequential index-ordered pass and the prompt reads identically.
+    const limit = pLimit(FILE_PROCESSING_CONCURRENCY);
+    const settled = await Promise.allSettled(csvFiles.map((csvFile) => limit(() => FileDetector.detectAndProcess(csvFile, {
+        allowedTypes: ["csv"],
+        csvOptions: options.csvOptions,
+    }))));
+    for (let i = 0; i < csvFiles.length; i++) {
+        const csvFile = csvFiles[i];
+        const outcome = settled[i];
+        if (outcome.status === "rejected") {
+            const error = outcome.reason;
             const filename = extractFilename(csvFile, i);
             const errMsg = error instanceof Error ? error.message : String(error);
             // #273: fail loud instead of embedding the error into the prompt text
@@ -1055,6 +1105,23 @@ async function processExplicitCsvFiles(options) {
             logger.error(`[CSV] ❌ Failed to process ${filename}: ${errMsg}`);
             throw ErrorFactory.csvProcessingFailed(filename, error instanceof Error ? error : new Error(errMsg));
         }
+        const result = outcome.value;
+        const filename = extractFilename(csvFile, i);
+        const filePath = typeof csvFile === "string" ? csvFile : filename;
+        let csvSection = `\n\n## CSV Data from "${filename}":\n`;
+        if (result.metadata) {
+            const metadataText = formatCSVMetadata(result.metadata);
+            if (metadataText) {
+                csvSection += metadataText + `\n\n`;
+            }
+        }
+        // Put the actual CSV content BEFORE the tool instructions —
+        // buildCSVToolInstructions references "the CSV data shown above"
+        // and the trailing position keeps that reference accurate.
+        csvSection += result.content;
+        csvSection += buildCSVToolInstructions(filePath);
+        options.input.text += csvSection;
+        logger.info(`[CSV] ✅ Processed: ${filename}`);
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.14.20",
+  "version": "12.15.0",
   "packageManager": "pnpm@10.15.1",
   "description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
   "author": {
@@ -92,6 +92,7 @@
     "test:media": "pnpm exec tsx test/continuous-test-suite-media-gen.ts",
     "test:media-registry-collisions": "pnpm exec tsx test/continuous-test-suite-media-registry-collisions.ts",
     "test:memory": "pnpm exec tsx test/continuous-test-suite-memory.ts",
+    "test:multimodal-rag": "pnpm exec tsx test/continuous-test-suite-multimodal-rag.ts",
     "test:proxy-sharing": "pnpm run build && pnpm exec tsx test/continuous-test-suite-proxy-sharing.ts",
     "test:openai-compat-streaming-retry": "pnpm exec tsx test/continuous-test-suite-openai-compat-streaming-retry.ts",
     "test:anthropic-streaming-retry": "pnpm exec tsx test/continuous-test-suite-anthropic-streaming-retry.ts",