@juspay/neurolink 10.11.2 → 10.11.3

package/dist/lib/utils/messageBuilder.js

@@ -13,6 +13,7 @@ import { FILE_TYPE_REGISTRY, lookupByExtension, } from "../processors/config/fil
 import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js";
 import { FileDetector } from "./fileDetector.js";
 import { detectIsoBmffImageMimeType } from "./isoBmff.js";
+import { needsAudioTranscode, supportsNativeAudio, toProviderCompatibleAudio, } from "../adapters/audioFormatSupport.js";
 import { getImageCache } from "./imageCache.js";
 import { ImageProcessor, imageUtils } from "./imageProcessor.js";
 import { logger } from "./logger.js";
@@ -37,6 +38,11 @@ import { estimateTokens } from "./tokenEstimation.js";
  * exactly the large media files whose estimate matters most.
  */
 const EXTENSION_TYPE_MAP = Object.fromEntries(FILE_TYPE_REGISTRY.flatMap((entry) => entry.extensions.map((ext) => [ext.slice(1), entry.fileType])));
+/**
+ * MIME type → routing type, derived from the same registry as
+ * {@link EXTENSION_TYPE_MAP} so the two can never disagree about a format.
+ */
+const MIMETYPE_TYPE_MAP = Object.fromEntries(FILE_TYPE_REGISTRY.flatMap((entry) => entry.mimeTypes.map((mime) => [mime.toLowerCase(), entry.fileType])));
 /**
  * Infer file type from extension in a file path or URL.
  * Returns undefined if no extension or unrecognized.
@@ -625,11 +631,85 @@ function enforceFileBudget(options, provider, model) {
         logger.warn(`[FileDetector] Aggregate file budget enforcement: excluded ${budgetResult.excluded.length} file(s)`);
     }
 }
+/**
+ * Per input, the file entries already folded into text and media.
+ *
+ * A WeakMap so a long-lived process cannot accumulate references to request
+ * payloads: the record vanishes with the input object it is keyed on.
+ */
+const PREPROCESSED_FILES = new WeakMap();
+/**
+ * Ceiling on reading one already-detected local file back off disk.
+ *
+ * Sized for the 100 MB this path admits from cold storage, not for the warm
+ * page cache the read usually hits — detection has just read the same bytes.
+ */
+const FILE_READ_TIMEOUT_MS = 30_000;
+/**
+ * Read a file input's bytes, or null when they cannot be had.
+ *
+ * Asynchronous because this path admits files up to 100 MB: a synchronous read
+ * of one blocks the event loop for every other in-flight request on the
+ * process, which for a server handling concurrent generations is not a
+ * micro-optimisation to trade away.
+ *
+ * A URL or data URI yields null rather than a fetch: those arrive already
+ * materialised by the time detection runs, and re-fetching a remote URL here
+ * would issue a second network request behind the caller's back.
+ */
+async function readFileInputBytes(file) {
+    try {
+        if (isFileWithMetadata(file)) {
+            return file.buffer;
+        }
+        if (Buffer.isBuffer(file)) {
+            return file;
+        }
+        if (typeof file === "string") {
+            const { readFile, stat } = await import("node:fs/promises");
+            // Two different hangs live here, and they need different guards.
+            //
+            // A FIFO or device node blocks inside the read syscall, where neither a
+            // timeout nor an abort can reach it — measured: with a signal attached,
+            // a blocked FIFO read stays pending through both open and mid-read. The
+            // timeout would return while that read sat there forever. So refuse
+            // anything that is not a regular file up front; that is the only thing
+            // that actually prevents this case.
+            //
+            // A regular file on a slow or hung mount does return control between
+            // chunks, so there the signal works (measured: rejects with AbortError
+            // in flight) — and it matters, because racing the promise alone leaves
+            // the read filling a buffer nobody will collect. Aborting in `finally`
+            // covers both exits; after a resolved read it is a no-op.
+            //
+            // stat-then-read is a TOCTOU window, but a narrow one, and it is strictly
+            // better than the unbounded read it replaces.
+            const stats = await withTimeout(stat(file), FILE_READ_TIMEOUT_MS);
+            if (!stats.isFile()) {
+                return null;
+            }
+            const controller = new AbortController();
+            try {
+                return await withTimeout(readFile(file, { signal: controller.signal }), FILE_READ_TIMEOUT_MS);
+            }
+            finally {
+                controller.abort();
+            }
+        }
+    }
+    catch {
+        // An unreadable file is not an error here: the metadata summary was
+        // already appended, so the caller degrades to previous behaviour. This
+        // covers the missing-file case that an `existsSync` pre-check used to
+        // (without the TOCTOU gap between check and read) and the timeout above.
+    }
+    return null;
+}
 /**
  * Append a detected file result to options.input based on its type.
  * Handles CSV, SVG, image, PDF, video, audio, archive, xlsx, docx, pptx, text, and unknown types.
  */
-function appendDetectedFileResult(result, file, options) {
+async function appendDetectedFileResult(result, file, options) {
     options.input ??= {};
     const filename = extractFilename(file);
     if (result.type === "csv") {
@@ -682,6 +762,16 @@ function appendDetectedFileResult(result, file, options) {
         if (result.content) {
             options.input.text += `\n\n## Audio File: "${filename}"\n${result.content}\n`;
         }
+        // Carry the bytes forward as well as the summary. Whether they are used is
+        // decided later, per provider: one that can listen receives the audio, one
+        // that cannot still gets the summary above and is no worse off than before.
+        const audioBytes = await readFileInputBytes(file);
+        if (audioBytes) {
+            options.input.nativeAudioFiles = [
+                ...(options.input.nativeAudioFiles || []),
+                { buffer: audioBytes, filename, mimeType: result.mimeType },
+            ];
+        }
         if (result.images && result.images.length > 0) {
             options.input.images = [
                 ...(options.input.images || []),
@@ -803,13 +893,52 @@ function warnIfVideoTranscriptionRequested(videoOptions) {
  * `input.files` — without this, mimetype-hint and text-file inputs
  * would silently never reach the model on those paths.
  */
+/**
+ * Record that one file entry has been folded into an input.
+ *
+ * Marked per entry as each completes, not per run: the loop throws on the
+ * first file it cannot process (#273, fail loud), and the SDK's own retry path
+ * re-invokes this function with the same input. Marking the whole run on entry
+ * would make that retry a silent no-op — permanently skipping the failed file
+ * and shipping a half-populated request with nothing surfaced. Marking the
+ * whole run on exit would instead re-process the files that had already
+ * succeeded, duplicating them. Per entry is the only version that is right in
+ * both directions.
+ */
+function markFileProcessed(input, entry) {
+    const processed = PREPROCESSED_FILES.get(input) ?? new Set();
+    processed.add(entry);
+    PREPROCESSED_FILES.set(input, processed);
+}
 export async function processUnifiedFilesArray(options, maxSize, provider) {
     options.input ??= {};
     if (!options.input.files || options.input.files.length === 0) {
         return;
     }
-    const totalFiles = options.input.files.length;
-    const files = options.input.files;
+    // Every result this function produces is *appended* — the summary onto
+    // `text`, the bytes onto `nativeAudioFiles`/`images`/`pdfFiles` — and
+    // `files` is only ever read, never consumed. Running it twice over the same
+    // entry therefore doubles the injected text and attaches the same recording
+    // twice, which a provider sees as two distinct files.
+    //
+    // That is reachable: providers whose native paths preprocess in both
+    // `generate()` and `executeStream()` share one `options.input` reference
+    // with `BaseProvider`'s real-stream → fake-stream fallback, so a retried
+    // stream runs this a second time over the same object.
+    //
+    // Tracked per *entry* rather than per input, because a caller that appends a
+    // file to an input it has already used must still get the new one processed
+    // — treating the whole input as done would silently drop it. Entries are
+    // compared by identity (or by value for a path string), which is what a
+    // repeat of the same attachment actually looks like.
+    const alreadyProcessed = PREPROCESSED_FILES.get(options.input) ?? new Set();
+    const pending = options.input.files.filter((entry) => !alreadyProcessed.has(entry));
+    if (pending.length === 0) {
+        logger.debug("[NEUROLINK] Every attached file has already been processed for this input — skipping to avoid duplicate attachments");
+        return;
+    }
+    const totalFiles = pending.length;
+    const files = pending;
     warnIfVideoTranscriptionRequested(options.videoOptions);
     return withSpan({
         name: "neurolink.file.process_all",
@@ -841,6 +970,7 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
                     if (registered) {
                         logger.info(`[NEUROLINK] File lazily registered: ${filename} (${fileSize} bytes) — deferred processing`);
                         includedCount++;
+                        markFileProcessed(inp2, file);
                         continue;
                     }
                 }
@@ -853,6 +983,16 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
                 const fileMimetypeHint = isFileWithMetadata(file)
                     ? file.mimetype
                     : undefined;
+                // The name has to travel the same way, and for the same reason: the
+                // line above unwraps the object to its buffer, so by the time
+                // detection resolves an extension there is no name left to read one
+                // from. Without this a `.tar` supplied as bytes-plus-name is
+                // unidentifiable — its "ustar" marker sits at byte 257, not at
+                // offset 0 — and reports "Could not extract content" for an archive
+                // that extracts perfectly when handed its filename.
+                const fileFilenameHint = isFileWithMetadata(file)
+                    ? file.filename
+                    : undefined;
                 const result = await FileDetector.detectAndProcess(rawFileInput, {
                     maxSize: genericFileMaxSize,
                     allowedTypes: [
@@ -881,12 +1021,14 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
                         : undefined,
                     provider: provider,
                     mimetypeHint: fileMimetypeHint,
+                    filenameHint: fileFilenameHint,
                 });
-                appendDetectedFileResult(result, file, options);
+                await appendDetectedFileResult(result, file, options);
                 includedCount++;
                 // Log what content type was added to the message
                 const contentType = result.type === "image" ? "image" : "text";
                 logger.info(`[NEUROLINK] File added to message: ${filename} as ${contentType} (type: ${result.type})`);
+                markFileProcessed(inp2, file);
             }
             catch (error) {
                 const errMsg = error instanceof Error ? error.message : String(error);
@@ -1223,8 +1365,14 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     // convertContentToProviderFormat at all.
     const hasPDFs = pdfFiles.length > 0 ||
         !!(inp.content && inp.content.some((c) => c.type === "pdf"));
-    // If no images or PDFs, use standard message building and convert to MultimodalChatMessage[]
-    if (!hasImages && !hasPDFs) {
+    // Audio that is to be delivered natively is multimodal for the same reason a
+    // PDF is: it becomes a non-text part. Without this an audio-only turn — the
+    // ordinary "transcribe this recording" request — took the text-only branch
+    // below, where the collected bytes have nowhere to go and only the metadata
+    // summary survives.
+    const hasNativeAudio = (inp.nativeAudioFiles?.length ?? 0) > 0 && supportsNativeAudio(provider);
+    // If no images, PDFs or audio, use standard message building and convert to MultimodalChatMessage[]
+    if (!hasImages && !hasPDFs && !hasNativeAudio) {
         // #289: CSV content[] items don't need vision, so they never reach the
         // multimodal converter below — process them into the prompt text here
         // (otherwise a `content: [{type:"csv"}]`-only request silently drops it).
@@ -1328,10 +1476,22 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     try {
         let userContent;
         if (inp.content && inp.content.length > 0) {
-            userContent = await convertContentToProviderFormat(inp.content, provider, model, options.pdfOptions);
-        }
-        else if ((inp.images && inp.images.length > 0) || pdfFiles.length > 0) {
-            userContent = await convertMultimodalToProviderFormat(inp.text ?? "", inp.images || [], pdfFiles, provider, model);
+            // Audio detected from `input.files` has to reach this branch too. A
+            // caller that supplies structured `content` AND attaches an audio file
+            // is not asking for the audio to be discarded — but this branch bypasses
+            // the multimodal converter below, so the bytes were dropped and only the
+            // metadata summary folded into `text` survived. Exactly the failure this
+            // change exists to remove, reintroduced through the other door.
+            userContent = await convertContentToProviderFormat(inp.content, provider, model, options.pdfOptions, inp.nativeAudioFiles ?? []);
+        }
+        else if ((inp.images && inp.images.length > 0) ||
+            pdfFiles.length > 0 ||
+            // Audio alone must still take the multimodal path. Without this clause an
+            // audio-only turn fell through to the plain-text branch below, so the
+            // recording was dropped and only its metadata summary — already folded
+            // into `text` — ever reached the model.
+            (inp.nativeAudioFiles?.length ?? 0) > 0) {
+            userContent = await convertMultimodalToProviderFormat(inp.text ?? "", inp.images || [], pdfFiles, provider, model, inp.nativeAudioFiles ?? []);
         }
         else {
             userContent = inp.text;
@@ -1407,7 +1567,7 @@ async function appendCsvContentToText(csvItems, baseText) {
 /**
  * Convert advanced content format to provider-specific format
  */
-async function convertContentToProviderFormat(content, provider, _model, pdfOptions) {
+async function convertContentToProviderFormat(content, provider, _model, pdfOptions, audioFiles = []) {
     const textContent = content.find((c) => c.type === "text");
     const imageContent = content.filter((c) => c.type === "image");
     const pdfContent = content.filter((c) => c.type === "pdf");
@@ -1418,13 +1578,25 @@ async function convertContentToProviderFormat(content, provider, _model, pdfOpti
     if (csvContent.length > 0) {
         text = await appendCsvContentToText(csvContent, text);
     }
-    const hasMultimodal = imageContent.length > 0 || pdfContent.length > 0;
+    // Audio the provider can actually read counts as multimodal content, on both
+    // of the checks below. Computed before the validation rather than after it:
+    // a request whose structured `content` carries no text but does carry an
+    // attached recording is a complete request, and leaving audio out of
+    // `hasMultimodal` rejected it as empty before delivery could be considered.
+    //
+    // The same flag then keeps it off the text-only early return, which would
+    // otherwise hand back a plain string and lose the bytes — the exact drop this
+    // change exists to stop, reached through a different branch. Gated on the
+    // provider accepting audio so one that cannot keeps the cheaper plain-text
+    // shape rather than an array carrying a part it will ignore.
+    const deliversAudio = audioFiles.length > 0 && supportsNativeAudio(provider);
+    const hasMultimodal = imageContent.length > 0 || pdfContent.length > 0 || deliversAudio;
     // Validate that we have at least some content
     if (!hasMultimodal && !text) {
         throw new Error("Content must include either text or multimodal content");
     }
-    // Text-only case (CSV has already been folded into `text`)
-    if (imageContent.length === 0 && pdfContent.length === 0) {
+    // Text-only case (CSV has already been folded into `text`).
+    if (!hasMultimodal) {
         return text;
     }
     // Extract images as Buffer | string array
@@ -1448,7 +1620,7 @@ async function convertContentToProviderFormat(content, provider, _model, pdfOpti
     await enforceAggregatePdfLimits(pdfFiles, provider, {
         trustSuppliedPageCounts: false,
     });
-    return await convertMultimodalToProviderFormat(text, images, pdfFiles, provider, _model);
+    return await convertMultimodalToProviderFormat(text, images, pdfFiles, provider, _model, audioFiles);
 }
 /**
  * Check if a string is an internet URL
@@ -1629,13 +1801,20 @@ function detectMimeTypeFromBuffer(buffer) {
     if (looksLikeSvgMarkup(buffer)) {
         return "image/svg+xml";
     }
-    // JPEG 2000: 12-byte signature box "....jP  ".
-    if (buffer.length >= 8 &&
+    // JPEG 2000: the full 12-byte signature box, trailing 0D 0A 87 0A included.
+    // Those four bytes are a line-ending probe that a transfer mangling newlines
+    // or stripping the eighth bit corrupts, so checking only length and brand
+    // accepts exactly the damaged files the signature exists to reject.
+    if (buffer.length >= 12 &&
         buffer[0] === 0x00 &&
         buffer[1] === 0x00 &&
         buffer[2] === 0x00 &&
         buffer[3] === 0x0c &&
-        buffer.toString("latin1", 4, 8) === "jP  ") {
+        buffer.toString("latin1", 4, 8) === "jP  " &&
+        buffer[8] === 0x0d &&
+        buffer[9] === 0x0a &&
+        buffer[10] === 0x87 &&
+        buffer[11] === 0x0a) {
         return "image/jp2";
     }
     return undefined;
@@ -1750,7 +1929,17 @@ async function processImageToBase64(image, index) {
     // arrive as URLs are downloaded further downstream and only become bytes
     // here. A no-op for the universal formats, so the common path is unaffected.
     if (needsVisionTranscode(mimeType)) {
-        const compatible = await toVisionCompatibleImage(Buffer.from(imageData, "base64"), mimeType);
+        // Guard the decoded bytes, not just the Buffer input. The buffer branch
+        // above is already checked, but a data: URI reaches here having only been
+        // regex-matched — so an oversized one was handed straight to sharp/ffmpeg,
+        // which decode it in full.
+        //
+        // Sized before the decode rather than after: checking the Buffer would mean
+        // allocating the very thing the limit exists to refuse.
+        const context = `image input at index ${index}`;
+        ImageProcessor.validateSize(base64DecodedByteLength(imageData), context);
+        const rawImage = Buffer.from(imageData, "base64");
+        const compatible = await toVisionCompatibleImage(rawImage, mimeType);
         if (compatible.converted) {
             imageData = compatible.buffer.toString("base64");
             mimeType = compatible.mimeType;
@@ -1806,19 +1995,84 @@ export async function normalizeVisionImageFormats(input) {
  * conversion. Reading every attached .png off disk just to confirm it is
  * already compatible would double the I/O of the common case for no benefit.
  */
+/**
+ * Byte length `Buffer.from(b64, "base64")` will allocate, computed without
+ * allocating it.
+ *
+ * Base64 encodes 3 bytes per 4 characters, so the encoded length settles the
+ * decoded length up front. The point is ordering: a size guard applied to the
+ * decoded Buffer has already paid for the allocation it exists to prevent, so
+ * every base64 site here checks this first and decodes second — the same shape
+ * as the file-path branch, which stats before it reads.
+ *
+ * This bounds the decode, not the whole request: the encoded string is already
+ * resident by the time we see it, so an oversized payload still costs its own
+ * length in memory. What it removes is the second, larger allocation on top.
+ *
+ * Whitespace is skipped because the decoder ignores it, which keeps the count
+ * exact rather than a conservative over-estimate that would reject legitimate
+ * payloads sitting just under the limit.
+ */
+function base64DecodedByteLength(base64) {
+    let significant = 0;
+    let padding = 0;
+    for (let i = 0; i < base64.length; i++) {
+        const code = base64.charCodeAt(i);
+        if (code === 32 || code === 9 || code === 10 || code === 13) {
+            continue;
+        }
+        significant++;
+        if (code === 61) {
+            padding++;
+        }
+    }
+    return Math.max(0, Math.floor(significant / 4) * 3 - padding);
+}
+/**
+ * Whether a payload is small enough to hand to an image decoder, reported
+ * rather than thrown. See {@link readImageSourceForConversion} for why this
+ * pass degrades instead of failing.
+ */
+function withinConversionByteLimit(bytes, context) {
+    try {
+        ImageProcessor.validateSize(bytes, context);
+        return true;
+    }
+    catch (error) {
+        logger.warn(`[messageBuilder] Skipping vision-format conversion for ${context}: ` +
+            `${error instanceof Error ? error.message : String(error)}`);
+        return false;
+    }
+}
+function withinConversionLimit(buffer, context) {
+    return withinConversionByteLimit(buffer.length, context);
+}
 async function readImageSourceForConversion(payload) {
+    // Both in-memory shapes are size-checked before they can reach a decoder,
+    // the same way the file-path branch below is. `withinConversionLimit`
+    // reports rather than throws, because this whole pass is best-effort: an
+    // image too large to convert is left in its original format for the
+    // downstream guard to reject, which is what an unconvertible image already
+    // does. Throwing here would turn a normalisation step into a hard failure.
     if (Buffer.isBuffer(payload)) {
         const mimeType = detectMimeTypeFromBuffer(payload);
-        return mimeType ? { buffer: payload, mimeType } : undefined;
+        if (!mimeType || !withinConversionLimit(payload, "image buffer")) {
+            return undefined;
+        }
+        return { buffer: payload, mimeType };
     }
     if (typeof payload !== "string") {
         return undefined;
     }
     if (payload.startsWith("data:")) {
         const match = payload.match(/^data:([^;]+);base64,(.+)$/);
-        return match
-            ? { buffer: Buffer.from(match[2], "base64"), mimeType: match[1] }
-            : undefined;
+        if (!match) {
+            return undefined;
+        }
+        if (!withinConversionByteLimit(base64DecodedByteLength(match[2]), "image data URI")) {
+            return undefined;
+        }
+        return { buffer: Buffer.from(match[2], "base64"), mimeType: match[1] };
     }
     if (isInternetUrl(payload)) {
         return undefined;
@@ -1933,7 +2187,7 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
 async function convertMultimodalToProviderFormat(text, images, 
 // The canonical entry shape (#309) rather than a fourth copy of it inline —
 // which is what let the render knobs stop short of this function.
-pdfFiles, provider, model) {
+pdfFiles, provider, model, audioFiles = []) {
     const content = [
         { type: "text", text },
     ];
@@ -1948,6 +2202,37 @@ pdfFiles, provider, model) {
             });
         }
     }
+    // Attach audio to providers that can listen. The metadata summary was
+    // already appended to `text` during detection, so this adds the recording
+    // itself rather than replacing the description — a model asked "how long is
+    // this?" keeps the exact answer, and one asked "what is said?" can now
+    // answer at all.
+    if (audioFiles.length > 0 && supportsNativeAudio(provider)) {
+        for (const audio of audioFiles) {
+            // Derived from the trimmed basename so a directory containing a dot
+            // (`/srv/v1.2/recording`) cannot be mistaken for the file's extension.
+            const base = safeBasename(audio.filename);
+            const dot = base.lastIndexOf(".");
+            const extension = dot > 0 ? base.slice(dot) : ".bin";
+            const compatible = await toProviderCompatibleAudio(audio.buffer, audio.mimeType, extension);
+            // A conversion that could not run leaves a container the provider does
+            // not accept. Sending it anyway turns a metadata answer into an opaque
+            // HTTP 400, so it is skipped and the summary stands.
+            if (needsAudioTranscode(compatible.mimeType)) {
+                logger.warn(`[Audio] Skipping native delivery of ${base}: ` +
+                    `${compatible.mimeType} is not accepted by ${provider} and could ` +
+                    `not be converted. The metadata summary was still included.`);
+                continue;
+            }
+            content.push({
+                type: "file",
+                data: compatible.buffer,
+                mediaType: compatible.mimeType,
+            });
+            logger.info(`[Audio] ✅ Added to content (native audio): ${base}` +
+                `${compatible.converted ? ` (converted to ${compatible.mimeType})` : ""}`);
+        }
+    }
     // Check if provider supports native PDF processing
     const supportsNativePDF = PDFProcessor.supportsNativePDF(provider);
     if (supportsNativePDF) {
@@ -2163,17 +2448,18 @@ function getFileSource(file) {
  * 10 KB is far below any real photo, so this affected essentially every image
  * attached by path — an ordinary JPEG, not just unusual formats.
  *
- * Scoped to images deliberately, and audio deserves spelling out because the
- * reason is not the one it appears to be. An audio file's message contains only
- * a metadata block — duration, codec, sample rate — on BOTH paths; no audio
- * bytes are handed to the provider either way. Moving audio to the eager path
- * would therefore change nothing about what the model receives. That audio
- * content never reaches the model at all is a separate and pre-existing gap,
- * not something this threshold decision can repair, and it is easy to mistake
- * for working code because the metadata block answers exactly the questions
- * ("how long is it?", "what sample rate?") a test is most tempted to ask.
- * Video is left alone for the opposite reason: its frames are measurably
- * present in the message and a model reads them correctly.
+ * Images were only the most visible case. The same reasoning generalises to
+ * every type whose content is not text, which is why the rule below is stated
+ * as an exclusion — preview lazily what a text slice can faithfully represent,
+ * process everything else — rather than as a list of media types. See
+ * {@link isEagerType} for what that costs and buys per modality.
+ *
+ * Audio deserves a note because it is the case most likely to look fine while
+ * being broken: its message carries a metadata block — duration, codec, sample
+ * rate — that answers exactly the questions ("how long is it?", "what sample
+ * rate?") a test is most tempted to ask, and answers them correctly with no
+ * audio whatsoever attached. A test asking those questions passes against a
+ * model that received nothing.
  *
  * Note this costs no extra memory: `tryRegisterFileReference` already calls
  * `getFileBuffer()` and reads the whole file to register it. The lazy path was
@@ -2182,45 +2468,69 @@ function getFileSource(file) {
  */
 function isEagerMultimodalFile(file) {
     if (typeof file === "string") {
-        return isImageLikeType(inferFileTypeFromExtension(file));
+        return isEagerType(inferFileTypeFromExtension(file));
     }
     if (Buffer.isBuffer(file)) {
-        return isImageLikeType(inferFileTypeFromBuffer(file));
+        return isEagerType(inferFileTypeFromBuffer(file));
     }
-    // A `FileWithMetadata` carries two independent declarations, and either one
-    // alone is enough: the shape exists for Slack/Curator-style uploads that
+    // A `FileWithMetadata` carries two independent declarations, and either is
+    // enough on its own: the shape exists for Slack/Curator-style uploads that
     // arrive as bytes plus a mimetype, so its `filename` may be extensionless or
-    // simply wrong. Reading them as a `??` chain meant the first *recognised*
-    // name won outright — `upload.pdf` with `mimetype: "image/png"` classified as
-    // a PDF and lost its pixels down the lazy path. Any declaration of an image
-    // is therefore decisive.
+    // simply wrong. Reading them as a `??` chain let the first *recognised* name
+    // win outright — `recording.txt` with `mimetype: "audio/mpeg"` classified as
+    // text, stayed lazy, and never reached the native-audio path this change
+    // exists to feed. A file is kept lazy only when nothing about it disagrees.
     const declared = [
         inferFileTypeFromExtension(file.filename),
         inferFileTypeFromMimetype(file.mimetype),
     ];
-    if (declared.some(isImageLikeType)) {
+    if (declared.some(isEagerType)) {
         return true;
     }
-    // The buffer sniff is the last resort rather than a third vote, because it is
-    // a substring scan: an HTML page with an inline `<svg>` icon in its head
-    // would otherwise be pulled onto the eager path and sent in full, which is
-    // the opposite of what the size tiers are for. It only speaks when nothing
-    // else did.
+    // The buffer sniff is a last resort rather than a third vote, because it is
+    // partly a substring scan: an HTML page with an inline `<svg>` icon in its
+    // head would otherwise be pulled onto the eager path and sent in full, which
+    // is the opposite of what the size tiers are for. It speaks only when neither
+    // declaration did.
     return declared.every((type) => type === undefined)
-        ? isImageLikeType(inferFileTypeFromBuffer(file.buffer))
+        ? isEagerType(inferFileTypeFromBuffer(file.buffer))
         : false;
 }
 /**
- * Whether a routing type should have its bytes preserved rather than previewed.
+ * Whether a routing type must be processed rather than previewed.
  *
  * "svg" is a separate routing type rather than a sub-case of "image" (it goes
  * to the sanitizer, not to a vision encoder), but it is still an image as far
  * as this decision is concerned: its markup IS its content, and previewing it
  * away leaves the model with nothing. Accepting both keeps this correct
  * whichever of the two type vocabularies the caller's map uses.
+ *
+ * Audio and video join them, for one shared reason: the lazy path never runs
+ * the detection branch that produces their model-visible content — the decoded
+ * audio buffer, the extracted video keyframes — so a lazily registered file is
+ * summarised away no matter what the dispatch side is willing to send.
+ *
+ * Video is the clearest demonstration that this is a delivery problem and not a
+ * format one. Keyframe extraction is identical across containers (three frames,
+ * ~38 KB each, for every one of mp4/wmv/flv/mpg/m2ts), and a frame pulled from
+ * any of them shows the test token perfectly legibly. Yet only mp4 answered
+ * correctly, because Gemini accepts an mp4 natively and never needed the
+ * frames; every container it cannot decode returned NOTHING_RECEIVED, since the
+ * frames that would have carried the answer were never extracted. Making video
+ * eager fixed all four at once.
+ *
+ * Documents and archives are here for the same reason once removed. The lazy
+ * preview is a truncated slice of the raw bytes, so it is only ever faithful
+ * for a file that IS text. A .rtf sliced raw is RTF control words, and a
+ * .bz2/.xz/.zst is compressed bytes — the model was told "binary file of
+ * unknown type" about files whose processors extract them cleanly.
+ *
+ * Which leaves plain text and CSV on the lazy path, and they are exactly the
+ * cases it was built for: a truncated sample of a large CSV is a faithful
+ * sample, and the file tools can read the rest on demand.
  */
-function isImageLikeType(type) {
-    return type === "image" || type === "svg";
+function isEagerType(type) {
+    return type !== undefined && type !== "text" && type !== "csv";
 }
 /**
  * Infer a routing type from a caller-declared mimetype.
@@ -2235,10 +2545,24 @@ function inferFileTypeFromMimetype(mimetype) {
         return undefined;
     }
     const normalized = mimetype.split(";")[0].trim().toLowerCase();
-    if (normalized === "image/svg+xml") {
-        return "svg";
+    // "application/octet-stream" is deliberately absent from the registry side of
+    // this lookup: it is the opaque sentinel a caller sends when it knows
+    // nothing, not a claim about content, and treating it as one would let a
+    // shrugging uploader force a routing decision.
+    if (normalized === "application/octet-stream") {
+        return undefined;
     }
-    return normalized.startsWith("image/") ? "image" : undefined;
+    const exact = MIMETYPE_TYPE_MAP[normalized];
+    if (exact) {
+        return exact;
+    }
+    // A family fallback for types the registry does not enumerate — `audio/webm`,
+    // `image/x-something`. The leading segment is enough to decide eager vs lazy
+    // even when the exact codec is unknown to us.
+    const family = normalized.split("/")[0];
+    return family === "image" || family === "audio" || family === "video"
+        ? family
+        : undefined;
 }
 /**
  * Try to register a file with the FileReferenceRegistry for lazy processing.