@juspay/neurolink 12.31.1 → 12.32.1

package/dist/types/processor.d.ts

@@ -714,6 +714,25 @@ export type ProcessedAudio = ProcessedFileBase & {
     transcript?: string;
     hasTranscript: boolean;
     transcriptionProvider?: string;
+    /**
+     * Language the transcription backend reported for the speech (#409), when
+     * it reported one.
+     *
+     * Not always a *detected* language: when the backend's response carries no
+     * language (Whisper's `verbose_json` usually does; Google/Azure responses
+     * vary), this falls back to the caller-requested `options.language` instead
+     * of going unset. A caller that must distinguish "the backend detected X"
+     * from "X is just what was asked for" cannot do so from this field alone.
+     */
+    transcriptionLanguage?: string;
+    /**
+     * Audio duration in seconds as measured by the transcription backend (#409).
+     *
+     * Kept apart from `metadata.duration`, which comes from the container
+     * header: the two disagree on a file with a broken or absent header, and the
+     * header is the one that is available without a transcription call.
+     */
+    transcriptionDuration?: number;
     /**
      * Why transcription produced nothing, when it did (#416). Absent on success.
      * Lets a caller distinguish "this audio has no speech" from "the transcription

package/dist/types/stream.d.ts

@@ -401,6 +401,23 @@ export type StreamOptions = {
          */
         outputFormat?: VisionImageOutputFormat;
     };
+    /**
+     * Audio transcription options for attached audio files (#413/#440).
+     *
+     * Mirrors `GenerateOptions.audioOptions`. `stream()` spreads its options
+     * rather than rebuilding them field by field, so declaring it here is what
+     * lets it reach the message builder.
+     */
+    audioOptions?: {
+        /** Backend: "openai" (aliases "whisper"), "google" or "azure". */
+        provider?: string;
+        /** Transcription model, e.g. "whisper-1". Backend-specific. */
+        transcriptionModel?: string;
+        /** Language hint, e.g. "en". */
+        language?: string;
+        /** OpenAI/Whisper-only context prompt to bias transcription; ignored by Google and Azure. */
+        prompt?: string;
+    };
     /**
      * Text-to-Speech (TTS) configuration for streaming
      *

package/dist/utils/fileDetector.js

@@ -340,15 +340,13 @@ export class FileDetector {
                 }
                 logger.warn(`[FileDetector] All fallback parsing failed for type "${detection.type}". ` +
                     `Attempted: ${options.allowedTypes.join(", ")}. Falling through to universal handler.`);
-                const csvOptions = options?.csvOptions;
-                const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider, options?.videoOptions, options?.officeOptions);
+                const result = await FileDetector.processFile(content, detection, options);
                 FileDetector.setFileResultSpanAttributes(span, result, inputFilename, detection.type);
                 return result;
             }
             const content = loadedUrl?.content ??
                 (await FileDetector.loadContent(input, detection, options));
-            const csvOptions = options?.csvOptions;
-            const result = await FileDetector.processFile(content, detection, csvOptions, options?.provider, options?.videoOptions, options?.officeOptions);
+            const result = await FileDetector.processFile(content, detection, options);
             FileDetector.setFileResultSpanAttributes(span, result, inputFilename, detection.type);
             return result;
         });
@@ -1063,31 +1061,33 @@ export class FileDetector {
     /**
      * Route to appropriate processor
      */
-    static async processFile(content, detection, options, provider, videoOptions, officeOptions) {
+    static async processFile(content, detection, options) {
         switch (detection.type) {
             case "csv":
                 // Pass original extension through to CSV processor; if detection has none,
                 // fall back to any extension provided in csvOptions.
                 return await CSVProcessor.process(content, {
-                    ...options,
-                    extension: detection.extension ?? options?.extension,
+                    ...options?.csvOptions,
+                    extension: detection.extension ?? options?.csvOptions?.extension,
                 });
             case "image":
                 return await ImageProcessor.process(content);
             case "pdf":
-                return await PDFProcessor.process(content, { provider });
+                return await PDFProcessor.process(content, {
+                    provider: options?.provider,
+                });
             case "svg":
                 // SVG is processed as text content (sanitized XML markup)
                 // AI providers don't support SVG as image format, so we extract text content
                 return await FileDetector.processSvgAsText(content, detection);
             case "video":
-                return await FileDetector.processVideoFile(content, detection, videoOptions);
+                return await FileDetector.processVideoFile(content, detection, options?.videoOptions);
             case "audio":
-                return await FileDetector.processAudioFile(content, detection);
+                return await FileDetector.processAudioFile(content, detection, options?.audioOptions);
             case "archive":
                 return await FileDetector.processArchiveFile(content, detection);
             case "xlsx":
-                return await FileDetector.processXlsxFile(content, detection, officeOptions);
+                return await FileDetector.processXlsxFile(content, detection, options?.officeOptions);
             case "docx":
                 return await FileDetector.processDocxFile(content, detection);
             case "pptx":
@@ -1197,7 +1197,7 @@ export class FileDetector {
     /**
      * Process audio file: extract metadata, tags, and cover art via AudioProcessor
      */
-    static async processAudioFile(content, detection) {
+    static async processAudioFile(content, detection, audioOptions) {
         const audioFilename = detection.metadata.filename || "audio";
         try {
             const audioResult = await (await getAudioProcessor()).processFile({
@@ -1206,18 +1206,40 @@ export class FileDetector {
                 mimetype: detection.mimeType || "audio/mpeg",
                 size: content.length,
                 buffer: content,
-            });
+            }, 
+            // #440: carry the caller's transcription provider/language/prompt
+            // through to the processor; previously these stopped here.
+            audioOptions);
             if (audioResult.success && audioResult.data) {
+                const audio = audioResult.data;
                 return {
                     type: "audio",
-                    content: audioResult.data.textContent ||
+                    content: audio.textContent ||
                         FileDetector.formatInformativePlaceholder("Audio", audioFilename, content, detection),
                     mimeType: detection.mimeType,
                     // Surface embedded cover art as an image content block
-                    images: audioResult.data.coverArt
-                        ? [audioResult.data.coverArt]
-                        : undefined,
-                    metadata: detection.metadata,
+                    images: audio.coverArt ? [audio.coverArt] : undefined,
+                    metadata: {
+                        ...detection.metadata,
+                        // #409: the processor's audio metadata used to stop here —
+                        // `detection.metadata` alone carries only size/filename/confidence,
+                        // so duration and everything about the transcript were dropped by
+                        // the very call that produced them.
+                        ...(audio.metadata.duration > 0
+                            ? { duration: audio.metadata.duration }
+                            : {}),
+                        ...(audio.transcriptionLanguage
+                            ? { language: audio.transcriptionLanguage }
+                            : {}),
+                        // Written whenever a backend actually ran, so 0 (a real empty
+                        // transcript) stays distinguishable from "never attempted".
+                        ...(audio.transcriptionProvider
+                            ? {
+                                transcriptionLength: audio.transcript?.length ?? 0,
+                                transcriptionProvider: audio.transcriptionProvider,
+                            }
+                            : {}),
+                    },
                 };
             }
         }

package/dist/utils/messageBuilder.js

@@ -11,7 +11,7 @@ import { enforceAggregateFileBudget, FILE_READ_BUDGET_PERCENT, } from "../contex
 import { isCSVContent, isTextContent, SIZE_TIER_THRESHOLDS, } from "../types/index.js";
 import { tracers, ATTR, withSpan } from "../telemetry/index.js";
 import { EXIF_PROBE_PREFIX_BYTES, mayCarryExifOrientation, needsVisionTranscode, normalizeImageOrientation, probeExifOrientation, toVisionCompatibleImage, } from "../adapters/imageFormatSupport.js";
-import { FILE_TYPE_REGISTRY, lookupByExtension, } from "../processors/config/fileTypeRegistry.js";
+import { extensionsForModality, FILE_TYPE_REGISTRY, lookupByExtension, } from "../processors/config/fileTypeRegistry.js";
 import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js";
 import { FileDetector } from "./fileDetector.js";
 import { detectIsoBmffImageMimeType } from "./isoBmff.js";
@@ -911,11 +911,41 @@ async function detectFileForUnifiedArray(file, options, provider, genericFileMax
                 transcribeAudio: options.videoOptions.transcribeAudio,
             }
             : undefined,
+        // #413/#440: same story for audio — the transcription backend, language
+        // and prompt a caller chose have to reach AudioProcessor, which is
+        // several hops down from here.
+        audioOptions: options.audioOptions
+            ? {
+                provider: options.audioOptions.provider,
+                transcriptionModel: options.audioOptions.transcriptionModel,
+                language: options.audioOptions.language,
+                prompt: options.audioOptions.prompt,
+            }
+            : undefined,
         provider: provider,
         mimetypeHint: fileMimetypeHint,
         filenameHint: fileFilenameHint,
     });
 }
+/**
+ * Guidance appended to the system prompt when an audio file is attached (#471).
+ *
+ * An audio attachment does not reach the model as audio on most providers: the
+ * processor transcribes it and inlines the transcript as text. Without saying
+ * so, models reliably answer "I cannot listen to audio" while the transcript
+ * of that very recording sits in the prompt above — the audio equivalent of
+ * the "no files attached" failure the CSV/PDF guidance exists to prevent.
+ *
+ * It also states the negative case, because a file whose transcription was
+ * skipped (no backend configured, unsupported format) still arrives, carrying
+ * metadata and a stated reason. A model told only "audio is transcribed" will
+ * otherwise invent speech content for a file that has none.
+ */
+const AUDIO_PROMPT_EXTENSIONS = extensionsForModality("audio");
+const AUDIO_TRANSCRIPTION_INSTRUCTIONS = `
+- Any audio file has ALREADY BEEN PROCESSED for you: its metadata, and its transcript when one could be produced, are inlined above under "## Audio File: ..." — you do not need to, and cannot, listen to the recording yourself.
+- Treat the text under "--- Transcript ---" as the verbatim spoken content of that recording, and answer from it directly. Do NOT reply that you are unable to process audio.
+- If an audio file shows no transcript, transcription was skipped and the stated reason is inlined with it. Say so rather than inventing what the audio might contain.`;
 export async function processUnifiedFilesArray(options, maxSize, provider) {
     options.input ??= {};
     if (!options.input.files || options.input.files.length === 0) {
@@ -962,6 +992,12 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
         inp2.text = inp2.text || "";
         let includedCount = 0;
         const fileRegistry = options.fileRegistry;
+        // #471: the system-prompt augmentation below has to know whether any
+        // audio actually made it in. Counted from the detection result rather
+        // than from `inp2.nativeAudioFiles`, which is only populated when the
+        // raw bytes could also be re-read — an audio file whose transcript was
+        // inlined but whose bytes were not would otherwise go unmentioned.
+        let audioIncludedCount = 0;
         // Only detection runs concurrently, and only for files that will not
         // take the lazy-registration branch.
         //
@@ -1022,6 +1058,9 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
                     (await detectFileForUnifiedArray(file, options, provider, genericFileMaxSize));
                 await appendDetectedFileResult(result, file, options);
                 includedCount++;
+                if (result.type === "audio") {
+                    audioIncludedCount++;
+                }
                 // Log what content type was added to the message
                 const contentType = result.type === "image" ? "image" : "text";
                 logger.info(`[NEUROLINK] File added to message: ${filename} as ${contentType} (type: ${result.type})`);
@@ -1065,7 +1104,7 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
 - The full content of the user's local file(s) is INLINED in this message under "## CSV Data from ..." / "## PDF Data from ..." / "## File: ..." headings — it is the actual file the user is asking about.
 - TREAT THE INLINED CONTENT AS IF IT WERE AN ATTACHMENT. Do NOT respond with "no files attached" or ask the user to re-upload — the data is already here.
 - DO NOT use GitHub tools (get_file_contents, search_code, etc.) for local files - they only work for remote repository files.
-- Analyze the inlined file content directly without attempting to fetch or read files using tools.`;
+- Analyze the inlined file content directly without attempting to fetch or read files using tools.${audioIncludedCount > 0 ? AUDIO_TRANSCRIPTION_INSTRUCTIONS : ""}`;
             const existingSystem = (options.systemPrompt || "").trim();
             options.systemPrompt = existingSystem
                 ? `${existingSystem}${filePromptAugmentation}`
@@ -1320,7 +1359,20 @@ function buildMultimodalSystemPrompt(options, hasPDFFiles) {
     const hasCSVFiles = (inp.csvFiles && inp.csvFiles.length > 0) ||
         (inp.files &&
             inp.files.some((f) => typeof f === "string" ? f.toLowerCase().endsWith(".csv") : false));
-    if (hasCSVFiles || hasPDFFiles) {
+    // #471: audio attachments. `processUnifiedFilesArray` has already run by the
+    // time this is called, so `nativeAudioFiles` is the authoritative signal —
+    // it is populated from the detector's verdict, which reads magic bytes and
+    // therefore recognises an audio file whose name says nothing. The remaining
+    // checks cover the paths that never reach the detector: an explicit
+    // `audioFiles` alias (folded away by `foldMediaAliasesIntoFiles`, but only
+    // once that has run) and a plain path attached by name.
+    const hasAudioFiles = (inp.nativeAudioFiles && inp.nativeAudioFiles.length > 0) ||
+        (inp.audioFiles && inp.audioFiles.length > 0) ||
+        (inp.files &&
+            inp.files.some((f) => typeof f === "string"
+                ? AUDIO_PROMPT_EXTENSIONS.some((ext) => f.toLowerCase().endsWith(ext))
+                : false));
+    if (hasCSVFiles || hasPDFFiles || hasAudioFiles) {
         const fileTypes = [];
         if (hasPDFFiles) {
             fileTypes.push("PDFs");
@@ -1328,13 +1380,22 @@ function buildMultimodalSystemPrompt(options, hasPDFFiles) {
         if (hasCSVFiles) {
             fileTypes.push("CSVs");
         }
+        if (hasAudioFiles) {
+            // Not "(transcribed)": `hasAudioFiles` only means an audio file was
+            // attached, not that transcription actually produced anything — a file
+            // with no backend configured, an unavailable pinned backend, or a
+            // format/size the backend rejects still arrives here with metadata
+            // only. AUDIO_TRANSCRIPTION_INSTRUCTIONS covers the per-file case; this
+            // list must not assert the opposite of what it says.
+            fileTypes.push("audio files (transcript may be unavailable)");
+        }
         systemPrompt += `\n\nIMPORTANT FILE HANDLING INSTRUCTIONS:
-- The full content of the user's local ${fileTypes.join(", ")} (and any images) is INLINED in this message under the "## CSV Data from ..." / "## PDF Data from ..." headings — it is the actual file the user is asking about.
+- The full content of the user's local ${fileTypes.join(", ")} (and any images) is INLINED in this message under the "## CSV Data from ..." / "## PDF Data from ..." / "## Audio File: ..." headings — it is the actual file the user is asking about.
 - TREAT THE INLINED CONTENT AS IF IT WERE AN ATTACHMENT. Do NOT respond with "no files attached" or ask the user to re-upload — the data is already here.
 - DO NOT use GitHub tools (get_file_contents, search_code, etc.) for local files - they only work for remote repository files
 - Analyze the provided file content directly without attempting to fetch or read files using tools
 - GitHub MCP tools are ONLY for remote repository operations, not local filesystem access
-- Use the file content shown in this message for your analysis`;
+- Use the file content shown in this message for your analysis${hasAudioFiles ? AUDIO_TRANSCRIPTION_INSTRUCTIONS : ""}`;
     }
     return systemPrompt;
 }