@juspay/neurolink 10.11.2 → 10.12.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [10.12.0](https://github.com/juspay/neurolink/compare/v10.11.3...v10.12.0) (2026-08-13)
+
+### Features
+
+- **(proxy):** manual limits refresh with dynamic quota windows ([e5c9fed](https://github.com/juspay/neurolink/commit/e5c9fed0c570a4583d8e83e04d09785e3e126d56))
+
+## [10.11.3](https://github.com/juspay/neurolink/compare/v10.11.2...v10.11.3) (2026-08-12)
+
+### Bug Fixes
+
+- **(multimodal):** deliver file content to the model instead of describing it ([6c2b2e6](https://github.com/juspay/neurolink/commit/6c2b2e6f96fcb0ab251256d905560e2761a7c8c0))
+
 ## [10.11.2](https://github.com/juspay/neurolink/compare/v10.11.1...v10.11.2) (2026-08-12)
 
 ### Bug Fixes

package/dist/adapters/audioFormatSupport.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Native audio delivery to providers that can listen.
+ *
+ * ## The gap this closes
+ *
+ * Until this module existed, attaching an audio file produced a message
+ * containing only a metadata block:
+ *
+ *   ## Audio File: "recording.mp3"
+ *   Duration: 19s | Codec: MPEG 2 Layer 3 | Bitrate: 32 kbps |
+ *   Sample Rate: 22050 Hz | Channels: 1 (Mono)
+ *
+ * No audio bytes were ever handed to the provider. Every question about what
+ * the recording *says* — transcribe this, who is speaking, what was agreed —
+ * was answered from a description of the file, and Gemini has accepted inline
+ * audio the whole time.
+ *
+ * The failure was invisible for an instructive reason: that metadata block
+ * answers precisely the questions a test is most tempted to ask. "How long is
+ * this audio?" and "what sample rate is it?" both succeed with no audio
+ * attached, so a suite built on them reports working audio support. It took an
+ * end-to-end test asking for a spoken word to expose it.
+ *
+ * ## Provider scope
+ *
+ * Deliberately a capability map rather than "send audio to everyone". A
+ * provider that cannot accept an audio part responds with an opaque HTTP 400,
+ * which is worse than the metadata summary it would otherwise have received —
+ * so an unlisted provider keeps the existing text-only behaviour and loses
+ * nothing.
+ *
+ * @module adapters/audioFormatSupport
+ */
+import type { AudioConversionResult } from "../types/index.js";
+/** Whether `provider` can be handed raw audio bytes. */
+export declare function supportsNativeAudio(provider: string): boolean;
+/** Whether `mimeType` must be re-encoded before a native provider will read it. */
+export declare function needsAudioTranscode(mimeType: string): boolean;
+/**
+ * Return audio bytes a native provider can read, transcoding when the source
+ * container is one it does not accept.
+ *
+ * Never throws for audio reasons. When conversion is impossible — no ffmpeg, an
+ * unreadable stream — the original bytes and MIME type come back with
+ * `converted: false`, and the caller falls back to the metadata summary. That
+ * keeps this from turning a previously-working (if limited) request into a
+ * failure.
+ *
+ * @param buffer - Raw audio bytes.
+ * @param mimeType - Detected MIME type of `buffer`.
+ * @param extension - Source extension, used so ffmpeg picks the right demuxer.
+ */
+export declare function toProviderCompatibleAudio(buffer: Buffer, mimeType: string, extension: string): Promise<AudioConversionResult>;

package/dist/adapters/audioFormatSupport.js

@@ -0,0 +1,200 @@
+/**
+ * Native audio delivery to providers that can listen.
+ *
+ * ## The gap this closes
+ *
+ * Until this module existed, attaching an audio file produced a message
+ * containing only a metadata block:
+ *
+ *   ## Audio File: "recording.mp3"
+ *   Duration: 19s | Codec: MPEG 2 Layer 3 | Bitrate: 32 kbps |
+ *   Sample Rate: 22050 Hz | Channels: 1 (Mono)
+ *
+ * No audio bytes were ever handed to the provider. Every question about what
+ * the recording *says* — transcribe this, who is speaking, what was agreed —
+ * was answered from a description of the file, and Gemini has accepted inline
+ * audio the whole time.
+ *
+ * The failure was invisible for an instructive reason: that metadata block
+ * answers precisely the questions a test is most tempted to ask. "How long is
+ * this audio?" and "what sample rate is it?" both succeed with no audio
+ * attached, so a suite built on them reports working audio support. It took an
+ * end-to-end test asking for a spoken word to expose it.
+ *
+ * ## Provider scope
+ *
+ * Deliberately a capability map rather than "send audio to everyone". A
+ * provider that cannot accept an audio part responds with an opaque HTTP 400,
+ * which is worse than the metadata summary it would otherwise have received —
+ * so an unlisted provider keeps the existing text-only behaviour and loses
+ * nothing.
+ *
+ * @module adapters/audioFormatSupport
+ */
+import { withTimeout } from "../utils/errorHandling.js";
+import { logger } from "../utils/logger.js";
+import { getFfmpegPath, runFfmpeg } from "./video/ffmpegAdapter.js";
+/**
+ * Ceiling for one audio conversion.
+ *
+ * Longer than the image equivalent because a lossless hour-long WAV is a
+ * legitimate input and re-encoding it is not instant, but still bounded so a
+ * wedged decoder cannot hold a generation request open indefinitely.
+ */
+const AUDIO_TRANSCODE_TIMEOUT_MS = 120_000;
+/**
+ * Providers that accept inline audio parts.
+ *
+ * Google's Gemini models (both Vertex and AI Studio) take audio as `inlineData`
+ * alongside text. Other providers are omitted rather than assumed: OpenAI's
+ * audio models use a different request shape than the chat-completions path
+ * NeuroLink builds here, and sending an audio part to a provider that does not
+ * expect one converts a working (if limited) response into a hard failure.
+ */
+const NATIVE_AUDIO_PROVIDERS = new Set([
+    "vertex",
+    "google-vertex",
+    "googlevertex",
+    "google-ai-studio",
+    "googleaistudio",
+    "google-ai",
+    "googleai",
+    "gemini",
+]);
+/**
+ * Audio MIME types the native providers accept as-is.
+ *
+ * Gemini's documented set. Anything outside it is transcoded rather than
+ * rejected, because the container a user happens to have — a voice memo in
+ * CAF, a Windows recording in WMA — says nothing about whether the audio
+ * inside is useful.
+ */
+const NATIVE_AUDIO_MIME_TYPES = new Set([
+    "audio/wav",
+    "audio/x-wav",
+    "audio/mpeg",
+    "audio/mp3",
+    "audio/aiff",
+    "audio/x-aiff",
+    "audio/aac",
+    "audio/ogg",
+    "audio/flac",
+    "audio/x-flac",
+]);
+/** MIME type every transcode targets. Universally accepted and compact. */
+const TRANSCODE_TARGET_MIME = "audio/mpeg";
+/** Whether `provider` can be handed raw audio bytes. */
+export function supportsNativeAudio(provider) {
+    return NATIVE_AUDIO_PROVIDERS.has(provider.toLowerCase().trim());
+}
+/** Whether `mimeType` must be re-encoded before a native provider will read it. */
+export function needsAudioTranscode(mimeType) {
+    return !NATIVE_AUDIO_MIME_TYPES.has(normalizeAudioMime(mimeType));
+}
+function normalizeAudioMime(mimeType) {
+    return mimeType.split(";")[0].trim().toLowerCase();
+}
+/**
+ * Re-encode audio to MP3 with ffmpeg.
+ *
+ * Temp files rather than stdin: several of the containers that need converting
+ * (CAF, WavPack, AU) carry their metadata in a trailer or require seeking, and
+ * a piped stream leaves ffmpeg unable to find it. The directory is removed in
+ * `finally` whether or not the conversion succeeded.
+ *
+ * Node builtins are imported dynamically because the browser bundle stubs
+ * `node:fs/promises` without `mkdtemp`; nothing in a browser spawns ffmpeg, so
+ * the import belongs at the point of use.
+ */
+async function transcodeToMp3(buffer, extension) {
+    const [{ randomUUID }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join },] = await Promise.all([
+        import("node:crypto"),
+        import("node:fs/promises"),
+        import("node:os"),
+        import("node:path"),
+    ]);
+    const workDir = await mkdtemp(join(tmpdir(), "neurolink-audio-"));
+    const inputPath = join(workDir, `${randomUUID()}${extension}`);
+    const outputPath = join(workDir, `${randomUUID()}.mp3`);
+    try {
+        await writeFile(inputPath, buffer);
+        await runFfmpeg([
+            "-y",
+            "-v",
+            "error",
+            "-i",
+            inputPath,
+            // Downmix and cap the rate: speech is the point, and a 48 kHz stereo
+            // re-encode of a mono voice memo triples the payload for nothing.
+            "-ac",
+            "1",
+            "-ar",
+            "16000",
+            "-c:a",
+            "libmp3lame",
+            "-q:a",
+            "4",
+            outputPath,
+        ], 
+        // Without this the call inherits runFfmpeg's frame-extraction default of
+        // 30s, which is sized for pulling a single video frame — so the 120s
+        // ceiling above, chosen precisely because re-encoding a lossless
+        // hour-long WAV is not instant, could never be reached. ffmpeg killed the
+        // transcode at 30s and the outer race never got to run.
+        { timeoutMs: AUDIO_TRANSCODE_TIMEOUT_MS });
+        return await readFile(outputPath);
+    }
+    finally {
+        await rm(workDir, { recursive: true, force: true }).catch(() => undefined);
+    }
+}
+/**
+ * Return audio bytes a native provider can read, transcoding when the source
+ * container is one it does not accept.
+ *
+ * Never throws for audio reasons. When conversion is impossible — no ffmpeg, an
+ * unreadable stream — the original bytes and MIME type come back with
+ * `converted: false`, and the caller falls back to the metadata summary. That
+ * keeps this from turning a previously-working (if limited) request into a
+ * failure.
+ *
+ * @param buffer - Raw audio bytes.
+ * @param mimeType - Detected MIME type of `buffer`.
+ * @param extension - Source extension, used so ffmpeg picks the right demuxer.
+ */
+export async function toProviderCompatibleAudio(buffer, mimeType, extension) {
+    const normalized = normalizeAudioMime(mimeType);
+    if (!needsAudioTranscode(normalized)) {
+        return { buffer, mimeType: normalized, converted: false };
+    }
+    // Resolving the binary first turns "ffmpeg is not installed" into one clear
+    // warning rather than a spawn error surfacing from inside the conversion.
+    const ffmpegAvailable = await getFfmpegPath()
+        .then(() => true)
+        .catch(() => false);
+    if (!ffmpegAvailable) {
+        logger.warn(`[audioFormatSupport] ${normalized} needs conversion before a provider can ` +
+            `read it, but ffmpeg is unavailable — falling back to a metadata-only ` +
+            `summary. Install ffmpeg (or set FFMPEG_PATH) to enable this format.`);
+        return { buffer, mimeType: normalized, converted: false };
+    }
+    try {
+        const converted = await withTimeout(transcodeToMp3(buffer, extension), AUDIO_TRANSCODE_TIMEOUT_MS, new Error(`audio transcode exceeded ${AUDIO_TRANSCODE_TIMEOUT_MS}ms`));
+        if (converted.length === 0) {
+            throw new Error("produced an empty audio stream");
+        }
+        logger.debug(`[audioFormatSupport] Transcoded ${normalized} → ${TRANSCODE_TARGET_MIME} ` +
+            `(${buffer.length} → ${converted.length} bytes) for native delivery`);
+        return {
+            buffer: converted,
+            mimeType: TRANSCODE_TARGET_MIME,
+            converted: true,
+        };
+    }
+    catch (error) {
+        logger.warn(`[audioFormatSupport] Could not convert ${normalized} for native delivery ` +
+            `— falling back to a metadata-only summary: ` +
+            `${error instanceof Error ? error.message.split("\n")[0] : String(error)}`);
+        return { buffer, mimeType: normalized, converted: false };
+    }
+}