@juspay/neurolink 12.26.3 → 12.27.0

package/dist/index.d.ts

@@ -72,6 +72,7 @@ export { AzureTTS, AzureTTSHandler, CartesiaTTS, CartesiaTTSHandler, ElevenLabsT
 export { BeatovenMusic, BeatovenMusicHandler, ElevenLabsMusic, ElevenLabsMusicHandler, LyriaMusic, LyriaMusicHandler, registerDefaultMusicHandlers, ReplicateMusic, ReplicateMusicHandler, } from "./music/index.js";
 export { DIDAvatar, DIDAvatarHandler, HeyGenAvatar, HeyGenAvatarHandler, registerDefaultAvatarHandlers, ReplicateAvatar, ReplicateAvatarHandler, } from "./avatar/index.js";
 export { isVertexVideoConfigured, KlingVideoHandler, registerDefaultVideoHandlers, ReplicateVideoHandler, RunwayVideoHandler, VertexVideoHandler, } from "./adapters/video/index.js";
+export { canDeliverVideoNatively, estimateVideoTokens, getVideoProviderConfig, isNativeVideoMimeType, supportsNativeVideo, VIDEO_PROVIDER_CONFIGS, } from "./adapters/videoFormatSupport.js";
 export { ImageGenService } from "./image-gen/ImageGenService.js";
 export { detectImageMimeType, sniffImageMimeType, } from "./utils/imageDetection.js";
 export { HITLManager } from "./hitl/hitlManager.js";

package/dist/index.js

@@ -145,6 +145,13 @@ export { BeatovenMusic, BeatovenMusicHandler, ElevenLabsMusic, ElevenLabsMusicHa
 export { DIDAvatar, DIDAvatarHandler, HeyGenAvatar, HeyGenAvatarHandler, registerDefaultAvatarHandlers, ReplicateAvatar, ReplicateAvatarHandler, } from "./avatar/index.js";
 // Video handlers
 export { isVertexVideoConfigured, KlingVideoHandler, registerDefaultVideoHandlers, ReplicateVideoHandler, RunwayVideoHandler, VertexVideoHandler, } from "./adapters/video/index.js";
+// Video *ingestion* capabilities — what each provider does with an attached
+// clip. Surfaced because the answer changes what a caller should send: a
+// provider on the native path is billed by duration and reads the audio
+// track, one on the frame path is billed per keyframe and hears nothing, and
+// without a way to ask, the only way to find out is to send a video and read
+// the reply.
+export { canDeliverVideoNatively, estimateVideoTokens, getVideoProviderConfig, isNativeVideoMimeType, supportsNativeVideo, VIDEO_PROVIDER_CONFIGS, } from "./adapters/videoFormatSupport.js";
 // Image generation + HITL — surfaced from their dedicated barrels
 export { ImageGenService } from "./image-gen/ImageGenService.js";
 export { detectImageMimeType, sniffImageMimeType, } from "./utils/imageDetection.js";

package/dist/providers/googleAiStudio/client.js

@@ -665,7 +665,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
                 // reported "no image attached" on multimodal calls.
                 const currentContents = [];
                 prependConversationMessages(currentContents, options.conversationMessages);
-                const userParts = await buildUserPartsWithMultimodal(options.input, options.input.text, "[GoogleAIStudio:stream]");
+                const userParts = await buildUserPartsWithMultimodal(options.input, options.input.text, "[GoogleAIStudio:stream]", modelName);
                 currentContents.push({
                     role: "user",
                     parts: userParts,
@@ -1220,7 +1220,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
                 // legitimately reported "no image / PDF attached".
                 const currentContents = [];
                 prependConversationMessages(currentContents, options.conversationMessages);
-                const userParts = await buildUserPartsWithMultimodal(options.input, promptText, "[GoogleAIStudio:generate]");
+                const userParts = await buildUserPartsWithMultimodal(options.input, promptText, "[GoogleAIStudio:generate]", modelName);
                 currentContents.push({
                     role: "user",
                     parts: userParts,

package/dist/providers/googleNativeGemini3/utils.d.ts

@@ -8,7 +8,7 @@
  * This module extracts the functions that are duplicated between the two
  * providers so they can share a single implementation.
  */
-import type { GenerateStopReason, ThinkingConfig, AgenticLoopOptions, ChatMessage, CollectedChunkResult, MinimalChatMessage, NativeFunctionCall, NativeFunctionResponse, NativeToolDeclarationsResult, NativeToolsConfig, StreamChannel, VertexNativePart, GeminiMultimodalInput, MultimodalAudioEntry } from "../../types/index.js";
+import type { GenerateStopReason, ThinkingConfig, AgenticLoopOptions, ChatMessage, CollectedChunkResult, MinimalChatMessage, NativeFunctionCall, NativeFunctionResponse, NativeToolDeclarationsResult, NativeToolsConfig, StreamChannel, VertexNativePart, GeminiMultimodalInput, MultimodalAudioEntry, MultimodalVideoEntry } from "../../types/index.js";
 import type { Tool, ToolExecutionGuards } from "../../types/index.js";
 /**
  * A per-turn tool execute map that deduplicates identical tool calls.
@@ -466,4 +466,26 @@ export declare function prependConversationMessages(contents: Array<{
  * summary in the text part.
  */
 export declare function appendNativeAudioParts(userParts: VertexNativePart[], audioFiles: MultimodalAudioEntry[] | undefined, logPrefix?: string): Promise<void>;
-export declare function buildUserPartsWithMultimodal(input: GeminiMultimodalInput | undefined, textOverride?: string, logPrefix?: string): Promise<VertexNativePart[]>;
+/**
+ * Append video to a Gemini request as `inlineData` parts.
+ *
+ * The native request body is assembled by hand rather than taken from the AI
+ * SDK's `file` parts, so — exactly as with audio — a `{ type: "file" }` part
+ * built upstream never arrives here. Video has to be added explicitly or it
+ * is silently dropped on both Gemini front ends while the capability table
+ * says it is supported.
+ *
+ * A clip the provider will not take inline is skipped rather than sent: an
+ * oversized or unsupported `inlineData` part fails the whole request, whereas
+ * skipping leaves the caller with the metadata summary and the keyframes,
+ * which is what it had before native delivery existed.
+ *
+ * `model` is forwarded to `canDeliverVideoNatively` so a Claude-on-Vertex
+ * request — same `providerName` ("vertex") as a Gemini-on-Vertex one, routed
+ * to the native Anthropic SDK instead by `GoogleVertexProvider.isAnthropicModel`
+ * — is answered as not natively video-capable rather than inheriting Vertex's
+ * Gemini row. Optional because AI Studio's caller never has a Claude model to
+ * pass.
+ */
+export declare function appendNativeVideoParts(userParts: VertexNativePart[], videoFiles: MultimodalVideoEntry[] | undefined, providerName: string, model?: string, logPrefix?: string): Promise<void>;
+export declare function buildUserPartsWithMultimodal(input: GeminiMultimodalInput | undefined, textOverride?: string, logPrefix?: string, model?: string): Promise<VertexNativePart[]>;

package/dist/providers/googleNativeGemini3/utils.js

@@ -13,6 +13,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { extname } from "node:path";
 import { DEFAULT_CONTEXT_GUARD_RATIO, DEFAULT_MAX_STEPS, DEFAULT_TOOL_MAX_RETRIES, DEFAULT_WRAPUP_TIME_LEAD_MS, } from "../../core/constants.js";
 import { needsAudioTranscode, toProviderCompatibleAudio, } from "../../adapters/audioFormatSupport.js";
+import { canDeliverVideoNatively } from "../../adapters/videoFormatSupport.js";
 import { logger } from "../../utils/logger.js";
 import { guardToolExecutor } from "../../core/toolExecutionGuards.js";
 import { resolveSamplingParams } from "../../models/modelRegistry.js";
@@ -1376,7 +1377,73 @@ export async function appendNativeAudioParts(userParts, audioFiles, logPrefix =
         logger.debug(`${logPrefix} Added native audio part for ${base} (${compatible.mimeType})`);
     }
 }
-export async function buildUserPartsWithMultimodal(input, textOverride, logPrefix = "[GeminiNative]") {
+/**
+ * Append video to a Gemini request as `inlineData` parts.
+ *
+ * The native request body is assembled by hand rather than taken from the AI
+ * SDK's `file` parts, so — exactly as with audio — a `{ type: "file" }` part
+ * built upstream never arrives here. Video has to be added explicitly or it
+ * is silently dropped on both Gemini front ends while the capability table
+ * says it is supported.
+ *
+ * A clip the provider will not take inline is skipped rather than sent: an
+ * oversized or unsupported `inlineData` part fails the whole request, whereas
+ * skipping leaves the caller with the metadata summary and the keyframes,
+ * which is what it had before native delivery existed.
+ *
+ * `model` is forwarded to `canDeliverVideoNatively` so a Claude-on-Vertex
+ * request — same `providerName` ("vertex") as a Gemini-on-Vertex one, routed
+ * to the native Anthropic SDK instead by `GoogleVertexProvider.isAnthropicModel`
+ * — is answered as not natively video-capable rather than inheriting Vertex's
+ * Gemini row. Optional because AI Studio's caller never has a Claude model to
+ * pass.
+ */
+export async function appendNativeVideoParts(userParts, videoFiles, providerName, model, logPrefix = "[GeminiNative]") {
+    if (!videoFiles || videoFiles.length === 0) {
+        return;
+    }
+    // Tracks source bytes already committed to this request's shared inline
+    // budget. This is the sole place video ceiling enforcement happens: the AI
+    // SDK `file`-part assembly in messageBuilder.ts never delivers video for
+    // either Gemini provider class (both own their request assembly end to
+    // end), so a duplicate of this check used to sit there too — permanently
+    // inert and unreachable by any test — and has been removed rather than
+    // left as a second, silently-driftable copy of the same logic.
+    //
+    // Seeded from the PDF, image and audio `inlineData` parts the caller has
+    // already pushed onto `userParts` before this runs, not just from sibling
+    // video clips: the 20MB Gemini request ceiling is shared by every inline
+    // part, so a clip that would fit alone can still overflow the request
+    // once those other parts are counted. `data` is base64, which costs 4
+    // bytes per 3 of source — decode the length back to source bytes so this
+    // total stays comparable to `video.buffer.length` below.
+    let committedVideoBytes = userParts.reduce((total, part) => {
+        if ("inlineData" in part) {
+            return total + Math.floor((part.inlineData.data.length * 3) / 4);
+        }
+        return total;
+    }, 0);
+    for (const video of videoFiles) {
+        // Split on both separators: a Windows-style name reaching a POSIX host
+        // would otherwise keep its whole path in the log line.
+        const base = video.filename.split(/[\\/]/).pop() ?? video.filename;
+        const decision = canDeliverVideoNatively(providerName, video, committedVideoBytes, model);
+        if (!decision.deliver) {
+            logger.warn(`${logPrefix} Sending keyframes instead of the clip for ${base}: ` +
+                `${decision.reason}.`);
+            continue;
+        }
+        committedVideoBytes += video.buffer.length;
+        userParts.push({
+            inlineData: {
+                mimeType: video.mimeType,
+                data: video.buffer.toString("base64"),
+            },
+        });
+        logger.debug(`${logPrefix} Added native video part for ${base} (${video.mimeType})`);
+    }
+}
+export async function buildUserPartsWithMultimodal(input, textOverride, logPrefix = "[GeminiNative]", model) {
     const text = typeof textOverride === "string" ? textOverride : (input?.text ?? "");
     const parts = [{ text }];
     if (input?.pdfFiles && input.pdfFiles.length > 0) {
@@ -1483,5 +1550,13 @@ export async function buildUserPartsWithMultimodal(input, textOverride, logPrefi
     // Vertex client left this front end advertising native audio via
     // NATIVE_AUDIO_PROVIDERS and then dropping the bytes on the floor.
     await appendNativeAudioParts(parts, input?.nativeAudioFiles, logPrefix);
+    // `google-ai-studio` rather than the caller's provider string: this helper
+    // is the AI Studio / Gemini 3 assembly path, and the capability table is
+    // keyed by provider name. Passing a name the table does not know would make
+    // every clip fall back to keyframes on the one front end that can watch it.
+    // `model` is threaded through for signature parity with the Vertex call
+    // site; AI Studio never hosts a Claude model, so it makes no delivery
+    // difference here, only in `canDeliverVideoNatively`'s Vertex branch.
+    await appendNativeVideoParts(parts, input?.nativeVideoFiles, "google-ai-studio", model, logPrefix);
     return parts;
 }

package/dist/providers/googleVertex/client.js

@@ -6,7 +6,7 @@ import path from "path";
 import { ErrorCategory, ErrorSeverity, } from "../../constants/enums.js";
 import { BaseProvider } from "../../core/baseProvider.js";
 import { unwrapImagePayload } from "../../adapters/imageFormatSupport.js";
-import { appendNativeAudioParts } from "../googleNativeGemini3/utils.js";
+import { appendNativeAudioParts, appendNativeVideoParts, } from "../googleNativeGemini3/utils.js";
 import { getMimeTypeForExtension } from "../../processors/config/mimeConstants.js";
 import { DEFAULT_GEMINI_STREAM_TIMEOUT_MS, DEFAULT_MAX_STEPS, DEFAULT_TOOL_MAX_RETRIES, GLOBAL_LOCATION_MODELS, resolveToolTimeoutMs, TOOL_STORAGE_TIMEOUT_MS, } from "../../core/constants.js";
 import { resolveRequestKind } from "../../core/resolveRequestKind.js";
@@ -1302,6 +1302,11 @@ export class GoogleVertexProvider extends BaseProvider {
                 });
             }
         }
+        // Video last, after images: `appendNativeVideoParts` seeds its
+        // request-wide byte budget from whatever `inlineData` parts already sit
+        // in `userParts`, so the images above must be pushed before it runs or
+        // their bytes are invisible to the ceiling check that follows.
+        await appendNativeVideoParts(userParts, multimodalInput?.nativeVideoFiles, "vertex", modelName, "[GoogleVertex]");
         // Prepend prior conversation turns before the current user message so
         // multi-turn callers (memory, loop REPL, agent flows) actually carry
         // context. Without this, the native Vertex Gemini stream rebuilt the
@@ -2114,6 +2119,11 @@ export class GoogleVertexProvider extends BaseProvider {
                 });
             }
         }
+        // Video last, after images: `appendNativeVideoParts` seeds its
+        // request-wide byte budget from whatever `inlineData` parts already sit
+        // in `userParts`, so the images above must be pushed before it runs or
+        // their bytes are invisible to the ceiling check that follows.
+        await appendNativeVideoParts(userParts, multimodalInput?.nativeVideoFiles, "vertex", modelName, "[GoogleVertex]");
         // Prepend prior conversation turns before the current user message so
         // multi-turn callers (memory, loop REPL, agent flows) carry context
         // into native Vertex Gemini generate. Without this, every call started

package/dist/types/file.d.ts

@@ -62,6 +62,105 @@ export type MultimodalAudioEntry = {
     /** Detected MIME type of `buffer`. */
     mimeType: string;
 };
+/**
+ * One video file destined for native delivery to a provider.
+ *
+ * Mirrors {@link MultimodalAudioEntry}: the bytes travel rather than the path,
+ * because whether a video is sent at all is decided per provider, after
+ * detection has already read the file.
+ *
+ * `durationSec` rides along because the native-delivery gate is expressed in
+ * seconds as well as bytes, and re-probing the container at dispatch time
+ * would mean a second ffprobe run for something the processor already
+ * measured. It is optional: probing can fail (no ffmpeg, an exotic container),
+ * and an unknown duration must not by itself disqualify a clip that is
+ * comfortably under the size ceiling.
+ */
+export type MultimodalVideoEntry = {
+    /** Raw video bytes, as detected. */
+    buffer: Buffer;
+    /** Display name; may be a full path, so log only its basename. */
+    filename: string;
+    /** Detected MIME type of `buffer`. */
+    mimeType: string;
+    /** Clip length in seconds, when the processor was able to measure it. */
+    durationSec?: number;
+};
+/**
+ * How one provider handles an attached video.
+ *
+ * The table lives in `adapters/videoFormatSupport.ts`; this is its row shape.
+ * `apiType` names the mechanism that is actually implemented, not the one a
+ * provider theoretically offers — Gemini also exposes a resumable Files API
+ * for clips beyond the inline ceiling, and until that is wired up calling this
+ * row "files-api" would misdescribe what happens to a 200 MB upload.
+ */
+export type VideoProviderConfig = {
+    /** Whether raw video bytes can be handed to this provider at all. */
+    readonly supportsNativeVideo: boolean;
+    /** How the video reaches the model. */
+    readonly apiType: "inline" | "frame-extraction";
+    /**
+     * Inline source-byte budget for the whole request, in MB. A clip is
+     * delivered natively only while its bytes plus every inline part already in
+     * the request stay within it; otherwise it falls back to keyframes.
+     * Meaningless when `supportsNativeVideo` is false, and set to 0 there rather
+     * than to a number that reads like a real limit.
+     */
+    readonly maxSizeMB: number;
+    /** Longest clip accepted natively, in seconds. 0 when not applicable. */
+    readonly maxDurationSec: number;
+    /** Whether the provider hears the video's audio track as well as seeing it. */
+    readonly supportsAudio: boolean;
+    /**
+     * Keyframe budget to aim for when this provider gets frames instead of the
+     * video. Advisory: an explicit `videoOptions.frames` always wins.
+     */
+    readonly recommendedFrameCount: number;
+};
+/**
+ * Media collected during file detection that a provider may be able to
+ * consume directly, rather than as a text summary.
+ *
+ * Grouped rather than passed as two more positional parameters: the message
+ * converters already take text, images, PDFs, provider and model, and each
+ * new modality added one more argument to a call nobody could read. A bag
+ * also means the next modality is a field, not another signature change at
+ * every call site.
+ */
+export type NativeMediaAttachments = {
+    readonly audio?: MultimodalAudioEntry[];
+    readonly video?: MultimodalVideoEntry[];
+    /**
+     * Transcode target for an incompatible image (see
+     * `GenerateOptions.imageOptions`). Not itself a native attachment, but
+     * bundled here for the same reason audio and video are: it is one more
+     * per-request option the message converters need at the same call sites.
+     */
+    readonly outputFormat?: VisionImageOutputFormat;
+};
+/**
+ * Outcome of asking whether one video may be handed to one provider as bytes.
+ *
+ * A plain boolean collapsed "this provider never watches video" with "this
+ * provider would have, but the clip is 400 MB" — and the two want different
+ * log lines and different advice. `reason` is populated exactly when
+ * `deliver` is false, and is phrased for a user to read.
+ *
+ * The accepting arm declares `reason?: undefined` rather than omitting the
+ * field. Not decoration: `build:react-hooks` type-checks this graph without
+ * `--strict`, and there a negated boolean-literal discriminant does not
+ * narrow, so `decision.reason` inside `if (!decision.deliver)` fails to
+ * compile unless the property exists on both arms. Declaring it keeps the
+ * union exact under strict and compilable under both.
+ */
+export type VideoDeliveryDecision = {
+    readonly deliver: true;
+    readonly reason?: undefined;
+} | {
+    readonly deliver: false;
+    readonly reason: string;
+};
 /**
  * Broad category a file format belongs to, as a human would name it.
  *
@@ -190,6 +289,14 @@ export type FileProcessingResult = {
         hasImages?: boolean;
         frameCount?: number;
         hasKeyframes?: boolean;
+        /**
+         * Clip length in seconds, when the processor could measure it.
+         *
+         * Surfaced here so the native-delivery gate does not have to re-probe a
+         * container the processor has already opened. Absent when probing failed
+         * — an unknown duration, not a zero-length clip.
+         */
+        durationSec?: number;
     };
 };
 /**

package/dist/types/generate.d.ts

@@ -17,7 +17,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
 import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
-import type { CSVProcessorOptions, FileWithMetadata, MultimodalAudioEntry, VisionImageOutputFormat } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata, MultimodalAudioEntry, MultimodalVideoEntry, VisionImageOutputFormat } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { Schema, Tool, ToolChoice } from "./tools.js";
 import type { StepResult, LanguageModel } from "./providers.js";
@@ -68,6 +68,17 @@ export type GenerateOptions = {
          */
         nativeAudioFiles?: MultimodalAudioEntry[];
         videoFiles?: Array<Buffer | string>;
+        /**
+         * Video collected during file detection, carried through to a provider
+         * that can watch it. Populated by detection rather than by callers.
+         *
+         * Separate from `videoFiles` above, which is the caller-facing input that
+         * yields a metadata summary plus extracted keyframes. This one carries the
+         * decoded bytes forward so a provider that accepts video receives the clip
+         * itself; providers that cannot fall back to the frames and this is
+         * ignored.
+         */
+        nativeVideoFiles?: MultimodalVideoEntry[];
         files?: Array<Buffer | string | FileWithMetadata>;
         content?: Content[];
         /**

package/dist/types/providers.d.ts

@@ -7,7 +7,7 @@ import { AIProviderName, AnthropicModels, BedrockModels, DeepSeekModels, GoogleA
 import type { ValidationSchema } from "./aliases.js";
 import type { DecisionLimits, DecisionRequest, DecisionResult } from "./decision.js";
 import type { EnhancedGenerateResult, TextGenerationOptions } from "./generate.js";
-import type { MultimodalAudioEntry } from "./file.js";
+import type { MultimodalAudioEntry, MultimodalVideoEntry } from "./file.js";
 import type { StreamOptions, StreamResult } from "./stream.js";
 import type { ProviderError, ProviderErrorRule } from "./errors.js";
 import type { ExternalMCPToolInfo } from "./externalMcp.js";
@@ -2206,6 +2206,13 @@ export type GeminiMultimodalInput = {
      * are already-materialised bytes with a resolved mime type.
      */
     nativeAudioFiles?: MultimodalAudioEntry[];
+    /**
+     * Video collected during file detection, carried through to the native
+     * request as `inlineData`. Distinct from the user-facing `videoFiles`: these
+     * are already-materialised bytes with a resolved mime type and, where it
+     * could be measured, the clip's duration.
+     */
+    nativeVideoFiles?: MultimodalVideoEntry[];
 };
 /**
  * Internal helpers used by the conversation-history builder in

package/dist/utils/fileDetector.js

@@ -1132,6 +1132,14 @@ export class FileDetector {
                         ...detection.metadata,
                         frameCount: videoResult.data.frameCount,
                         hasKeyframes: videoResult.data.hasKeyframes,
+                        // Carried so native delivery can apply its duration ceiling
+                        // without opening the container a second time. The processor
+                        // reports 0 when nothing could probe it; that is "unknown", not a
+                        // zero-length clip, so it is left off rather than passed through
+                        // as a number a ceiling check would happily accept.
+                        ...(videoResult.data.metadata.duration > 0
+                            ? { durationSec: videoResult.data.metadata.duration }
+                            : {}),
                     },
                 };
             }

package/dist/utils/messageBuilder.js

@@ -16,6 +16,7 @@ import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js";
 import { FileDetector } from "./fileDetector.js";
 import { detectIsoBmffImageMimeType } from "./isoBmff.js";
 import { needsAudioTranscode, supportsNativeAudio, toProviderCompatibleAudio, } from "../adapters/audioFormatSupport.js";
+import { supportsNativeVideo } from "../adapters/videoFormatSupport.js";
 import { getImageCache } from "./imageCache.js";
 import { ImageProcessor, imageUtils } from "./imageProcessor.js";
 import { logger } from "./logger.js";
@@ -697,6 +698,26 @@ async function appendDetectedFileResult(result, file, options) {
             ];
             logger.info(`[FileDetector] Added ${result.images.length} video keyframes as images`);
         }
+        // Carry the bytes forward alongside the summary and the frames, the same
+        // way audio does. Whether the clip is sent is decided later, per provider:
+        // one that can watch receives it, one that cannot still gets the keyframes
+        // and is no worse off than before. This matters most where the frames are
+        // absent — ffmpeg is optional, and without it the summary was everything
+        // the model saw.
+        const videoBytes = await readFileInputBytes(file);
+        if (videoBytes) {
+            options.input.nativeVideoFiles = [
+                ...(options.input.nativeVideoFiles || []),
+                {
+                    buffer: videoBytes,
+                    filename,
+                    mimeType: result.mimeType,
+                    ...(typeof result.metadata?.durationSec === "number"
+                        ? { durationSec: result.metadata.durationSec }
+                        : {}),
+                },
+            ];
+        }
         logger.info(`[FileDetector] ✅ Video: ${filename}`);
     }
     else if (result.type === "audio") {
@@ -1376,8 +1397,14 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     // summary survives.
     const hasNativeAudio = (inp.nativeAudioFiles?.length ?? 0) > 0 &&
         supportsNativeAudio(provider, model);
-    // If no images, PDFs or audio, use standard message building and convert to MultimodalChatMessage[]
-    if (!hasImages && !hasPDFs && !hasNativeAudio) {
+    // Video that is to be delivered natively is multimodal for exactly the
+    // reason audio is: it becomes a non-text part. Gated on the provider
+    // accepting video so one that only ever sees keyframes keeps the cheaper
+    // shape it already had.
+    const hasNativeVideo = (inp.nativeVideoFiles?.length ?? 0) > 0 &&
+        supportsNativeVideo(provider, model);
+    // If no images, PDFs, audio or video, use standard message building and convert to MultimodalChatMessage[]
+    if (!hasImages && !hasPDFs && !hasNativeAudio && !hasNativeVideo) {
         // #289: CSV content[] items don't need vision, so they never reach the
         // multimodal converter below — process them into the prompt text here
         // (otherwise a `content: [{type:"csv"}]`-only request silently drops it).
@@ -1495,7 +1522,11 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
             // audio-only turn fell through to the plain-text branch below, so the
             // recording was dropped and only its metadata summary — already folded
             // into `text` — ever reached the model.
-            (inp.nativeAudioFiles?.length ?? 0) > 0) {
+            (inp.nativeAudioFiles?.length ?? 0) > 0 ||
+            // Same for a video-only turn. On a machine without ffmpeg there are no
+            // keyframes to put in `images`, so this clause is the only thing keeping
+            // the clip out of the plain-text branch.
+            (inp.nativeVideoFiles?.length ?? 0) > 0) {
             userContent = await convertMultimodalToProviderFormat(inp.text ?? "", inp.images || [], pdfFiles, provider, model, {
                 audioFiles: inp.nativeAudioFiles ?? [],
                 outputFormat: options.imageOptions?.outputFormat,
@@ -1597,6 +1628,13 @@ async function convertContentToProviderFormat(content, provider, model, pdfOptio
     // change exists to stop, reached through a different branch. Gated on the
     // provider accepting audio so one that cannot keeps the cheaper plain-text
     // shape rather than an array carrying a part it will ignore.
+    //
+    // Video does not get the same treatment here: native delivery for the two
+    // Gemini provider classes never reaches this function with any consequence
+    // (both fully own their own request assembly — see the module comment on
+    // `appendNativeVideoParts` in googleNativeGemini3/utils.ts, the actual and
+    // only place a video clip is attached), so a `deliversVideo` flag here would
+    // gate nothing but a value nobody reads.
     const deliversAudio = audioFiles.length > 0 && supportsNativeAudio(provider, model);
     const hasMultimodal = imageContent.length > 0 || pdfContent.length > 0 || deliversAudio;
     // Validate that we have at least some content
@@ -2252,6 +2290,22 @@ mediaOptions = {}) {
                 `${compatible.converted ? ` (converted to ${compatible.mimeType})` : ""}`);
         }
     }
+    // Video is deliberately not attached here. This function's output — an AI
+    // SDK `content` array — only ever reaches a real request for providers that
+    // call `BaseProvider`'s shared message building unmodified, and neither
+    // Gemini provider class does: `GoogleAIStudioProvider.generate()` and
+    // `GoogleVertexProvider.generate()` are full overrides that never call
+    // `super.generate()`, and `BaseProvider.stream()`'s early multimodal check
+    // builds a `messages` value only to test `hasVideoFrames()` on it before
+    // discarding it — so a video part built here never travels anywhere. The
+    // one delivery mechanism both Gemini front ends actually use is the
+    // hand-built `inlineData` request in `appendNativeVideoParts`
+    // (googleNativeGemini3/utils.ts), which carries its own aggregate-ceiling
+    // enforcement. An earlier version of this function duplicated that
+    // ceiling check here too, permanently inert and with no test able to
+    // reach it — that duplicate has been removed rather than left as a second
+    // place the same bug could be introduced without either delivery or a
+    // failing test ever showing it.
     // Check if provider supports native PDF processing
     const supportsNativePDF = PDFProcessor.supportsNativePDF(provider);
     if (supportsNativePDF) {

package/docs-site/static/search-index.json

@@ -3874,6 +3874,8 @@
 {"objectID":"73d7935973c0a686585af87403eac8b065162f05b5a7c956894d5cc17807712d","title":"PDF Documents","url":"/docs/features/multimodal#pdf-documents","content":"| Provider              | Supported | Max Size | Max Pages | Processing Mode  | Notes                                   |\n| --------------------- | --------- | -------- | --------- | ---------------- | --------------------------------------- |\n| Google Vertex AI  | ✅        | 5 MB     | 100       | Native PDF       | Best for document analysis              |\n| Anthropic         | ✅        | 5 MB     | 100       | Native PDF       | Claude excels at document understanding |\n| AWS Bedrock       | ✅        | 5 MB     | 100       | Native PDF       | Via Claude models                       |\n| Google AI Studio  | ✅        | 2000 MB  | 100       | Native PDF       | Handles very large files                |\n| OpenAI            | ✅        | 10 MB    | 100       | Files API        | , ,            |\n| Azure OpenAI      | ✅        | 10 MB    | 100       | Files API        | Uses OpenAI Files API                   |\n| LiteLLM           | ✅        | 10 MB    | 100       | Proxy            | Depends on upstream model               |\n| OpenAI Compatible | ✅        | 10 MB    | 100       | Varies           | Server-dependent                        |\n| Mistral           | ✅        | 10 MB    | 100       | Native PDF       | Native support                          |\n| Hugging Face      | ✅        | 10 MB    | 100       | Model-dependent  | Varies by model                         |\n| Ollama            | ❌        | -        | -         | -                | Not supported                           |\n| OpenRouter        | ⚠️        | Varies   | Varies    | Depends on model | Route-dependent                         |\n| AWS SageMaker     | ❌        | -        | -         | -                | Not supported                           |","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"PDF Documents","lvl3":""}},
 {"objectID":"96bf471d5383c56e1f0144483b12e8a6f6c221ab852ed86b42f9ea057e68a91d","title":"CSV/Spreadsheet Data","url":"/docs/features/multimodal#csvspreadsheet-data","content":"| Provider          | Supported | Max Rows | Format Options      | Notes                                 |\n| ----------------- | --------- | -------- | ------------------- | ------------------------------------- |\n| All Providers | ✅        | 10,000   | raw, json, markdown | Universal support - processed as text |\n\nCSV support works with all providers because files are converted to text before sending to the AI model. The file is parsed and formatted (raw CSV, JSON, or Markdown table) before inclusion in the prompt.\n\nFormat Recommendations:\nRaw format - Best for large files (minimal token usage)\nJSON format - Best for structured data processing\nMarkdown format - Best for small datasets (\\<100 rows), readable tables","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CSV/Spreadsheet Data","lvl3":""}},
 {"objectID":"b52c712e324d5d40df527e8a5877d9ea0ae63d2ae80bd022cfbaffc9c3c91a82","title":"Audio Input","url":"/docs/features/multimodal#audio-input","content":"| Provider             | Native Audio | Transcription | Real-time | Max Duration | Notes                               |\n| -------------------- | ------------ | ------------- | --------- | ------------ | ----------------------------------- |\n| Google AI Studio | ✅           | ✅            | ✅        | 1 hour       | Best for real-time voice            |\n| Google Vertex AI | ✅           | ✅            | ✅        | 1 hour       | Native Gemini audio support         |\n| OpenAI           | ❌           | ✅ Whisper    | ❌        | 25 MB        | Excellent transcription accuracy    |\n| Azure OpenAI     | ❌           | ✅ Whisper    | ❌        | 25 MB        | Via Whisper integration             |\n| Anthropic        | ❌           | Via fallback  | ❌        | -            | Uses transcription approach         |\n| AWS Bedrock      | ❌           | Via fallback  | ❌        | -            | Uses transcription approach         |\n| Others           | ❌           | Via fallback  | ❌        | -            | Audio transcribed before processing |\n\nFor comprehensive audio documentation, see the Audio Input Guide.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Audio Input","lvl3":""}},
+{"objectID":"6c75d56315cf5bcec368645e07a84c0b2b067c8052da1028e9deb65d16aab9ee","title":"Video","url":"/docs/features/multimodal#video","content":"Two mechanisms, and which one a provider gets changes what it can answer.\n\nNative (inline) — the video file itself travels in the request. The model\nsees continuous motion and hears the audio track. Only Google's Gemini front\nends accept this today.\n\nFrame extraction — ffmpeg pulls keyframes at intervals and they are\nattached as ordinary images, alongside a metadata summary in the prompt text.\nThe model sees a handful of stills and hears nothing.\n\n| Provider                             | Mechanism        | Inline ceiling | Audio track | Default frame budget |\n| ------------------------------------ | ---------------- | -------------- | ----------- | -------------------- |\n| Google AI Studio                 | Native (inline)  | 14 MB          | ✅ heard    | 16 (fallback only)   |\n| Google Vertex AI (Gemini models) | Native (inline)  | 14 MB          | ✅ heard    | 16 (fallback only)   |\n| OpenAI / Azure OpenAI            | Frame extraction | –              | ❌          | 8                    |\n| Anthropic / AWS Bedrock          | Frame extraction | –              | ❌          | 8                    |\n| Mistral                          | Frame extraction | –              | ❌          | 8                    |\n| LiteLLM / OpenRouter             | Frame extraction | –              | ❌          | 8                    |\n| Ollama / llama.cpp               | Frame extraction | –              | ❌          | 4                    |\n\nThe 14 MB ceiling is one shared source-byte budget for the whole request,\nnot a separate per-clip limit. Before a clip ships natively, its bytes are\nadded to the source bytes of every inline part already in the request, and\nthe clip is delivered only if the total stays within 14 MB. Those parts\ninclude images (keyframes among them), PDFs, audio and any clip accepted\nearlier. With nothing else attached, that reduces to the clip's own size.\nThe budget is in source bytes because inline data travels as base64 in a\nJSON body, which costs four bytes per th","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Video","lvl3":""}},
+{"objectID":"623b369433add73c32c5d31532a82b33ea81350ec9e96ead3a7dc6262db5c09f","title":"Supported containers","url":"/docs/features/multimodal#supported-containers","content":", , , , , , , ,  and\nthe other formats in the detector's video registry are all accepted for frame\nextraction. Inline delivery is narrower — it is limited to the containers\nGemini reads directly (mp4, mpeg, mov, avi, flv, webm, wmv, 3gpp) — and a\ncontainer outside that set falls back to keyframes rather than being\nre-encoded. Transcoding a long recording mid-request costs minutes of CPU for\na payload that would usually breach the inline ceiling anyway.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Supported containers","lvl3":""}},
 {"objectID":"0a6aa016f95cc342e4dd2a5dd050bd8ded76685337214426811889d4fdd671b7","title":"Quick Start","url":"/docs/features/multimodal#quick-start","content":"CLI:\n\n`bash","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Quick Start","lvl3":""}},
 {"objectID":"de245f012caaeb0388d4424c7d969aed288374590e5c1f58a0016a1ae1996e17","title":"Single image","url":"/docs/features/multimodal#single-image","content":"npx @juspay/neurolink generate \"Describe this interface\" \\\n  --image ./designs/dashboard.png --provider google-ai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Single image","lvl3":""}},
 {"objectID":"05b1002029479df0ee1386f3f8434822765351def1e4c18cf8c6349b373418fa","title":"Remote URL","url":"/docs/features/multimodal#remote-url","content":"npx @juspay/neurolink generate \"Analyze this diagram\" \\\n  --image https://example.com/architecture.png --provider openai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Remote URL","lvl3":""}},
@@ -3892,6 +3894,14 @@
 {"objectID":"0ffdeaf2a4d433739fb1f96c6487b2e35e254c4459ead6ecbfd3dcb3a98de8b3","title":"Explicit CSV with options","url":"/docs/features/multimodal#explicit-csv-with-options","content":"npx @juspay/neurolink generate \"Summarize data\" \\\n  --csv ./data.csv \\\n  --csv-max-rows 500 \\\n  --csv-format raw\ntypescript\n// Auto-detect (recommended)\nawait neurolink.generate({\n  input: {\n    text: \"Analyze this sales data\",\n    files: [\"./sales.csv\"], // Auto-detected as CSV\n  },\n});\n\n// Explicit CSV with options\nawait neurolink.generate({\n  input: {\n    text: \"Compare quarterly data\",\n    csvFiles: [\"./q1.csv\", \"./q2.csv\"],\n  },\n  csvOptions: {\n    maxRows: 1000,\n    formatStyle: \"json\", // or \"raw\", \"markdown\"\n  },\n});\n`","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Explicit CSV with options","lvl3":""}},
 {"objectID":"93535721732536c013e0204b266f2a131be12a7745f6c88a6e61cdeb76704728","title":"CSV Format Options","url":"/docs/features/multimodal#csv-format-options","content":"Three format styles:\nRaw format (default)\nBest for large files\nMinimal token usage\nPreserves original CSV structure\nJSON format\nStructured data processing\nEasier for AI to parse\nHigher token usage\nMarkdown format\nReadable tables\nGood for small datasets (\\<100 rows)\nModerate token usage","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CSV Format Options","lvl3":""}},
 {"objectID":"427b303d134e7e002fe1fbb0ae86243f33506e795e3029cdfb7994c2014c0db4","title":"CSV Best Practices","url":"/docs/features/multimodal#csv-best-practices","content":"Use raw format for large files to minimize token usage\nUse JSON format for structured processing when AI needs to manipulate data\nLimit to 1000 rows by default (configurable up to 10,000)\nCombine CSV with visualization images for comprehensive analysis\nWorks with ALL providers (not just vision-capable models)","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CSV Best Practices","lvl3":""}},
+{"objectID":"5f2f649cdd7639b3fc3fb30871df07bf3331ad446ac4d23340aaab8976ccf1a7","title":"Frame-extraction options","url":"/docs/features/multimodal#frame-extraction-options","content":"These control the keyframes and are ignored by a provider on the native path.\n\nWithout an explicit , the interval is chosen from the clip's duration\n— roughly every second for clips under 10s, widening to every three minutes\nfor recordings over half an hour, always capped at 100 frames.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Frame-extraction options","lvl3":""}},
+{"objectID":"2c4c2bb10b6fea62d177994e6df1345d05606b65f8122b3c969cf214df3363f1","title":"CLI usage","url":"/docs/features/multimodal#cli-usage","content":"`bash","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"CLI usage","lvl3":""}},
+{"objectID":"915a9e9701a9e6b02890e744550ed66754549288fa044c90bd2dd873e6b48c2a","title":"Auto-detect","url":"/docs/features/multimodal#auto-detect","content":"neurolink generate \"Describe this video\" --file demo.mp4 --provider google-ai","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Auto-detect","lvl3":""}},
+{"objectID":"12ce623ba884b34886674b24daff6ce9e58465c68dc25f4f66c01229c6855513","title":"Explicit flag","url":"/docs/features/multimodal#explicit-flag","content":"neurolink generate \"Summarise this\" --video standup.mp4 --provider vertex","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Explicit flag","lvl3":""}},
+{"objectID":"08774b3c1d8fd456cb7d64ad98281449b9271854310a9a713048ae2112572d5c","title":"Frame-extraction knobs","url":"/docs/features/multimodal#frame-extraction-knobs","content":"neurolink generate \"Describe each scene\" --file demo.mp4 \\\n  --provider openai --video-frames 16 --video-quality 90 --video-format jpeg\n--transcribe-audiovideoOptions.transcribeAudiotranscribeAudio`.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Frame-extraction knobs","lvl3":""}},
+{"objectID":"d0faa964a225271bc41d6c279e3e925829ac8e8828518387c39ba60d97cd363f","title":"Asking which mechanism applies","url":"/docs/features/multimodal#asking-which-mechanism-applies","content":"The capability table is exported, so a caller can ask before sending — useful\nfor choosing a provider, or for budgeting.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Asking which mechanism applies","lvl3":""}},
+{"objectID":"10a8f5bf0ea5129e242eb09d5db285a4d91aac091c171050471ebd7eb528cb72","title":"Best practices","url":"/docs/features/multimodal#best-practices","content":"Ask an audio question of a Gemini provider only. Nothing on the frame\n  path can hear the clip, so \"what was said\" has no answer there.\nKeep clips under 14 MB when you want native delivery. Re-encoding a\n  screen recording at a lower bitrate usually gets a long clip under the\n  ceiling without losing anything a model would read.\nRaise  rather than  for detail over time. Frames cost\n  roughly 250 tokens each regardless of quality; more of them is what buys\n  coverage of a clip where things change.\nLower  for local runtimes. Ollama and llama.cpp hold every\n  frame in the same memory as the model; four is the practical default.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Best practices","lvl3":""}},
+{"objectID":"7bd6ebb59bff22b19f26c8c358eb7012f8a73d4f90eed575ca136f632c6f9bc6","title":"Troubleshooting","url":"/docs/features/multimodal#troubleshooting","content":"| Symptom                                             | Cause                                                                   | Fix                                                                     |\n| --------------------------------------------------- | ----------------------------------------------------------------------- | ----------------------------------------------------------------------- |\n| Model describes the file but not its content        | No keyframes extracted — ffmpeg missing — and the provider takes frames | Install ffmpeg, or use a Gemini provider, which needs none              |\n|  in the logs | The clip failed the inline gate; the reason is on the same line         | Shorten or re-encode it, or accept the frames                           |\n| Model cannot hear speech on a Gemini provider       | The clip exceeded the inline ceiling and fell back to frames            | Get the source under 14 MB                                              |\n|  produced no transcript         | Not implemented yet (#433); the flag is a no-op that only warns         | None yet — track #1757 |\n| Only the first seconds of a long clip are described | The frame budget was hit before the end                                 | Raise ; the interval then spreads evenly across the whole clip  |","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Troubleshooting","lvl3":""}},
 {"objectID":"79318378a7df7cc816465eb447fa4bd2ba9e6344f17b0a9cf6661377bb03e7e2","title":"Combining Multiple Input Types","url":"/docs/features/multimodal#combining-multiple-input-types","content":"NeuroLink excels at combining different media types in a single request.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Combining Multiple Input Types","lvl3":""}},
 {"objectID":"412f09080a5801a35250df393e2e0694602d5312e2b3d1b017fe469bf9dc0474","title":"Batch with Multimodal (CLI)","url":"/docs/features/multimodal#batch-with-multimodal-cli","content":"The  command supports , , , and . The file(s) are attached identically to every prompt in the batch (a one-line notice is printed to stderr, unconditionally — it is not suppressed by , so it never corrupts  output written to stdout):\n\n(auto-detect) is not available in , because it collides with the  positional (the prompts-list path). Use the explicit  /  /  /  flags instead.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"Batch with Multimodal (CLI)","lvl3":""}},
 {"objectID":"66d907236327d676dbe6e05193c2793f7c5990dce2ad113cf078eb1ff4c73216","title":"File validation & troubleshooting (CLI)","url":"/docs/features/multimodal#file-validation-troubleshooting-cli","content":"Before any provider call, the CLI validates local file inputs across , , and :\nA path that points at a directory, doesn't exist, or can't be read (e.g. a permissions error) is rejected up front with a clear error and troubleshooting hints — no cryptic / deep in processing. This also applies to the   prompts-list positional itself.\nA large file (images > 10 MB, CSV/PDF > 50 MB) prints a non-blocking warning to stderr. Like the batch attachment notice, this is unconditional — visible without  and regardless of  — and never mixes into stdout, so  output stays valid JSON even when large-file warnings fire.\n\nThis runs even under  and without API keys configured.","hierarchy":{"lvl0":"Features","lvl1":"Multimodal Capabilities Guide","lvl2":"File validation & troubleshooting (CLI)","lvl3":""}},

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "12.26.3",
+  "version": "12.27.0",
   "packageManager": "pnpm@10.15.1",
   "description": "The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002/call) for routing, tool selection and context budgeting. MCP-native (4 transports), voice TTS/STT/realtime, RAG, agents, memory, compaction, 9 observability exporters. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · TypeSafe Jev and more.",
   "author": {
@@ -240,8 +240,9 @@
     "pre-push": "pnpm run check:deps && pnpm run build && pnpm run test:provider-structure && pnpm run test:model-manifests",
     "check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit",
     "test:file-formats": "pnpm exec tsx test/continuous-test-suite-file-formats.ts",
-    "test:multimodal": "pnpm run test:file-formats && pnpm run test:multimodal:sdk",
+    "test:multimodal": "pnpm run test:file-formats && pnpm run test:multimodal:sdk && pnpm run test:video-native",
     "test:multimodal:sdk": "pnpm exec tsx test/continuous-test-suite-multimodal-sdk.ts",
+    "test:video-native": "pnpm exec tsx test/continuous-test-suite-video-native.ts",
     "test:mcp-result-cache": "pnpm exec tsx test/continuous-test-suite-mcp-result-cache.ts",
     "test:mcp-breaker-resolved-errors": "pnpm exec tsx test/continuous-test-suite-mcp-breaker-resolved-errors.ts",
     "test:archive:security": "pnpm exec tsx test/continuous-test-suite-archive-security.ts",