@juspay/neurolink 12.26.3 → 12.27.1

package/dist/index.d.ts

@@ -72,6 +72,7 @@ export { AzureTTS, AzureTTSHandler, CartesiaTTS, CartesiaTTSHandler, ElevenLabsT
 export { BeatovenMusic, BeatovenMusicHandler, ElevenLabsMusic, ElevenLabsMusicHandler, LyriaMusic, LyriaMusicHandler, registerDefaultMusicHandlers, ReplicateMusic, ReplicateMusicHandler, } from "./music/index.js";
 export { DIDAvatar, DIDAvatarHandler, HeyGenAvatar, HeyGenAvatarHandler, registerDefaultAvatarHandlers, ReplicateAvatar, ReplicateAvatarHandler, } from "./avatar/index.js";
 export { isVertexVideoConfigured, KlingVideoHandler, registerDefaultVideoHandlers, ReplicateVideoHandler, RunwayVideoHandler, VertexVideoHandler, } from "./adapters/video/index.js";
+export { canDeliverVideoNatively, estimateVideoTokens, getVideoProviderConfig, isNativeVideoMimeType, supportsNativeVideo, VIDEO_PROVIDER_CONFIGS, } from "./adapters/videoFormatSupport.js";
 export { ImageGenService } from "./image-gen/ImageGenService.js";
 export { detectImageMimeType, sniffImageMimeType, } from "./utils/imageDetection.js";
 export { HITLManager } from "./hitl/hitlManager.js";

package/dist/index.js

@@ -145,6 +145,13 @@ export { BeatovenMusic, BeatovenMusicHandler, ElevenLabsMusic, ElevenLabsMusicHa
 export { DIDAvatar, DIDAvatarHandler, HeyGenAvatar, HeyGenAvatarHandler, registerDefaultAvatarHandlers, ReplicateAvatar, ReplicateAvatarHandler, } from "./avatar/index.js";
 // Video handlers
 export { isVertexVideoConfigured, KlingVideoHandler, registerDefaultVideoHandlers, ReplicateVideoHandler, RunwayVideoHandler, VertexVideoHandler, } from "./adapters/video/index.js";
+// Video *ingestion* capabilities — what each provider does with an attached
+// clip. Surfaced because the answer changes what a caller should send: a
+// provider on the native path is billed by duration and reads the audio
+// track, one on the frame path is billed per keyframe and hears nothing, and
+// without a way to ask, the only way to find out is to send a video and read
+// the reply.
+export { canDeliverVideoNatively, estimateVideoTokens, getVideoProviderConfig, isNativeVideoMimeType, supportsNativeVideo, VIDEO_PROVIDER_CONFIGS, } from "./adapters/videoFormatSupport.js";
 // Image generation + HITL — surfaced from their dedicated barrels
 export { ImageGenService } from "./image-gen/ImageGenService.js";
 export { detectImageMimeType, sniffImageMimeType, } from "./utils/imageDetection.js";

package/dist/providers/googleAiStudio/client.js

@@ -665,7 +665,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
                 // reported "no image attached" on multimodal calls.
                 const currentContents = [];
                 prependConversationMessages(currentContents, options.conversationMessages);
-                const userParts = await buildUserPartsWithMultimodal(options.input, options.input.text, "[GoogleAIStudio:stream]");
+                const userParts = await buildUserPartsWithMultimodal(options.input, options.input.text, "[GoogleAIStudio:stream]", modelName);
                 currentContents.push({
                     role: "user",
                     parts: userParts,
@@ -1220,7 +1220,7 @@ export class GoogleAIStudioProvider extends BaseProvider {
                 // legitimately reported "no image / PDF attached".
                 const currentContents = [];
                 prependConversationMessages(currentContents, options.conversationMessages);
-                const userParts = await buildUserPartsWithMultimodal(options.input, promptText, "[GoogleAIStudio:generate]");
+                const userParts = await buildUserPartsWithMultimodal(options.input, promptText, "[GoogleAIStudio:generate]", modelName);
                 currentContents.push({
                     role: "user",
                     parts: userParts,

package/dist/providers/googleNativeGemini3/utils.d.ts

@@ -8,7 +8,7 @@
  * This module extracts the functions that are duplicated between the two
  * providers so they can share a single implementation.
  */
-import type { GenerateStopReason, ThinkingConfig, AgenticLoopOptions, ChatMessage, CollectedChunkResult, MinimalChatMessage, NativeFunctionCall, NativeFunctionResponse, NativeToolDeclarationsResult, NativeToolsConfig, StreamChannel, VertexNativePart, GeminiMultimodalInput, MultimodalAudioEntry } from "../../types/index.js";
+import type { GenerateStopReason, ThinkingConfig, AgenticLoopOptions, ChatMessage, CollectedChunkResult, MinimalChatMessage, NativeFunctionCall, NativeFunctionResponse, NativeToolDeclarationsResult, NativeToolsConfig, StreamChannel, VertexNativePart, GeminiMultimodalInput, MultimodalAudioEntry, MultimodalVideoEntry } from "../../types/index.js";
 import type { Tool, ToolExecutionGuards } from "../../types/index.js";
 /**
  * A per-turn tool execute map that deduplicates identical tool calls.
@@ -466,4 +466,26 @@ export declare function prependConversationMessages(contents: Array<{
  * summary in the text part.
  */
 export declare function appendNativeAudioParts(userParts: VertexNativePart[], audioFiles: MultimodalAudioEntry[] | undefined, logPrefix?: string): Promise<void>;
-export declare function buildUserPartsWithMultimodal(input: GeminiMultimodalInput | undefined, textOverride?: string, logPrefix?: string): Promise<VertexNativePart[]>;
+/**
+ * Append video to a Gemini request as `inlineData` parts.
+ *
+ * The native request body is assembled by hand rather than taken from the AI
+ * SDK's `file` parts, so — exactly as with audio — a `{ type: "file" }` part
+ * built upstream never arrives here. Video has to be added explicitly or it
+ * is silently dropped on both Gemini front ends while the capability table
+ * says it is supported.
+ *
+ * A clip the provider will not take inline is skipped rather than sent: an
+ * oversized or unsupported `inlineData` part fails the whole request, whereas
+ * skipping leaves the caller with the metadata summary and the keyframes,
+ * which is what it had before native delivery existed.
+ *
+ * `model` is forwarded to `canDeliverVideoNatively` so a Claude-on-Vertex
+ * request — same `providerName` ("vertex") as a Gemini-on-Vertex one, routed
+ * to the native Anthropic SDK instead by `GoogleVertexProvider.isAnthropicModel`
+ * — is answered as not natively video-capable rather than inheriting Vertex's
+ * Gemini row. Optional because AI Studio's caller never has a Claude model to
+ * pass.
+ */
+export declare function appendNativeVideoParts(userParts: VertexNativePart[], videoFiles: MultimodalVideoEntry[] | undefined, providerName: string, model?: string, logPrefix?: string): Promise<void>;
+export declare function buildUserPartsWithMultimodal(input: GeminiMultimodalInput | undefined, textOverride?: string, logPrefix?: string, model?: string): Promise<VertexNativePart[]>;

package/dist/providers/googleNativeGemini3/utils.js

@@ -13,6 +13,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { extname } from "node:path";
 import { DEFAULT_CONTEXT_GUARD_RATIO, DEFAULT_MAX_STEPS, DEFAULT_TOOL_MAX_RETRIES, DEFAULT_WRAPUP_TIME_LEAD_MS, } from "../../core/constants.js";
 import { needsAudioTranscode, toProviderCompatibleAudio, } from "../../adapters/audioFormatSupport.js";
+import { canDeliverVideoNatively } from "../../adapters/videoFormatSupport.js";
 import { logger } from "../../utils/logger.js";
 import { guardToolExecutor } from "../../core/toolExecutionGuards.js";
 import { resolveSamplingParams } from "../../models/modelRegistry.js";
@@ -1376,7 +1377,73 @@ export async function appendNativeAudioParts(userParts, audioFiles, logPrefix =
         logger.debug(`${logPrefix} Added native audio part for ${base} (${compatible.mimeType})`);
     }
 }
-export async function buildUserPartsWithMultimodal(input, textOverride, logPrefix = "[GeminiNative]") {
+/**
+ * Append video to a Gemini request as `inlineData` parts.
+ *
+ * The native request body is assembled by hand rather than taken from the AI
+ * SDK's `file` parts, so — exactly as with audio — a `{ type: "file" }` part
+ * built upstream never arrives here. Video has to be added explicitly or it
+ * is silently dropped on both Gemini front ends while the capability table
+ * says it is supported.
+ *
+ * A clip the provider will not take inline is skipped rather than sent: an
+ * oversized or unsupported `inlineData` part fails the whole request, whereas
+ * skipping leaves the caller with the metadata summary and the keyframes,
+ * which is what it had before native delivery existed.
+ *
+ * `model` is forwarded to `canDeliverVideoNatively` so a Claude-on-Vertex
+ * request — same `providerName` ("vertex") as a Gemini-on-Vertex one, routed
+ * to the native Anthropic SDK instead by `GoogleVertexProvider.isAnthropicModel`
+ * — is answered as not natively video-capable rather than inheriting Vertex's
+ * Gemini row. Optional because AI Studio's caller never has a Claude model to
+ * pass.
+ */
+export async function appendNativeVideoParts(userParts, videoFiles, providerName, model, logPrefix = "[GeminiNative]") {
+    if (!videoFiles || videoFiles.length === 0) {
+        return;
+    }
+    // Tracks source bytes already committed to this request's shared inline
+    // budget. This is the sole place video ceiling enforcement happens: the AI
+    // SDK `file`-part assembly in messageBuilder.ts never delivers video for
+    // either Gemini provider class (both own their request assembly end to
+    // end), so a duplicate of this check used to sit there too — permanently
+    // inert and unreachable by any test — and has been removed rather than
+    // left as a second, silently-driftable copy of the same logic.
+    //
+    // Seeded from the PDF, image and audio `inlineData` parts the caller has
+    // already pushed onto `userParts` before this runs, not just from sibling
+    // video clips: the 20MB Gemini request ceiling is shared by every inline
+    // part, so a clip that would fit alone can still overflow the request
+    // once those other parts are counted. `data` is base64, which costs 4
+    // bytes per 3 of source — decode the length back to source bytes so this
+    // total stays comparable to `video.buffer.length` below.
+    let committedVideoBytes = userParts.reduce((total, part) => {
+        if ("inlineData" in part) {
+            return total + Math.floor((part.inlineData.data.length * 3) / 4);
+        }
+        return total;
+    }, 0);
+    for (const video of videoFiles) {
+        // Split on both separators: a Windows-style name reaching a POSIX host
+        // would otherwise keep its whole path in the log line.
+        const base = video.filename.split(/[\\/]/).pop() ?? video.filename;
+        const decision = canDeliverVideoNatively(providerName, video, committedVideoBytes, model);
+        if (!decision.deliver) {
+            logger.warn(`${logPrefix} Sending keyframes instead of the clip for ${base}: ` +
+                `${decision.reason}.`);
+            continue;
+        }
+        committedVideoBytes += video.buffer.length;
+        userParts.push({
+            inlineData: {
+                mimeType: video.mimeType,
+                data: video.buffer.toString("base64"),
+            },
+        });
+        logger.debug(`${logPrefix} Added native video part for ${base} (${video.mimeType})`);
+    }
+}
+export async function buildUserPartsWithMultimodal(input, textOverride, logPrefix = "[GeminiNative]", model) {
     const text = typeof textOverride === "string" ? textOverride : (input?.text ?? "");
     const parts = [{ text }];
     if (input?.pdfFiles && input.pdfFiles.length > 0) {
@@ -1483,5 +1550,13 @@ export async function buildUserPartsWithMultimodal(input, textOverride, logPrefi
     // Vertex client left this front end advertising native audio via
     // NATIVE_AUDIO_PROVIDERS and then dropping the bytes on the floor.
     await appendNativeAudioParts(parts, input?.nativeAudioFiles, logPrefix);
+    // `google-ai-studio` rather than the caller's provider string: this helper
+    // is the AI Studio / Gemini 3 assembly path, and the capability table is
+    // keyed by provider name. Passing a name the table does not know would make
+    // every clip fall back to keyframes on the one front end that can watch it.
+    // `model` is threaded through for signature parity with the Vertex call
+    // site; AI Studio never hosts a Claude model, so it makes no delivery
+    // difference here, only in `canDeliverVideoNatively`'s Vertex branch.
+    await appendNativeVideoParts(parts, input?.nativeVideoFiles, "google-ai-studio", model, logPrefix);
     return parts;
 }

package/dist/providers/googleVertex/client.js

@@ -6,7 +6,7 @@ import path from "path";
 import { ErrorCategory, ErrorSeverity, } from "../../constants/enums.js";
 import { BaseProvider } from "../../core/baseProvider.js";
 import { unwrapImagePayload } from "../../adapters/imageFormatSupport.js";
-import { appendNativeAudioParts } from "../googleNativeGemini3/utils.js";
+import { appendNativeAudioParts, appendNativeVideoParts, } from "../googleNativeGemini3/utils.js";
 import { getMimeTypeForExtension } from "../../processors/config/mimeConstants.js";
 import { DEFAULT_GEMINI_STREAM_TIMEOUT_MS, DEFAULT_MAX_STEPS, DEFAULT_TOOL_MAX_RETRIES, GLOBAL_LOCATION_MODELS, resolveToolTimeoutMs, TOOL_STORAGE_TIMEOUT_MS, } from "../../core/constants.js";
 import { resolveRequestKind } from "../../core/resolveRequestKind.js";
@@ -1302,6 +1302,11 @@ export class GoogleVertexProvider extends BaseProvider {
                 });
             }
         }
+        // Video last, after images: `appendNativeVideoParts` seeds its
+        // request-wide byte budget from whatever `inlineData` parts already sit
+        // in `userParts`, so the images above must be pushed before it runs or
+        // their bytes are invisible to the ceiling check that follows.
+        await appendNativeVideoParts(userParts, multimodalInput?.nativeVideoFiles, "vertex", modelName, "[GoogleVertex]");
         // Prepend prior conversation turns before the current user message so
         // multi-turn callers (memory, loop REPL, agent flows) actually carry
         // context. Without this, the native Vertex Gemini stream rebuilt the
@@ -2114,6 +2119,11 @@ export class GoogleVertexProvider extends BaseProvider {
                 });
             }
         }
+        // Video last, after images: `appendNativeVideoParts` seeds its
+        // request-wide byte budget from whatever `inlineData` parts already sit
+        // in `userParts`, so the images above must be pushed before it runs or
+        // their bytes are invisible to the ceiling check that follows.
+        await appendNativeVideoParts(userParts, multimodalInput?.nativeVideoFiles, "vertex", modelName, "[GoogleVertex]");
         // Prepend prior conversation turns before the current user message so
         // multi-turn callers (memory, loop REPL, agent flows) carry context
         // into native Vertex Gemini generate. Without this, every call started

package/dist/proxy/proxyTracer.d.ts

@@ -127,6 +127,15 @@ export declare function recordFallbackAttempt(attrs: {
     errorMessage?: string;
     durationMs: number;
 }): void;
+/** Reset leg-health state between tests. */
+export declare const __fallbackLegHealthTestHooks: {
+    threshold: number;
+    maxTrackedLegs: number;
+    trackedCount: () => number;
+    reset: () => void;
+    streakOf: (legKey: string) => number;
+    isReported: (legKey: string) => boolean;
+};
 export { ProxyTracer };
 /** Standard SERVER span covers every proxy door, including parsing and admission failures. */
 export declare function startProxyHttpTrace(metadata: RuntimeRequestMetadata, headers: Record<string, string>): {

package/dist/proxy/proxyTracer.js

@@ -757,7 +757,96 @@ class ProxyTracer {
         return this.bridge.injectContext({}, trace.setSpan(context.active(), this.rootSpan));
     }
 }
+/**
+ * Consecutive failures a configured fallback leg may accumulate before the
+ * proxy says so out loud.
+ *
+ * A leg that is attempted and never succeeds is invisible today: the counters
+ * record every attempt and every failure, but nothing reads them back, so the
+ * condition surfaces only if an operator happens to query telemetry. Meanwhile
+ * every request falls through to the next leg, which can bill on a different
+ * account at a different rate, so the first symptom is a cost change rather
+ * than an error. Twenty is low enough to catch a dead leg within seconds of
+ * sustained traffic and high enough that ordinary account rotation or a brief
+ * cooldown never trips it.
+ */
+const DEAD_FALLBACK_LEG_THRESHOLD = 20;
+/**
+ * Upper bound on tracked legs.
+ *
+ * Not every key comes from the configured chain. The auto-provider path records
+ * `auto-provider/<requested model>`, and the requested model is whatever the
+ * caller asked for, so a client that fails requests under varying model names
+ * mints a new key each time. Without a bound these collections grow for the
+ * lifetime of the process. The cap is far above any real fallback chain, so a
+ * genuine leg is never evicted in practice.
+ */
+const MAX_TRACKED_FALLBACK_LEGS = 256;
+/** Consecutive failures per `provider/model`, reset by any success. */
+const fallbackFailureStreaks = new Map();
+/** Legs already reported, so a dead leg is stated once and not once per turn. */
+const reportedDeadFallbackLegs = new Set();
+/**
+ * Record a streak, evicting the least recently touched leg past the cap.
+ *
+ * `Map` and `Set` iterate in insertion order and re-setting an existing key does
+ * not move it, so a key is deleted before being re-added. That makes recency the
+ * eviction order and keeps a leg that is actively failing from being dropped in
+ * favour of a stale one.
+ */
+function touchFallbackLegStreak(legKey, streak) {
+    fallbackFailureStreaks.delete(legKey);
+    fallbackFailureStreaks.set(legKey, streak);
+    while (fallbackFailureStreaks.size > MAX_TRACKED_FALLBACK_LEGS) {
+        const oldest = fallbackFailureStreaks.keys().next();
+        if (oldest.done) {
+            break;
+        }
+        fallbackFailureStreaks.delete(oldest.value);
+        reportedDeadFallbackLegs.delete(oldest.value);
+    }
+}
+/**
+ * Report a fallback leg that has stopped serving, and its recovery.
+ *
+ * Deliberately keyed per leg rather than globally: a second leg going dark must
+ * still be reported while the first is known dead.
+ */
+function trackFallbackLegHealth(legKey, status, errorMessage) {
+    if (status === "success") {
+        fallbackFailureStreaks.delete(legKey);
+        if (reportedDeadFallbackLegs.delete(legKey)) {
+            logger.always(`[proxy] fallback ${legKey} is serving again after being reported dead`);
+        }
+        return;
+    }
+    const streak = (fallbackFailureStreaks.get(legKey) ?? 0) + 1;
+    touchFallbackLegStreak(legKey, streak);
+    if (streak < DEAD_FALLBACK_LEG_THRESHOLD) {
+        return;
+    }
+    if (reportedDeadFallbackLegs.has(legKey)) {
+        return;
+    }
+    // Marked reported only once the report is out. Marking first meant a throwing
+    // logger — swallowed by the caller's catch — silenced the leg permanently,
+    // which is the one outcome this whole mechanism exists to prevent.
+    logger.always(`[proxy] fallback ${legKey} has failed ${streak} consecutive attempts with ` +
+        `no success — this leg is not serving. Traffic is falling through to the ` +
+        `next leg in the chain, which may bill differently. Check the account's ` +
+        `quota and credentials` +
+        (errorMessage ? `; last error: ${errorMessage.slice(0, 200)}` : ""));
+    reportedDeadFallbackLegs.add(legKey);
+}
 export function recordFallbackAttempt(attrs) {
+    // Leg health is tracked outside the metrics try/catch: a telemetry failure
+    // must not be the reason an operator never hears that a leg is dead.
+    try {
+        trackFallbackLegHealth(`${attrs.provider}/${attrs.model}`, attrs.status, attrs.errorMessage);
+    }
+    catch {
+        // Never let health reporting break a request.
+    }
     try {
         const m = getProxyMetrics();
         const labels = { provider: attrs.provider, model: attrs.model };
@@ -776,6 +865,18 @@ export function recordFallbackAttempt(attrs) {
         // metrics are best-effort
     }
 }
+/** Reset leg-health state between tests. */
+export const __fallbackLegHealthTestHooks = {
+    threshold: DEAD_FALLBACK_LEG_THRESHOLD,
+    maxTrackedLegs: MAX_TRACKED_FALLBACK_LEGS,
+    trackedCount: () => fallbackFailureStreaks.size,
+    reset: () => {
+        fallbackFailureStreaks.clear();
+        reportedDeadFallbackLegs.clear();
+    },
+    streakOf: (legKey) => fallbackFailureStreaks.get(legKey) ?? 0,
+    isReported: (legKey) => reportedDeadFallbackLegs.has(legKey),
+};
 export { ProxyTracer };
 /** Standard SERVER span covers every proxy door, including parsing and admission failures. */
 export function startProxyHttpTrace(metadata, headers) {

package/dist/types/file.d.ts

@@ -62,6 +62,105 @@ export type MultimodalAudioEntry = {
     /** Detected MIME type of `buffer`. */
     mimeType: string;
 };
+/**
+ * One video file destined for native delivery to a provider.
+ *
+ * Mirrors {@link MultimodalAudioEntry}: the bytes travel rather than the path,
+ * because whether a video is sent at all is decided per provider, after
+ * detection has already read the file.
+ *
+ * `durationSec` rides along because the native-delivery gate is expressed in
+ * seconds as well as bytes, and re-probing the container at dispatch time
+ * would mean a second ffprobe run for something the processor already
+ * measured. It is optional: probing can fail (no ffmpeg, an exotic container),
+ * and an unknown duration must not by itself disqualify a clip that is
+ * comfortably under the size ceiling.
+ */
+export type MultimodalVideoEntry = {
+    /** Raw video bytes, as detected. */
+    buffer: Buffer;
+    /** Display name; may be a full path, so log only its basename. */
+    filename: string;
+    /** Detected MIME type of `buffer`. */
+    mimeType: string;
+    /** Clip length in seconds, when the processor was able to measure it. */
+    durationSec?: number;
+};
+/**
+ * How one provider handles an attached video.
+ *
+ * The table lives in `adapters/videoFormatSupport.ts`; this is its row shape.
+ * `apiType` names the mechanism that is actually implemented, not the one a
+ * provider theoretically offers — Gemini also exposes a resumable Files API
+ * for clips beyond the inline ceiling, and until that is wired up calling this
+ * row "files-api" would misdescribe what happens to a 200 MB upload.
+ */
+export type VideoProviderConfig = {
+    /** Whether raw video bytes can be handed to this provider at all. */
+    readonly supportsNativeVideo: boolean;
+    /** How the video reaches the model. */
+    readonly apiType: "inline" | "frame-extraction";
+    /**
+     * Inline source-byte budget for the whole request, in MB. A clip is
+     * delivered natively only while its bytes plus every inline part already in
+     * the request stay within it; otherwise it falls back to keyframes.
+     * Meaningless when `supportsNativeVideo` is false, and set to 0 there rather
+     * than to a number that reads like a real limit.
+     */
+    readonly maxSizeMB: number;
+    /** Longest clip accepted natively, in seconds. 0 when not applicable. */
+    readonly maxDurationSec: number;
+    /** Whether the provider hears the video's audio track as well as seeing it. */
+    readonly supportsAudio: boolean;
+    /**
+     * Keyframe budget to aim for when this provider gets frames instead of the
+     * video. Advisory: an explicit `videoOptions.frames` always wins.
+     */
+    readonly recommendedFrameCount: number;
+};
+/**
+ * Media collected during file detection that a provider may be able to
+ * consume directly, rather than as a text summary.
+ *
+ * Grouped rather than passed as two more positional parameters: the message
+ * converters already take text, images, PDFs, provider and model, and each
+ * new modality added one more argument to a call nobody could read. A bag
+ * also means the next modality is a field, not another signature change at
+ * every call site.
+ */
+export type NativeMediaAttachments = {
+    readonly audio?: MultimodalAudioEntry[];
+    readonly video?: MultimodalVideoEntry[];
+    /**
+     * Transcode target for an incompatible image (see
+     * `GenerateOptions.imageOptions`). Not itself a native attachment, but
+     * bundled here for the same reason audio and video are: it is one more
+     * per-request option the message converters need at the same call sites.
+     */
+    readonly outputFormat?: VisionImageOutputFormat;
+};
+/**
+ * Outcome of asking whether one video may be handed to one provider as bytes.
+ *
+ * A plain boolean collapsed "this provider never watches video" with "this
+ * provider would have, but the clip is 400 MB" — and the two want different
+ * log lines and different advice. `reason` is populated exactly when
+ * `deliver` is false, and is phrased for a user to read.
+ *
+ * The accepting arm declares `reason?: undefined` rather than omitting the
+ * field. Not decoration: `build:react-hooks` type-checks this graph without
+ * `--strict`, and there a negated boolean-literal discriminant does not
+ * narrow, so `decision.reason` inside `if (!decision.deliver)` fails to
+ * compile unless the property exists on both arms. Declaring it keeps the
+ * union exact under strict and compilable under both.
+ */
+export type VideoDeliveryDecision = {
+    readonly deliver: true;
+    readonly reason?: undefined;
+} | {
+    readonly deliver: false;
+    readonly reason: string;
+};
 /**
  * Broad category a file format belongs to, as a human would name it.
  *
@@ -190,6 +289,14 @@ export type FileProcessingResult = {
         hasImages?: boolean;
         frameCount?: number;
         hasKeyframes?: boolean;
+        /**
+         * Clip length in seconds, when the processor could measure it.
+         *
+         * Surfaced here so the native-delivery gate does not have to re-probe a
+         * container the processor has already opened. Absent when probing failed
+         * — an unknown duration, not a zero-length clip.
+         */
+        durationSec?: number;
     };
 };
 /**

package/dist/types/generate.d.ts

@@ -17,7 +17,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
 import type { MusicOptions, MusicResult } from "./music.js";
 import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
 import type { NeurolinkCredentials } from "./providers.js";
-import type { CSVProcessorOptions, FileWithMetadata, MultimodalAudioEntry, VisionImageOutputFormat } from "./file.js";
+import type { CSVProcessorOptions, FileWithMetadata, MultimodalAudioEntry, MultimodalVideoEntry, VisionImageOutputFormat } from "./file.js";
 import type { WorkflowConfig } from "./workflow.js";
 import type { Schema, Tool, ToolChoice } from "./tools.js";
 import type { StepResult, LanguageModel } from "./providers.js";
@@ -68,6 +68,17 @@ export type GenerateOptions = {
          */
         nativeAudioFiles?: MultimodalAudioEntry[];
         videoFiles?: Array<Buffer | string>;
+        /**
+         * Video collected during file detection, carried through to a provider
+         * that can watch it. Populated by detection rather than by callers.
+         *
+         * Separate from `videoFiles` above, which is the caller-facing input that
+         * yields a metadata summary plus extracted keyframes. This one carries the
+         * decoded bytes forward so a provider that accepts video receives the clip
+         * itself; providers that cannot fall back to the frames and this is
+         * ignored.
+         */
+        nativeVideoFiles?: MultimodalVideoEntry[];
         files?: Array<Buffer | string | FileWithMetadata>;
         content?: Content[];
         /**

package/dist/types/providers.d.ts

@@ -7,7 +7,7 @@ import { AIProviderName, AnthropicModels, BedrockModels, DeepSeekModels, GoogleA
 import type { ValidationSchema } from "./aliases.js";
 import type { DecisionLimits, DecisionRequest, DecisionResult } from "./decision.js";
 import type { EnhancedGenerateResult, TextGenerationOptions } from "./generate.js";
-import type { MultimodalAudioEntry } from "./file.js";
+import type { MultimodalAudioEntry, MultimodalVideoEntry } from "./file.js";
 import type { StreamOptions, StreamResult } from "./stream.js";
 import type { ProviderError, ProviderErrorRule } from "./errors.js";
 import type { ExternalMCPToolInfo } from "./externalMcp.js";
@@ -2206,6 +2206,13 @@ export type GeminiMultimodalInput = {
      * are already-materialised bytes with a resolved mime type.
      */
     nativeAudioFiles?: MultimodalAudioEntry[];
+    /**
+     * Video collected during file detection, carried through to the native
+     * request as `inlineData`. Distinct from the user-facing `videoFiles`: these
+     * are already-materialised bytes with a resolved mime type and, where it
+     * could be measured, the clip's duration.
+     */
+    nativeVideoFiles?: MultimodalVideoEntry[];
 };
 /**
  * Internal helpers used by the conversation-history builder in

package/dist/utils/fileDetector.js

@@ -1132,6 +1132,14 @@ export class FileDetector {
                         ...detection.metadata,
                         frameCount: videoResult.data.frameCount,
                         hasKeyframes: videoResult.data.hasKeyframes,
+                        // Carried so native delivery can apply its duration ceiling
+                        // without opening the container a second time. The processor
+                        // reports 0 when nothing could probe it; that is "unknown", not a
+                        // zero-length clip, so it is left off rather than passed through
+                        // as a number a ceiling check would happily accept.
+                        ...(videoResult.data.metadata.duration > 0
+                            ? { durationSec: videoResult.data.metadata.duration }
+                            : {}),
                     },
                 };
             }

package/dist/utils/messageBuilder.js

@@ -16,6 +16,7 @@ import { ErrorFactory, NeuroLinkError, withTimeout } from "./errorHandling.js";
 import { FileDetector } from "./fileDetector.js";
 import { detectIsoBmffImageMimeType } from "./isoBmff.js";
 import { needsAudioTranscode, supportsNativeAudio, toProviderCompatibleAudio, } from "../adapters/audioFormatSupport.js";
+import { supportsNativeVideo } from "../adapters/videoFormatSupport.js";
 import { getImageCache } from "./imageCache.js";
 import { ImageProcessor, imageUtils } from "./imageProcessor.js";
 import { logger } from "./logger.js";
@@ -697,6 +698,26 @@ async function appendDetectedFileResult(result, file, options) {
             ];
             logger.info(`[FileDetector] Added ${result.images.length} video keyframes as images`);
         }
+        // Carry the bytes forward alongside the summary and the frames, the same
+        // way audio does. Whether the clip is sent is decided later, per provider:
+        // one that can watch receives it, one that cannot still gets the keyframes
+        // and is no worse off than before. This matters most where the frames are
+        // absent — ffmpeg is optional, and without it the summary was everything
+        // the model saw.
+        const videoBytes = await readFileInputBytes(file);
+        if (videoBytes) {
+            options.input.nativeVideoFiles = [
+                ...(options.input.nativeVideoFiles || []),
+                {
+                    buffer: videoBytes,
+                    filename,
+                    mimeType: result.mimeType,
+                    ...(typeof result.metadata?.durationSec === "number"
+                        ? { durationSec: result.metadata.durationSec }
+                        : {}),
+                },
+            ];
+        }
         logger.info(`[FileDetector] ✅ Video: ${filename}`);
     }
     else if (result.type === "audio") {
@@ -1376,8 +1397,14 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
     // summary survives.
     const hasNativeAudio = (inp.nativeAudioFiles?.length ?? 0) > 0 &&
         supportsNativeAudio(provider, model);
-    // If no images, PDFs or audio, use standard message building and convert to MultimodalChatMessage[]
-    if (!hasImages && !hasPDFs && !hasNativeAudio) {
+    // Video that is to be delivered natively is multimodal for exactly the
+    // reason audio is: it becomes a non-text part. Gated on the provider
+    // accepting video so one that only ever sees keyframes keeps the cheaper
+    // shape it already had.
+    const hasNativeVideo = (inp.nativeVideoFiles?.length ?? 0) > 0 &&
+        supportsNativeVideo(provider, model);
+    // If no images, PDFs, audio or video, use standard message building and convert to MultimodalChatMessage[]
+    if (!hasImages && !hasPDFs && !hasNativeAudio && !hasNativeVideo) {
         // #289: CSV content[] items don't need vision, so they never reach the
         // multimodal converter below — process them into the prompt text here
         // (otherwise a `content: [{type:"csv"}]`-only request silently drops it).
@@ -1495,7 +1522,11 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
             // audio-only turn fell through to the plain-text branch below, so the
             // recording was dropped and only its metadata summary — already folded
             // into `text` — ever reached the model.
-            (inp.nativeAudioFiles?.length ?? 0) > 0) {
+            (inp.nativeAudioFiles?.length ?? 0) > 0 ||
+            // Same for a video-only turn. On a machine without ffmpeg there are no
+            // keyframes to put in `images`, so this clause is the only thing keeping
+            // the clip out of the plain-text branch.
+            (inp.nativeVideoFiles?.length ?? 0) > 0) {
             userContent = await convertMultimodalToProviderFormat(inp.text ?? "", inp.images || [], pdfFiles, provider, model, {
                 audioFiles: inp.nativeAudioFiles ?? [],
                 outputFormat: options.imageOptions?.outputFormat,
@@ -1597,6 +1628,13 @@ async function convertContentToProviderFormat(content, provider, model, pdfOptio
     // change exists to stop, reached through a different branch. Gated on the
     // provider accepting audio so one that cannot keeps the cheaper plain-text
     // shape rather than an array carrying a part it will ignore.
+    //
+    // Video does not get the same treatment here: native delivery for the two
+    // Gemini provider classes never reaches this function with any consequence
+    // (both fully own their own request assembly — see the module comment on
+    // `appendNativeVideoParts` in googleNativeGemini3/utils.ts, the actual and
+    // only place a video clip is attached), so a `deliversVideo` flag here would
+    // gate nothing but a value nobody reads.
     const deliversAudio = audioFiles.length > 0 && supportsNativeAudio(provider, model);
     const hasMultimodal = imageContent.length > 0 || pdfContent.length > 0 || deliversAudio;
     // Validate that we have at least some content
@@ -2252,6 +2290,22 @@ mediaOptions = {}) {
                 `${compatible.converted ? ` (converted to ${compatible.mimeType})` : ""}`);
         }
     }
+    // Video is deliberately not attached here. This function's output — an AI
+    // SDK `content` array — only ever reaches a real request for providers that
+    // call `BaseProvider`'s shared message building unmodified, and neither
+    // Gemini provider class does: `GoogleAIStudioProvider.generate()` and
+    // `GoogleVertexProvider.generate()` are full overrides that never call
+    // `super.generate()`, and `BaseProvider.stream()`'s early multimodal check
+    // builds a `messages` value only to test `hasVideoFrames()` on it before
+    // discarding it — so a video part built here never travels anywhere. The
+    // one delivery mechanism both Gemini front ends actually use is the
+    // hand-built `inlineData` request in `appendNativeVideoParts`
+    // (googleNativeGemini3/utils.ts), which carries its own aggregate-ceiling
+    // enforcement. An earlier version of this function duplicated that
+    // ceiling check here too, permanently inert and with no test able to
+    // reach it — that duplicate has been removed rather than left as a second
+    // place the same bug could be introduced without either delivery or a
+    // failing test ever showing it.
     // Check if provider supports native PDF processing
     const supportsNativePDF = PDFProcessor.supportsNativePDF(provider);
     if (supportsNativePDF) {