@juspay/neurolink 12.26.3 → 12.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,8 +1,8 @@
1
- ## [12.26.3](https://github.com/juspay/neurolink/compare/v12.26.2...v12.26.3) (2026-09-26)
1
+ ## [12.27.0](https://github.com/juspay/neurolink/compare/v12.26.3...v12.27.0) (2026-09-26)
2
2
 
3
- ### Bug Fixes
3
+ ### Features
4
4
 
5
- - **(config):** honour DOTENV_CONFIG_PATH in the implicit .env load ([375ac67](https://github.com/juspay/neurolink/commit/375ac6736df088f0cd34a066daf097b628b69ac1)), closes [#1735](https://github.com/juspay/neurolink/issues/1735)
5
+ - **(video):** deliver video natively to Gemini instead of keyframes only ([b4a50da](https://github.com/juspay/neurolink/commit/b4a50dae0c14341f983c2f0ca16ceed744a62b04)), closes [#433](https://github.com/juspay/neurolink/issues/433) [#1757](https://github.com/juspay/neurolink/issues/1757) [#421](https://github.com/juspay/neurolink/issues/421) [#439](https://github.com/juspay/neurolink/issues/439) [#444](https://github.com/juspay/neurolink/issues/444) [#463](https://github.com/juspay/neurolink/issues/463) [#525](https://github.com/juspay/neurolink/issues/525)
6
6
 
7
7
  ## [11.2.3](https://github.com/juspay/neurolink/compare/v11.2.2...v11.2.3) (2026-08-19)
8
8
 
@@ -0,0 +1,134 @@
1
+ /**
2
+ * Native video delivery to providers that can watch.
3
+ *
4
+ * ## The gap this closes
5
+ *
6
+ * Attaching a video produced a metadata block and a handful of downsampled
7
+ * JPEGs:
8
+ *
9
+ * ## Video File: "demo.mp4"
10
+ * Duration: 0:47 | 1920x1080 | avc | 30 fps | 3 keyframes extracted
11
+ *
12
+ * Every provider got the same treatment, Gemini included — and Gemini has
13
+ * accepted inline video, with its audio track, the whole time. A 47-second
14
+ * screen recording reached it as three still frames at 768px, so anything
15
+ * between them (a click, a transition, a spoken sentence) was simply not in
16
+ * the request.
17
+ *
18
+ * Two things make that worse than the equivalent audio gap was. First, the
19
+ * frames come from ffmpeg, and ffmpeg is an optional dependency: on a machine
20
+ * without it `extractKeyframes` returns an empty array and the model receives
21
+ * the metadata line alone — a video attachment that conveys nothing visual at
22
+ * all, with no error. Second, the metadata block answers exactly the questions
23
+ * a test tends to ask ("how long is it?", "what resolution?"), so the failure
24
+ * reads as working support until someone asks what happens in the clip.
25
+ *
26
+ * Delivering the file itself needs no ffmpeg, which is what makes this the
27
+ * fix rather than "extract more frames".
28
+ *
29
+ * ## Provider scope
30
+ *
31
+ * A capability table rather than "send video to everyone". A provider that
32
+ * cannot accept a video part answers with an opaque HTTP 400, which is
33
+ * strictly worse than the metadata-plus-frames summary it would otherwise
34
+ * have received — so an unlisted provider keeps the existing behaviour and
35
+ * loses nothing.
36
+ *
37
+ * ## Why there is no transcode path here
38
+ *
39
+ * `audioFormatSupport` re-encodes a container Gemini will not read, because a
40
+ * voice memo is seconds of audio and the conversion is cheap. The video
41
+ * equivalent is not: re-encoding a half-hour recording is minutes of CPU
42
+ * inside a generation request, for a payload that will usually breach the
43
+ * inline ceiling anyway. An unsupported container therefore falls back to
44
+ * keyframes, which is the behaviour that already existed.
45
+ *
46
+ * @module adapters/videoFormatSupport
47
+ */
48
+ import type { MultimodalVideoEntry, VideoDeliveryDecision, VideoProviderConfig } from "../types/index.js";
49
+ /**
50
+ * Per-provider video handling.
51
+ *
52
+ * Keys are lowercase canonical names and the aliases each provider is
53
+ * addressed by elsewhere in the codebase; `getVideoProviderConfig` normalises
54
+ * before looking up, so a caller never has to know which spelling it holds.
55
+ *
56
+ * Only Google's Gemini front ends carry `supportsNativeVideo: true`. The rest
57
+ * are listed deliberately rather than left to the unknown-provider default:
58
+ * an explicit row is how `getVideoProviderConfig` distinguishes "this provider
59
+ * takes frames" from "nobody has looked at this provider yet", and the frame
60
+ * budgets differ enough between them to be worth stating.
61
+ */
62
+ export declare const VIDEO_PROVIDER_CONFIGS: Readonly<Record<string, VideoProviderConfig>>;
63
+ /**
64
+ * The video-handling row for `provider`, or null when there is none.
65
+ *
66
+ * Null means "not described here", which is not the same as "takes frames":
67
+ * an unrecognised provider still receives keyframes, because that is the
68
+ * pipeline's default, but nothing in this table asserts it will understand
69
+ * them. Callers wanting the safe reading should treat null as no native
70
+ * video, which is what {@link supportsNativeVideo} does.
71
+ *
72
+ * `model` disambiguates Vertex: `GoogleVertexProvider` sends any model whose
73
+ * id contains "claude" to the native Anthropic SDK, which never sees the
74
+ * video part at all, so a Claude-on-Vertex request gets the frame-extraction
75
+ * row (the same one plain `"anthropic"` gets) rather than the Gemini row the
76
+ * provider name alone would suggest. Omitting `model` keeps the previous,
77
+ * provider-only lookup, which is still correct for every non-Vertex caller —
78
+ * AI Studio and the bare `"gemini"` alias never host Claude models.
79
+ */
80
+ export declare function getVideoProviderConfig(provider: string, model?: string): VideoProviderConfig | null;
81
+ /** Whether `provider` (optionally, running `model`) can be handed raw video
82
+ * bytes. See {@link getVideoProviderConfig} for the Claude-on-Vertex case. */
83
+ export declare function supportsNativeVideo(provider: string, model?: string): boolean;
84
+ /** Whether `mimeType` is a container a native provider will read as-is. */
85
+ export declare function isNativeVideoMimeType(mimeType: string): boolean;
86
+ /**
87
+ * Rough token cost of putting one video in front of one provider.
88
+ *
89
+ * Two quite different prices, because two quite different payloads: a native
90
+ * provider is billed for the clip's duration, a frame-extraction provider for
91
+ * the frames it is sent. Asking for one number without saying which mode
92
+ * applies is how a budget ends up an order of magnitude out, so the provider
93
+ * decides the formula rather than the caller.
94
+ *
95
+ * An estimate, not a quote — resolution settings, prompt text and provider
96
+ * pricing changes all move the real figure.
97
+ *
98
+ * @param options.provider - Provider the video is destined for.
99
+ * @param options.durationSec - Clip length. 0 when unknown.
100
+ * @param options.frameCount - Frames that would be extracted. Defaults to the
101
+ * provider's `recommendedFrameCount`, or 8 for an unlisted provider.
102
+ * @param options.hasTranscription - Whether a speech transcript is included.
103
+ * Ignored for a native provider, which already hears the audio track.
104
+ */
105
+ export declare function estimateVideoTokens(options: {
106
+ provider: string;
107
+ durationSec: number;
108
+ frameCount?: number;
109
+ hasTranscription?: boolean;
110
+ }): number;
111
+ /**
112
+ * Whether one specific clip may go to one specific provider as bytes.
113
+ *
114
+ * Every rejection carries a reason the caller can log verbatim, because the
115
+ * user-visible symptom of all of them is identical — "it only described the
116
+ * file" — and the remedies are not: shorten the clip, re-encode the
117
+ * container, or switch provider.
118
+ *
119
+ * An unknown duration is not a rejection. Probing fails on exotic containers
120
+ * and on machines without ffmpeg, and refusing a 2 MB clip because nothing
121
+ * measured it would reintroduce the frames-only behaviour precisely where
122
+ * frames are least likely to be available.
123
+ *
124
+ * @param priorNativeVideoBytes - Source bytes already committed to the
125
+ * *same request's* inline budget: every inline image (keyframes
126
+ * included), PDF and audio part assembled before the clips, plus each
127
+ * clip accepted earlier. The caller accumulates this running total, since
128
+ * this function is otherwise stateless and cannot see the rest of the
129
+ * request. Defaults to 0, which makes the size check below a pure
130
+ * per-clip check.
131
+ * @param model - See {@link getVideoProviderConfig}: disambiguates a
132
+ * Claude-on-Vertex request from a real Gemini-on-Vertex one.
133
+ */
134
+ export declare function canDeliverVideoNatively(provider: string, video: MultimodalVideoEntry, priorNativeVideoBytes?: number, model?: string): VideoDeliveryDecision;
@@ -0,0 +1,379 @@
1
+ /**
2
+ * Native video delivery to providers that can watch.
3
+ *
4
+ * ## The gap this closes
5
+ *
6
+ * Attaching a video produced a metadata block and a handful of downsampled
7
+ * JPEGs:
8
+ *
9
+ * ## Video File: "demo.mp4"
10
+ * Duration: 0:47 | 1920x1080 | avc | 30 fps | 3 keyframes extracted
11
+ *
12
+ * Every provider got the same treatment, Gemini included — and Gemini has
13
+ * accepted inline video, with its audio track, the whole time. A 47-second
14
+ * screen recording reached it as three still frames at 768px, so anything
15
+ * between them (a click, a transition, a spoken sentence) was simply not in
16
+ * the request.
17
+ *
18
+ * Two things make that worse than the equivalent audio gap was. First, the
19
+ * frames come from ffmpeg, and ffmpeg is an optional dependency: on a machine
20
+ * without it `extractKeyframes` returns an empty array and the model receives
21
+ * the metadata line alone — a video attachment that conveys nothing visual at
22
+ * all, with no error. Second, the metadata block answers exactly the questions
23
+ * a test tends to ask ("how long is it?", "what resolution?"), so the failure
24
+ * reads as working support until someone asks what happens in the clip.
25
+ *
26
+ * Delivering the file itself needs no ffmpeg, which is what makes this the
27
+ * fix rather than "extract more frames".
28
+ *
29
+ * ## Provider scope
30
+ *
31
+ * A capability table rather than "send video to everyone". A provider that
32
+ * cannot accept a video part answers with an opaque HTTP 400, which is
33
+ * strictly worse than the metadata-plus-frames summary it would otherwise
34
+ * have received — so an unlisted provider keeps the existing behaviour and
35
+ * loses nothing.
36
+ *
37
+ * ## Why there is no transcode path here
38
+ *
39
+ * `audioFormatSupport` re-encodes a container Gemini will not read, because a
40
+ * voice memo is seconds of audio and the conversion is cheap. The video
41
+ * equivalent is not: re-encoding a half-hour recording is minutes of CPU
42
+ * inside a generation request, for a payload that will usually breach the
43
+ * inline ceiling anyway. An unsupported container therefore falls back to
44
+ * keyframes, which is the behaviour that already existed.
45
+ *
46
+ * @module adapters/videoFormatSupport
47
+ */
48
+ /**
49
+ * Ceiling for natively-delivered video in one request, in MB of *source*
50
+ * bytes.
51
+ *
52
+ * Gemini's documented limit is 20 MB for the whole request. Inline data is
53
+ * base64 in a JSON body, which costs 4 bytes per 3, so a 20 MB source file
54
+ * arrives as ~26.7 MB of request and is rejected.
55
+ *
56
+ * The 20 MB limit is request-wide, not per-part: a video alone consumes it
57
+ * fully once its encoded size reaches 20 MB, leaving nothing for the prompt,
58
+ * system instructions, tool schemas or the JSON structure carrying all of
59
+ * them in the same request. Dividing the raw budget by the 4/3 inflation —
60
+ * 20 ÷ 4/3 = 15 MB — finds the largest source size whose *video part alone*
61
+ * still fits, but that calculation has no margin in it: 15 × 4/3 = 20.0 MB
62
+ * exactly, so any non-empty prompt sharing that request pushes the total
63
+ * over the documented limit and the request is rejected upstream, for a
64
+ * reason this module never sees. That was the source of the previous value
65
+ * here — it satisfied "fits under 20 MB" only for a request containing
66
+ * nothing else, which no real request is.
67
+ *
68
+ * 14 MB is used instead: it still encodes to well under the 20 MB limit
69
+ * (14 × 4/3 ≈ 18.7 MB), leaving roughly 1.3 MB of headroom in the same
70
+ * request for the prompt and everything else sharing it.
71
+ *
72
+ * This is a request-wide budget, not a per-clip one: two clips at 12 MB each
73
+ * pass this check individually but together would arrive as ~32 MB of
74
+ * request. `canDeliverVideoNatively`'s optional `priorNativeVideoBytes`
75
+ * parameter lets a caller iterating multiple clips in one request charge
76
+ * each clip against what earlier clips already committed, so the ceiling is
77
+ * enforced across the whole request rather than reset per clip.
78
+ *
79
+ * Clips above this are not an error — they fall back to keyframe extraction,
80
+ * exactly as before this module existed. Lifting the ceiling properly means
81
+ * wiring the resumable Files API, which is a separate piece of work.
82
+ */
83
+ const GEMINI_INLINE_VIDEO_MAX_MB = 14;
84
+ /**
85
+ * Longest clip Gemini will reason over natively.
86
+ *
87
+ * In practice the size ceiling above binds first — 15 MB of H.264 is minutes,
88
+ * not an hour — so this is a backstop for the case where it does not: a very
89
+ * low-bitrate or largely static recording can be long and small at once.
90
+ */
91
+ const GEMINI_MAX_VIDEO_DURATION_SEC = 3600;
92
+ /**
93
+ * Per-provider video handling.
94
+ *
95
+ * Keys are lowercase canonical names and the aliases each provider is
96
+ * addressed by elsewhere in the codebase; `getVideoProviderConfig` normalises
97
+ * before looking up, so a caller never has to know which spelling it holds.
98
+ *
99
+ * Only Google's Gemini front ends carry `supportsNativeVideo: true`. The rest
100
+ * are listed deliberately rather than left to the unknown-provider default:
101
+ * an explicit row is how `getVideoProviderConfig` distinguishes "this provider
102
+ * takes frames" from "nobody has looked at this provider yet", and the frame
103
+ * budgets differ enough between them to be worth stating.
104
+ */
105
+ export const VIDEO_PROVIDER_CONFIGS = Object.freeze({
106
+ // --- Gemini: inline video, audio track included -------------------------
107
+ ...Object.fromEntries([
108
+ "vertex",
109
+ "google-vertex",
110
+ "googlevertex",
111
+ "google-ai-studio",
112
+ "googleaistudio",
113
+ "google-ai",
114
+ "googleai",
115
+ "gemini",
116
+ ].map((name) => [
117
+ name,
118
+ Object.freeze({
119
+ supportsNativeVideo: true,
120
+ apiType: "inline",
121
+ maxSizeMB: GEMINI_INLINE_VIDEO_MAX_MB,
122
+ maxDurationSec: GEMINI_MAX_VIDEO_DURATION_SEC,
123
+ supportsAudio: true,
124
+ // Only consulted on the fallback path — a clip over the inline
125
+ // ceiling. Generous because that clip is by definition a long one.
126
+ recommendedFrameCount: 16,
127
+ }),
128
+ ])),
129
+ // --- Frame extraction: vision models with no video part -----------------
130
+ ...Object.fromEntries([
131
+ "openai",
132
+ "azure",
133
+ "anthropic",
134
+ "bedrock",
135
+ "amazon-bedrock",
136
+ "mistral",
137
+ "litellm",
138
+ "openrouter",
139
+ "openai-compatible",
140
+ "huggingface",
141
+ "sagemaker",
142
+ "deepseek",
143
+ "nvidia-nim",
144
+ "lmstudio",
145
+ "cerebras",
146
+ "sambanova",
147
+ ].map((name) => [
148
+ name,
149
+ Object.freeze({
150
+ supportsNativeVideo: false,
151
+ apiType: "frame-extraction",
152
+ maxSizeMB: 0,
153
+ maxDurationSec: 0,
154
+ supportsAudio: false,
155
+ recommendedFrameCount: 8,
156
+ }),
157
+ ])),
158
+ // --- Frame extraction on a tighter budget -------------------------------
159
+ // Local runtimes hold every frame in the same memory as the model, so a
160
+ // 16-frame request is the difference between an answer and an OOM.
161
+ ...Object.fromEntries(["ollama", "llamacpp", "llama-cpp"].map((name) => [
162
+ name,
163
+ Object.freeze({
164
+ supportsNativeVideo: false,
165
+ apiType: "frame-extraction",
166
+ maxSizeMB: 0,
167
+ maxDurationSec: 0,
168
+ supportsAudio: false,
169
+ recommendedFrameCount: 4,
170
+ }),
171
+ ])),
172
+ });
173
+ /**
174
+ * Video MIME types Gemini accepts inline.
175
+ *
176
+ * Its documented set. A container outside it is left to keyframe extraction
177
+ * rather than transcoded — see the module header for why.
178
+ */
179
+ const NATIVE_VIDEO_MIME_TYPES = new Set([
180
+ "video/mp4",
181
+ "video/mpeg",
182
+ "video/mpg",
183
+ "video/mov",
184
+ "video/quicktime",
185
+ "video/avi",
186
+ "video/x-msvideo",
187
+ "video/x-flv",
188
+ "video/webm",
189
+ "video/wmv",
190
+ "video/x-ms-wmv",
191
+ "video/3gpp",
192
+ ]);
193
+ /** Tokens one extracted keyframe costs a vision model, approximately. */
194
+ const TOKENS_PER_FRAME = 250;
195
+ /**
196
+ * Tokens one second of natively-delivered video costs.
197
+ *
198
+ * Gemini bills video at roughly 258 tokens per second at default resolution
199
+ * (one frame per second plus the audio track). Rounded to 260 — the estimate
200
+ * is for budgeting, and false precision in a constant invites treating it as
201
+ * a quoted price.
202
+ */
203
+ const TOKENS_PER_NATIVE_VIDEO_SECOND = 260;
204
+ /** Tokens one minute of transcribed speech costs, approximately. */
205
+ const TOKENS_PER_TRANSCRIPT_MINUTE = 400;
206
+ /**
207
+ * Duration assumed for a natively-delivered clip whose length is unknown.
208
+ *
209
+ * `canDeliverVideoNatively` treats an unknown duration as no reason to
210
+ * refuse (probing fails on exotic containers and on machines without
211
+ * ffmpeg — exactly the clips most likely to reach this branch), so a clip
212
+ * priced here with `durationSec: 0` can still be the one that just travelled
213
+ * to Gemini as bytes. Pricing that as 0 tokens would make a real payload
214
+ * look free. 5s is a deliberately small floor: enough that no natively
215
+ * delivered clip is ever budgeted as free, without pretending to know how
216
+ * long an unmeasured one actually runs.
217
+ */
218
+ const MIN_NATIVE_VIDEO_ESTIMATE_SEC = 5;
219
+ function normalizeProvider(provider) {
220
+ return provider.toLowerCase().trim();
221
+ }
222
+ function normalizeVideoMime(mimeType) {
223
+ return mimeType.split(";")[0].trim().toLowerCase();
224
+ }
225
+ /** Vertex aliases GoogleVertexProvider serves — the only front end that can
226
+ * route a single request to either Gemini or Claude depending on model id. */
227
+ const VERTEX_ALIASES = new Set([
228
+ "vertex",
229
+ "google-vertex",
230
+ "googlevertex",
231
+ ]);
232
+ /**
233
+ * Whether `model` is one `GoogleVertexProvider.isAnthropicModel` (client.ts)
234
+ * would route to the native Anthropic Vertex SDK rather than Gemini.
235
+ *
236
+ * Duplicated here rather than imported: `isAnthropicModel` lives in
237
+ * `providers/googleVertex/client.ts`, and importing a provider module from an
238
+ * adapter would put `adapters/` on the wrong side of the provider/adapter
239
+ * dependency direction. `providerImageAdapter.ts` carries the same
240
+ * `model.includes("claude")` check for the same reason — keep both in sync
241
+ * if Vertex's routing predicate ever changes.
242
+ */
243
+ function isClaudeModelId(model) {
244
+ return model.toLowerCase().includes("claude");
245
+ }
246
+ /**
247
+ * The video-handling row for `provider`, or null when there is none.
248
+ *
249
+ * Null means "not described here", which is not the same as "takes frames":
250
+ * an unrecognised provider still receives keyframes, because that is the
251
+ * pipeline's default, but nothing in this table asserts it will understand
252
+ * them. Callers wanting the safe reading should treat null as no native
253
+ * video, which is what {@link supportsNativeVideo} does.
254
+ *
255
+ * `model` disambiguates Vertex: `GoogleVertexProvider` sends any model whose
256
+ * id contains "claude" to the native Anthropic SDK, which never sees the
257
+ * video part at all, so a Claude-on-Vertex request gets the frame-extraction
258
+ * row (the same one plain `"anthropic"` gets) rather than the Gemini row the
259
+ * provider name alone would suggest. Omitting `model` keeps the previous,
260
+ * provider-only lookup, which is still correct for every non-Vertex caller —
261
+ * AI Studio and the bare `"gemini"` alias never host Claude models.
262
+ */
263
+ export function getVideoProviderConfig(provider, model) {
264
+ const normalized = normalizeProvider(provider);
265
+ if (VERTEX_ALIASES.has(normalized) && model && isClaudeModelId(model)) {
266
+ return VIDEO_PROVIDER_CONFIGS.anthropic;
267
+ }
268
+ return VIDEO_PROVIDER_CONFIGS[normalized] ?? null;
269
+ }
270
+ /** Whether `provider` (optionally, running `model`) can be handed raw video
271
+ * bytes. See {@link getVideoProviderConfig} for the Claude-on-Vertex case. */
272
+ export function supportsNativeVideo(provider, model) {
273
+ return getVideoProviderConfig(provider, model)?.supportsNativeVideo ?? false;
274
+ }
275
+ /** Whether `mimeType` is a container a native provider will read as-is. */
276
+ export function isNativeVideoMimeType(mimeType) {
277
+ return NATIVE_VIDEO_MIME_TYPES.has(normalizeVideoMime(mimeType));
278
+ }
279
+ /**
280
+ * Rough token cost of putting one video in front of one provider.
281
+ *
282
+ * Two quite different prices, because two quite different payloads: a native
283
+ * provider is billed for the clip's duration, a frame-extraction provider for
284
+ * the frames it is sent. Asking for one number without saying which mode
285
+ * applies is how a budget ends up an order of magnitude out, so the provider
286
+ * decides the formula rather than the caller.
287
+ *
288
+ * An estimate, not a quote — resolution settings, prompt text and provider
289
+ * pricing changes all move the real figure.
290
+ *
291
+ * @param options.provider - Provider the video is destined for.
292
+ * @param options.durationSec - Clip length. 0 when unknown.
293
+ * @param options.frameCount - Frames that would be extracted. Defaults to the
294
+ * provider's `recommendedFrameCount`, or 8 for an unlisted provider.
295
+ * @param options.hasTranscription - Whether a speech transcript is included.
296
+ * Ignored for a native provider, which already hears the audio track.
297
+ */
298
+ export function estimateVideoTokens(options) {
299
+ const { provider, durationSec, frameCount, hasTranscription } = options;
300
+ const config = getVideoProviderConfig(provider);
301
+ const seconds = Math.max(0, durationSec);
302
+ if (config?.supportsNativeVideo) {
303
+ // seconds === 0 covers both "genuinely 0s" and "unmeasured" (the
304
+ // documented meaning of durationSec: 0 above) — either way a clip that
305
+ // reaches this branch is being delivered natively, so it is never free.
306
+ const nativeSeconds = seconds > 0 ? seconds : MIN_NATIVE_VIDEO_ESTIMATE_SEC;
307
+ return Math.ceil(nativeSeconds * TOKENS_PER_NATIVE_VIDEO_SECOND);
308
+ }
309
+ const frames = Math.max(0, frameCount ?? config?.recommendedFrameCount ?? 8);
310
+ const transcriptTokens = hasTranscription
311
+ ? Math.ceil(seconds / 60) * TOKENS_PER_TRANSCRIPT_MINUTE
312
+ : 0;
313
+ return frames * TOKENS_PER_FRAME + transcriptTokens;
314
+ }
315
+ /**
316
+ * Whether one specific clip may go to one specific provider as bytes.
317
+ *
318
+ * Every rejection carries a reason the caller can log verbatim, because the
319
+ * user-visible symptom of all of them is identical — "it only described the
320
+ * file" — and the remedies are not: shorten the clip, re-encode the
321
+ * container, or switch provider.
322
+ *
323
+ * An unknown duration is not a rejection. Probing fails on exotic containers
324
+ * and on machines without ffmpeg, and refusing a 2 MB clip because nothing
325
+ * measured it would reintroduce the frames-only behaviour precisely where
326
+ * frames are least likely to be available.
327
+ *
328
+ * @param priorNativeVideoBytes - Source bytes already committed to the
329
+ * *same request's* inline budget: every inline image (keyframes
330
+ * included), PDF and audio part assembled before the clips, plus each
331
+ * clip accepted earlier. The caller accumulates this running total, since
332
+ * this function is otherwise stateless and cannot see the rest of the
333
+ * request. Defaults to 0, which makes the size check below a pure
334
+ * per-clip check.
335
+ * @param model - See {@link getVideoProviderConfig}: disambiguates a
336
+ * Claude-on-Vertex request from a real Gemini-on-Vertex one.
337
+ */
338
+ export function canDeliverVideoNatively(provider, video, priorNativeVideoBytes = 0, model) {
339
+ const config = getVideoProviderConfig(provider, model);
340
+ if (!config?.supportsNativeVideo) {
341
+ return {
342
+ deliver: false,
343
+ reason: `${provider} does not accept video attachments`,
344
+ };
345
+ }
346
+ const mimeType = normalizeVideoMime(video.mimeType);
347
+ if (!isNativeVideoMimeType(mimeType)) {
348
+ return {
349
+ deliver: false,
350
+ reason: `${mimeType} is not a container ${provider} reads inline`,
351
+ };
352
+ }
353
+ // Charged cumulatively: this clip's bytes plus everything the same request
354
+ // has already committed to the inline budget (other inline parts and
355
+ // earlier clips). With the default of 0 this is a per-clip-only check.
356
+ const aggregateBytes = Math.max(0, priorNativeVideoBytes) + video.buffer.length;
357
+ const sizeMB = video.buffer.length / (1024 * 1024);
358
+ const aggregateMB = aggregateBytes / (1024 * 1024);
359
+ if (aggregateMB > config.maxSizeMB) {
360
+ const reason = priorNativeVideoBytes > 0
361
+ ? `the clip is ${sizeMB.toFixed(1)}MB, which would bring this ` +
362
+ `request's inline attachments to ${aggregateMB.toFixed(1)}MB, ` +
363
+ `over the ${config.maxSizeMB}MB inline ceiling shared by the whole ` +
364
+ `request for ${provider}`
365
+ : `the clip is ${sizeMB.toFixed(1)}MB, over the ${config.maxSizeMB}MB ` +
366
+ `inline ceiling for ${provider}`;
367
+ return { deliver: false, reason };
368
+ }
369
+ if (typeof video.durationSec === "number" &&
370
+ Number.isFinite(video.durationSec) &&
371
+ video.durationSec > config.maxDurationSec) {
372
+ return {
373
+ deliver: false,
374
+ reason: `the clip runs ${Math.round(video.durationSec)}s, over the ` +
375
+ `${config.maxDurationSec}s ceiling for ${provider}`,
376
+ };
377
+ }
378
+ return { deliver: true };
379
+ }