@juspay/neurolink 12.26.3 → 12.27.1

package/CHANGELOG.md

@@ -1,8 +1,8 @@
-## [12.26.3](https://github.com/juspay/neurolink/compare/v12.26.2...v12.26.3) (2026-09-26)
+## [12.27.1](https://github.com/juspay/neurolink/compare/v12.27.0...v12.27.1) (2026-09-26)
 
 ### Bug Fixes
 
-- **(config):** honour DOTENV_CONFIG_PATH in the implicit .env load ([375ac67](https://github.com/juspay/neurolink/commit/375ac6736df088f0cd34a066daf097b628b69ac1)), closes [#1735](https://github.com/juspay/neurolink/issues/1735)
+- **(proxy):** report a fallback leg that has stopped serving ([96f8bd0](https://github.com/juspay/neurolink/commit/96f8bd00bb5582b47886cd5771964db4814fe9ea))
 
 ## [11.2.3](https://github.com/juspay/neurolink/compare/v11.2.2...v11.2.3) (2026-08-19)
 

package/dist/adapters/videoFormatSupport.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Native video delivery to providers that can watch.
+ *
+ * ## The gap this closes
+ *
+ * Attaching a video produced a metadata block and a handful of downsampled
+ * JPEGs:
+ *
+ *   ## Video File: "demo.mp4"
+ *   Duration: 0:47 | 1920x1080 | avc | 30 fps | 3 keyframes extracted
+ *
+ * Every provider got the same treatment, Gemini included — and Gemini has
+ * accepted inline video, with its audio track, the whole time. A 47-second
+ * screen recording reached it as three still frames at 768px, so anything
+ * between them (a click, a transition, a spoken sentence) was simply not in
+ * the request.
+ *
+ * Two things make that worse than the equivalent audio gap was. First, the
+ * frames come from ffmpeg, and ffmpeg is an optional dependency: on a machine
+ * without it `extractKeyframes` returns an empty array and the model receives
+ * the metadata line alone — a video attachment that conveys nothing visual at
+ * all, with no error. Second, the metadata block answers exactly the questions
+ * a test tends to ask ("how long is it?", "what resolution?"), so the failure
+ * reads as working support until someone asks what happens in the clip.
+ *
+ * Delivering the file itself needs no ffmpeg, which is what makes this the
+ * fix rather than "extract more frames".
+ *
+ * ## Provider scope
+ *
+ * A capability table rather than "send video to everyone". A provider that
+ * cannot accept a video part answers with an opaque HTTP 400, which is
+ * strictly worse than the metadata-plus-frames summary it would otherwise
+ * have received — so an unlisted provider keeps the existing behaviour and
+ * loses nothing.
+ *
+ * ## Why there is no transcode path here
+ *
+ * `audioFormatSupport` re-encodes a container Gemini will not read, because a
+ * voice memo is seconds of audio and the conversion is cheap. The video
+ * equivalent is not: re-encoding a half-hour recording is minutes of CPU
+ * inside a generation request, for a payload that will usually breach the
+ * inline ceiling anyway. An unsupported container therefore falls back to
+ * keyframes, which is the behaviour that already existed.
+ *
+ * @module adapters/videoFormatSupport
+ */
+import type { MultimodalVideoEntry, VideoDeliveryDecision, VideoProviderConfig } from "../types/index.js";
+/**
+ * Per-provider video handling.
+ *
+ * Keys are lowercase canonical names and the aliases each provider is
+ * addressed by elsewhere in the codebase; `getVideoProviderConfig` normalises
+ * before looking up, so a caller never has to know which spelling it holds.
+ *
+ * Only Google's Gemini front ends carry `supportsNativeVideo: true`. The rest
+ * are listed deliberately rather than left to the unknown-provider default:
+ * an explicit row is how `getVideoProviderConfig` distinguishes "this provider
+ * takes frames" from "nobody has looked at this provider yet", and the frame
+ * budgets differ enough between them to be worth stating.
+ */
+export declare const VIDEO_PROVIDER_CONFIGS: Readonly<Record<string, VideoProviderConfig>>;
+/**
+ * The video-handling row for `provider`, or null when there is none.
+ *
+ * Null means "not described here", which is not the same as "takes frames":
+ * an unrecognised provider still receives keyframes, because that is the
+ * pipeline's default, but nothing in this table asserts it will understand
+ * them. Callers wanting the safe reading should treat null as no native
+ * video, which is what {@link supportsNativeVideo} does.
+ *
+ * `model` disambiguates Vertex: `GoogleVertexProvider` sends any model whose
+ * id contains "claude" to the native Anthropic SDK, which never sees the
+ * video part at all, so a Claude-on-Vertex request gets the frame-extraction
+ * row (the same one plain `"anthropic"` gets) rather than the Gemini row the
+ * provider name alone would suggest. Omitting `model` keeps the previous,
+ * provider-only lookup, which is still correct for every non-Vertex caller —
+ * AI Studio and the bare `"gemini"` alias never host Claude models.
+ */
+export declare function getVideoProviderConfig(provider: string, model?: string): VideoProviderConfig | null;
+/** Whether `provider` (optionally, running `model`) can be handed raw video
+ * bytes. See {@link getVideoProviderConfig} for the Claude-on-Vertex case. */
+export declare function supportsNativeVideo(provider: string, model?: string): boolean;
+/** Whether `mimeType` is a container a native provider will read as-is. */
+export declare function isNativeVideoMimeType(mimeType: string): boolean;
+/**
+ * Rough token cost of putting one video in front of one provider.
+ *
+ * Two quite different prices, because two quite different payloads: a native
+ * provider is billed for the clip's duration, a frame-extraction provider for
+ * the frames it is sent. Asking for one number without saying which mode
+ * applies is how a budget ends up an order of magnitude out, so the provider
+ * decides the formula rather than the caller.
+ *
+ * An estimate, not a quote — resolution settings, prompt text and provider
+ * pricing changes all move the real figure.
+ *
+ * @param options.provider - Provider the video is destined for.
+ * @param options.durationSec - Clip length. 0 when unknown.
+ * @param options.frameCount - Frames that would be extracted. Defaults to the
+ *   provider's `recommendedFrameCount`, or 8 for an unlisted provider.
+ * @param options.hasTranscription - Whether a speech transcript is included.
+ *   Ignored for a native provider, which already hears the audio track.
+ */
+export declare function estimateVideoTokens(options: {
+    provider: string;
+    durationSec: number;
+    frameCount?: number;
+    hasTranscription?: boolean;
+}): number;
+/**
+ * Whether one specific clip may go to one specific provider as bytes.
+ *
+ * Every rejection carries a reason the caller can log verbatim, because the
+ * user-visible symptom of all of them is identical — "it only described the
+ * file" — and the remedies are not: shorten the clip, re-encode the
+ * container, or switch provider.
+ *
+ * An unknown duration is not a rejection. Probing fails on exotic containers
+ * and on machines without ffmpeg, and refusing a 2 MB clip because nothing
+ * measured it would reintroduce the frames-only behaviour precisely where
+ * frames are least likely to be available.
+ *
+ * @param priorNativeVideoBytes - Source bytes already committed to the
+ *   *same request's* inline budget: every inline image (keyframes
+ *   included), PDF and audio part assembled before the clips, plus each
+ *   clip accepted earlier. The caller accumulates this running total, since
+ *   this function is otherwise stateless and cannot see the rest of the
+ *   request. Defaults to 0, which makes the size check below a pure
+ *   per-clip check.
+ * @param model - See {@link getVideoProviderConfig}: disambiguates a
+ *   Claude-on-Vertex request from a real Gemini-on-Vertex one.
+ */
+export declare function canDeliverVideoNatively(provider: string, video: MultimodalVideoEntry, priorNativeVideoBytes?: number, model?: string): VideoDeliveryDecision;

package/dist/adapters/videoFormatSupport.js

@@ -0,0 +1,379 @@
+/**
+ * Native video delivery to providers that can watch.
+ *
+ * ## The gap this closes
+ *
+ * Attaching a video produced a metadata block and a handful of downsampled
+ * JPEGs:
+ *
+ *   ## Video File: "demo.mp4"
+ *   Duration: 0:47 | 1920x1080 | avc | 30 fps | 3 keyframes extracted
+ *
+ * Every provider got the same treatment, Gemini included — and Gemini has
+ * accepted inline video, with its audio track, the whole time. A 47-second
+ * screen recording reached it as three still frames at 768px, so anything
+ * between them (a click, a transition, a spoken sentence) was simply not in
+ * the request.
+ *
+ * Two things make that worse than the equivalent audio gap was. First, the
+ * frames come from ffmpeg, and ffmpeg is an optional dependency: on a machine
+ * without it `extractKeyframes` returns an empty array and the model receives
+ * the metadata line alone — a video attachment that conveys nothing visual at
+ * all, with no error. Second, the metadata block answers exactly the questions
+ * a test tends to ask ("how long is it?", "what resolution?"), so the failure
+ * reads as working support until someone asks what happens in the clip.
+ *
+ * Delivering the file itself needs no ffmpeg, which is what makes this the
+ * fix rather than "extract more frames".
+ *
+ * ## Provider scope
+ *
+ * A capability table rather than "send video to everyone". A provider that
+ * cannot accept a video part answers with an opaque HTTP 400, which is
+ * strictly worse than the metadata-plus-frames summary it would otherwise
+ * have received — so an unlisted provider keeps the existing behaviour and
+ * loses nothing.
+ *
+ * ## Why there is no transcode path here
+ *
+ * `audioFormatSupport` re-encodes a container Gemini will not read, because a
+ * voice memo is seconds of audio and the conversion is cheap. The video
+ * equivalent is not: re-encoding a half-hour recording is minutes of CPU
+ * inside a generation request, for a payload that will usually breach the
+ * inline ceiling anyway. An unsupported container therefore falls back to
+ * keyframes, which is the behaviour that already existed.
+ *
+ * @module adapters/videoFormatSupport
+ */
+/**
+ * Ceiling for natively-delivered video in one request, in MB of *source*
+ * bytes.
+ *
+ * Gemini's documented limit is 20 MB for the whole request. Inline data is
+ * base64 in a JSON body, which costs 4 bytes per 3, so a 20 MB source file
+ * arrives as ~26.7 MB of request and is rejected.
+ *
+ * The 20 MB limit is request-wide, not per-part: a video alone consumes it
+ * fully once its encoded size reaches 20 MB, leaving nothing for the prompt,
+ * system instructions, tool schemas or the JSON structure carrying all of
+ * them in the same request. Dividing the raw budget by the 4/3 inflation —
+ * 20 ÷ 4/3 = 15 MB — finds the largest source size whose *video part alone*
+ * still fits, but that calculation has no margin in it: 15 × 4/3 = 20.0 MB
+ * exactly, so any non-empty prompt sharing that request pushes the total
+ * over the documented limit and the request is rejected upstream, for a
+ * reason this module never sees. That was the source of the previous value
+ * here — it satisfied "fits under 20 MB" only for a request containing
+ * nothing else, which no real request is.
+ *
+ * 14 MB is used instead: it still encodes to well under the 20 MB limit
+ * (14 × 4/3 ≈ 18.7 MB), leaving roughly 1.3 MB of headroom in the same
+ * request for the prompt and everything else sharing it.
+ *
+ * This is a request-wide budget, not a per-clip one: two clips at 12 MB each
+ * pass this check individually but together would arrive as ~32 MB of
+ * request. `canDeliverVideoNatively`'s optional `priorNativeVideoBytes`
+ * parameter lets a caller iterating multiple clips in one request charge
+ * each clip against what earlier clips already committed, so the ceiling is
+ * enforced across the whole request rather than reset per clip.
+ *
+ * Clips above this are not an error — they fall back to keyframe extraction,
+ * exactly as before this module existed. Lifting the ceiling properly means
+ * wiring the resumable Files API, which is a separate piece of work.
+ */
+const GEMINI_INLINE_VIDEO_MAX_MB = 14;
+/**
+ * Longest clip Gemini will reason over natively.
+ *
+ * In practice the size ceiling above binds first — 15 MB of H.264 is minutes,
+ * not an hour — so this is a backstop for the case where it does not: a very
+ * low-bitrate or largely static recording can be long and small at once.
+ */
+const GEMINI_MAX_VIDEO_DURATION_SEC = 3600;
+/**
+ * Per-provider video handling.
+ *
+ * Keys are lowercase canonical names and the aliases each provider is
+ * addressed by elsewhere in the codebase; `getVideoProviderConfig` normalises
+ * before looking up, so a caller never has to know which spelling it holds.
+ *
+ * Only Google's Gemini front ends carry `supportsNativeVideo: true`. The rest
+ * are listed deliberately rather than left to the unknown-provider default:
+ * an explicit row is how `getVideoProviderConfig` distinguishes "this provider
+ * takes frames" from "nobody has looked at this provider yet", and the frame
+ * budgets differ enough between them to be worth stating.
+ */
+export const VIDEO_PROVIDER_CONFIGS = Object.freeze({
+    // --- Gemini: inline video, audio track included -------------------------
+    ...Object.fromEntries([
+        "vertex",
+        "google-vertex",
+        "googlevertex",
+        "google-ai-studio",
+        "googleaistudio",
+        "google-ai",
+        "googleai",
+        "gemini",
+    ].map((name) => [
+        name,
+        Object.freeze({
+            supportsNativeVideo: true,
+            apiType: "inline",
+            maxSizeMB: GEMINI_INLINE_VIDEO_MAX_MB,
+            maxDurationSec: GEMINI_MAX_VIDEO_DURATION_SEC,
+            supportsAudio: true,
+            // Only consulted on the fallback path — a clip over the inline
+            // ceiling. Generous because that clip is by definition a long one.
+            recommendedFrameCount: 16,
+        }),
+    ])),
+    // --- Frame extraction: vision models with no video part -----------------
+    ...Object.fromEntries([
+        "openai",
+        "azure",
+        "anthropic",
+        "bedrock",
+        "amazon-bedrock",
+        "mistral",
+        "litellm",
+        "openrouter",
+        "openai-compatible",
+        "huggingface",
+        "sagemaker",
+        "deepseek",
+        "nvidia-nim",
+        "lmstudio",
+        "cerebras",
+        "sambanova",
+    ].map((name) => [
+        name,
+        Object.freeze({
+            supportsNativeVideo: false,
+            apiType: "frame-extraction",
+            maxSizeMB: 0,
+            maxDurationSec: 0,
+            supportsAudio: false,
+            recommendedFrameCount: 8,
+        }),
+    ])),
+    // --- Frame extraction on a tighter budget -------------------------------
+    // Local runtimes hold every frame in the same memory as the model, so a
+    // 16-frame request is the difference between an answer and an OOM.
+    ...Object.fromEntries(["ollama", "llamacpp", "llama-cpp"].map((name) => [
+        name,
+        Object.freeze({
+            supportsNativeVideo: false,
+            apiType: "frame-extraction",
+            maxSizeMB: 0,
+            maxDurationSec: 0,
+            supportsAudio: false,
+            recommendedFrameCount: 4,
+        }),
+    ])),
+});
+/**
+ * Video MIME types Gemini accepts inline.
+ *
+ * Its documented set. A container outside it is left to keyframe extraction
+ * rather than transcoded — see the module header for why.
+ */
+const NATIVE_VIDEO_MIME_TYPES = new Set([
+    "video/mp4",
+    "video/mpeg",
+    "video/mpg",
+    "video/mov",
+    "video/quicktime",
+    "video/avi",
+    "video/x-msvideo",
+    "video/x-flv",
+    "video/webm",
+    "video/wmv",
+    "video/x-ms-wmv",
+    "video/3gpp",
+]);
+/** Tokens one extracted keyframe costs a vision model, approximately. */
+const TOKENS_PER_FRAME = 250;
+/**
+ * Tokens one second of natively-delivered video costs.
+ *
+ * Gemini bills video at roughly 258 tokens per second at default resolution
+ * (one frame per second plus the audio track). Rounded to 260 — the estimate
+ * is for budgeting, and false precision in a constant invites treating it as
+ * a quoted price.
+ */
+const TOKENS_PER_NATIVE_VIDEO_SECOND = 260;
+/** Tokens one minute of transcribed speech costs, approximately. */
+const TOKENS_PER_TRANSCRIPT_MINUTE = 400;
+/**
+ * Duration assumed for a natively-delivered clip whose length is unknown.
+ *
+ * `canDeliverVideoNatively` treats an unknown duration as no reason to
+ * refuse (probing fails on exotic containers and on machines without
+ * ffmpeg — exactly the clips most likely to reach this branch), so a clip
+ * priced here with `durationSec: 0` can still be the one that just travelled
+ * to Gemini as bytes. Pricing that as 0 tokens would make a real payload
+ * look free. 5s is a deliberately small floor: enough that no natively
+ * delivered clip is ever budgeted as free, without pretending to know how
+ * long an unmeasured one actually runs.
+ */
+const MIN_NATIVE_VIDEO_ESTIMATE_SEC = 5;
+function normalizeProvider(provider) {
+    return provider.toLowerCase().trim();
+}
+function normalizeVideoMime(mimeType) {
+    return mimeType.split(";")[0].trim().toLowerCase();
+}
+/** Vertex aliases GoogleVertexProvider serves — the only front end that can
+ * route a single request to either Gemini or Claude depending on model id. */
+const VERTEX_ALIASES = new Set([
+    "vertex",
+    "google-vertex",
+    "googlevertex",
+]);
+/**
+ * Whether `model` is one `GoogleVertexProvider.isAnthropicModel` (client.ts)
+ * would route to the native Anthropic Vertex SDK rather than Gemini.
+ *
+ * Duplicated here rather than imported: `isAnthropicModel` lives in
+ * `providers/googleVertex/client.ts`, and importing a provider module from an
+ * adapter would put `adapters/` on the wrong side of the provider/adapter
+ * dependency direction. `providerImageAdapter.ts` carries the same
+ * `model.includes("claude")` check for the same reason — keep both in sync
+ * if Vertex's routing predicate ever changes.
+ */
+function isClaudeModelId(model) {
+    return model.toLowerCase().includes("claude");
+}
+/**
+ * The video-handling row for `provider`, or null when there is none.
+ *
+ * Null means "not described here", which is not the same as "takes frames":
+ * an unrecognised provider still receives keyframes, because that is the
+ * pipeline's default, but nothing in this table asserts it will understand
+ * them. Callers wanting the safe reading should treat null as no native
+ * video, which is what {@link supportsNativeVideo} does.
+ *
+ * `model` disambiguates Vertex: `GoogleVertexProvider` sends any model whose
+ * id contains "claude" to the native Anthropic SDK, which never sees the
+ * video part at all, so a Claude-on-Vertex request gets the frame-extraction
+ * row (the same one plain `"anthropic"` gets) rather than the Gemini row the
+ * provider name alone would suggest. Omitting `model` keeps the previous,
+ * provider-only lookup, which is still correct for every non-Vertex caller —
+ * AI Studio and the bare `"gemini"` alias never host Claude models.
+ */
+export function getVideoProviderConfig(provider, model) {
+    const normalized = normalizeProvider(provider);
+    if (VERTEX_ALIASES.has(normalized) && model && isClaudeModelId(model)) {
+        return VIDEO_PROVIDER_CONFIGS.anthropic;
+    }
+    return VIDEO_PROVIDER_CONFIGS[normalized] ?? null;
+}
+/** Whether `provider` (optionally, running `model`) can be handed raw video
+ * bytes. See {@link getVideoProviderConfig} for the Claude-on-Vertex case. */
+export function supportsNativeVideo(provider, model) {
+    return getVideoProviderConfig(provider, model)?.supportsNativeVideo ?? false;
+}
+/** Whether `mimeType` is a container a native provider will read as-is. */
+export function isNativeVideoMimeType(mimeType) {
+    return NATIVE_VIDEO_MIME_TYPES.has(normalizeVideoMime(mimeType));
+}
+/**
+ * Rough token cost of putting one video in front of one provider.
+ *
+ * Two quite different prices, because two quite different payloads: a native
+ * provider is billed for the clip's duration, a frame-extraction provider for
+ * the frames it is sent. Asking for one number without saying which mode
+ * applies is how a budget ends up an order of magnitude out, so the provider
+ * decides the formula rather than the caller.
+ *
+ * An estimate, not a quote — resolution settings, prompt text and provider
+ * pricing changes all move the real figure.
+ *
+ * @param options.provider - Provider the video is destined for.
+ * @param options.durationSec - Clip length. 0 when unknown.
+ * @param options.frameCount - Frames that would be extracted. Defaults to the
+ *   provider's `recommendedFrameCount`, or 8 for an unlisted provider.
+ * @param options.hasTranscription - Whether a speech transcript is included.
+ *   Ignored for a native provider, which already hears the audio track.
+ */
+export function estimateVideoTokens(options) {
+    const { provider, durationSec, frameCount, hasTranscription } = options;
+    const config = getVideoProviderConfig(provider);
+    const seconds = Math.max(0, durationSec);
+    if (config?.supportsNativeVideo) {
+        // seconds === 0 covers both "genuinely 0s" and "unmeasured" (the
+        // documented meaning of durationSec: 0 above) — either way a clip that
+        // reaches this branch is being delivered natively, so it is never free.
+        const nativeSeconds = seconds > 0 ? seconds : MIN_NATIVE_VIDEO_ESTIMATE_SEC;
+        return Math.ceil(nativeSeconds * TOKENS_PER_NATIVE_VIDEO_SECOND);
+    }
+    const frames = Math.max(0, frameCount ?? config?.recommendedFrameCount ?? 8);
+    const transcriptTokens = hasTranscription
+        ? Math.ceil(seconds / 60) * TOKENS_PER_TRANSCRIPT_MINUTE
+        : 0;
+    return frames * TOKENS_PER_FRAME + transcriptTokens;
+}
+/**
+ * Whether one specific clip may go to one specific provider as bytes.
+ *
+ * Every rejection carries a reason the caller can log verbatim, because the
+ * user-visible symptom of all of them is identical — "it only described the
+ * file" — and the remedies are not: shorten the clip, re-encode the
+ * container, or switch provider.
+ *
+ * An unknown duration is not a rejection. Probing fails on exotic containers
+ * and on machines without ffmpeg, and refusing a 2 MB clip because nothing
+ * measured it would reintroduce the frames-only behaviour precisely where
+ * frames are least likely to be available.
+ *
+ * @param priorNativeVideoBytes - Source bytes already committed to the
+ *   *same request's* inline budget: every inline image (keyframes
+ *   included), PDF and audio part assembled before the clips, plus each
+ *   clip accepted earlier. The caller accumulates this running total, since
+ *   this function is otherwise stateless and cannot see the rest of the
+ *   request. Defaults to 0, which makes the size check below a pure
+ *   per-clip check.
+ * @param model - See {@link getVideoProviderConfig}: disambiguates a
+ *   Claude-on-Vertex request from a real Gemini-on-Vertex one.
+ */
+export function canDeliverVideoNatively(provider, video, priorNativeVideoBytes = 0, model) {
+    const config = getVideoProviderConfig(provider, model);
+    if (!config?.supportsNativeVideo) {
+        return {
+            deliver: false,
+            reason: `${provider} does not accept video attachments`,
+        };
+    }
+    const mimeType = normalizeVideoMime(video.mimeType);
+    if (!isNativeVideoMimeType(mimeType)) {
+        return {
+            deliver: false,
+            reason: `${mimeType} is not a container ${provider} reads inline`,
+        };
+    }
+    // Charged cumulatively: this clip's bytes plus everything the same request
+    // has already committed to the inline budget (other inline parts and
+    // earlier clips). With the default of 0 this is a per-clip-only check.
+    const aggregateBytes = Math.max(0, priorNativeVideoBytes) + video.buffer.length;
+    const sizeMB = video.buffer.length / (1024 * 1024);
+    const aggregateMB = aggregateBytes / (1024 * 1024);
+    if (aggregateMB > config.maxSizeMB) {
+        const reason = priorNativeVideoBytes > 0
+            ? `the clip is ${sizeMB.toFixed(1)}MB, which would bring this ` +
+                `request's inline attachments to ${aggregateMB.toFixed(1)}MB, ` +
+                `over the ${config.maxSizeMB}MB inline ceiling shared by the whole ` +
+                `request for ${provider}`
+            : `the clip is ${sizeMB.toFixed(1)}MB, over the ${config.maxSizeMB}MB ` +
+                `inline ceiling for ${provider}`;
+        return { deliver: false, reason };
+    }
+    if (typeof video.durationSec === "number" &&
+        Number.isFinite(video.durationSec) &&
+        video.durationSec > config.maxDurationSec) {
+        return {
+            deliver: false,
+            reason: `the clip runs ${Math.round(video.durationSec)}s, over the ` +
+                `${config.maxDurationSec}s ceiling for ${provider}`,
+        };
+    }
+    return { deliver: true };
+}