@juspay/neurolink 9.10.0 → 9.10.1

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [9.10.1](https://github.com/juspay/neurolink/compare/v9.10.0...v9.10.1) (2026-02-21)
+
+### Bug Fixes
+
+- **(video-analysis):** add stream support for video analysis ([938aeef](https://github.com/juspay/neurolink/commit/938aeef876277360700d2a7192155af1f1316f28))
+
 ## [9.10.0](https://github.com/juspay/neurolink/compare/v9.9.0...v9.10.0) (2026-02-20)
 
 ### Features

package/dist/adapters/video/videoAnalyzer.d.ts

@@ -8,16 +8,16 @@
  */
 import { AIProviderName } from "../../constants/enums.js";
 import type { CoreMessage } from "ai";
-export declare function analyzeVideoWithVertexAI(frames: CoreMessage, options?: {
+export declare function analyzeVideoWithVertexAI(messages: CoreMessage[], options?: {
     project?: string;
     location?: string;
     model?: string;
 }): Promise<string>;
-export declare function analyzeVideoWithGeminiAPI(frames: CoreMessage, options?: {
+export declare function analyzeVideoWithGeminiAPI(messages: CoreMessage[], options?: {
     apiKey?: string;
     model?: string;
 }): Promise<string>;
-export declare function analyzeVideo(frames: CoreMessage, options?: {
+export declare function analyzeVideo(messages: CoreMessage[], options?: {
     provider?: AIProviderName;
     project?: string;
     location?: string;

package/dist/adapters/video/videoAnalyzer.js

@@ -9,23 +9,35 @@
 import { AIProviderName, ErrorSeverity, ErrorCategory, } from "../../constants/enums.js";
 import { logger } from "../../utils/logger.js";
 import { readFile } from "node:fs/promises";
-import { NeuroLinkError } from "../../utils/errorHandling.js";
+import { NeuroLinkError, ErrorFactory } from "../../utils/errorHandling.js";
 // ---------------------------------------------------------------------------
 // Shared config
 // ---------------------------------------------------------------------------
 const DEFAULT_MODEL = "gemini-2.0-flash";
 const DEFAULT_LOCATION = "us-central1";
+/**
+ * Extract content items from user messages
+ *
+ * @param messages - Array of CoreMessage objects
+ * @returns Flattened array of content items from user messages
+ */
+function extractUserContent(messages) {
+    const userMessages = messages.filter((msg) => msg.role === "user");
+    return userMessages.flatMap((msg) => Array.isArray(msg.content) ? msg.content : []);
+}
 /**
  * Convert CoreMessage content array to Gemini parts format
  *
- * @param contentArray - Array of content items from CoreMessage
+ * @param messages - Array of CoreMessage objects
  * @returns Array of parts in Gemini API format
  */
-function buildContentParts(frames) {
-    const contentArray = Array.isArray(frames.content) ? frames.content : [];
-    return contentArray.map((item) => {
-        if (item.type === "text" && item.text) {
-            return { text: item.text };
+function buildContentParts(messages) {
+    const allContent = extractUserContent(messages);
+    return allContent
+        .map((item) => {
+        if (item.type === "text") {
+            // Accept text parts regardless of whether text is empty
+            return { text: item.text || "" };
         }
         else if (item.type === "image" && item.image) {
             let base64Data;
@@ -38,7 +50,7 @@ function buildContentParts(frames) {
                 base64Data = item.image.replace(/^data:image\/[a-z]+;base64,/, "");
             }
             else {
-                throw new Error(`Invalid image data type: expected string, Buffer, or Uint8Array, got ${typeof item.image}`);
+                throw ErrorFactory.invalidConfiguration("image data type", `expected string, Buffer, or Uint8Array, got ${typeof item.image}`, { itemType: item.type, dataType: typeof item.image });
             }
             return {
                 inlineData: {
@@ -47,8 +59,14 @@ function buildContentParts(frames) {
                 },
             };
         }
-        throw new Error(`Invalid content type: ${item.type}`);
-    });
+        else if (item.type === "file") {
+            // Skip file parts - not supported in Gemini parts format
+            return null;
+        }
+        // Return null for unsupported types
+        return null;
+    })
+        .filter((part) => part !== null);
 }
 /**
  * Configuration for video frame analysis.
@@ -88,7 +106,7 @@ Ensure the final response is fully self-sufficient and does not reference extern
 // ---------------------------------------------------------------------------
 // Vertex AI
 // ---------------------------------------------------------------------------
-export async function analyzeVideoWithVertexAI(frames, options = {}) {
+export async function analyzeVideoWithVertexAI(messages, options = {}) {
     const startTime = Date.now();
     const { GoogleGenAI } = await import("@google/genai");
     // Get default config and merge with provided options
@@ -96,9 +114,9 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
     const project = options.project ?? config.project;
     const location = options.location ?? config.location;
     const model = options.model || DEFAULT_MODEL;
-    // Extract content array from CoreMessage
-    const contentArray = Array.isArray(frames.content) ? frames.content : [];
-    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(messages);
+    const frameCount = parts.filter((part) => "inlineData" in part && part.inlineData).length;
     logger.debug("[GeminiVideoAnalyzer] Analyzing video with Vertex AI", {
         project,
         location,
@@ -106,8 +124,6 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
         frameCount,
     });
     const ai = new GoogleGenAI({ vertexai: true, project, location });
-    // Convert frames content to parts array for Gemini
-    const parts = buildContentParts(frames);
     const response = await ai.models.generateContent({
         model,
         config: buildConfig(),
@@ -129,7 +145,7 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
 // ---------------------------------------------------------------------------
 // Gemini API (Google AI)
 // ---------------------------------------------------------------------------
-export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
+export async function analyzeVideoWithGeminiAPI(messages, options = {}) {
     const startTime = Date.now();
     const { GoogleGenAI } = await import("@google/genai");
     const apiKey = options.apiKey || process.env.GOOGLE_AI_API_KEY;
@@ -137,16 +153,14 @@ export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
     if (!apiKey) {
         throw new Error("GOOGLE_AI_API_KEY environment variable is required for Gemini API video analysis");
     }
-    // Extract content array from CoreMessage
-    const contentArray = Array.isArray(frames.content) ? frames.content : [];
-    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(messages);
+    const frameCount = parts.filter((part) => "inlineData" in part && part.inlineData).length;
     logger.debug("[GeminiVideoAnalyzer] Analyzing video with Gemini API", {
         model,
         frameCount,
     });
     const ai = new GoogleGenAI({ apiKey });
-    // Convert frames content to parts array for Gemini
-    const parts = buildContentParts(frames);
     logger.debug("[GeminiVideoAnalyzer] Generating analysis with frames");
     const response = await ai.models.generateContent({
         model,
@@ -207,15 +221,15 @@ async function getVertexConfig() {
     }
     return { project, location };
 }
-export async function analyzeVideo(frames, options = {}) {
+export async function analyzeVideo(messages, options = {}) {
     const provider = options.provider || AIProviderName.AUTO;
     // Vertex — only when GOOGLE_VERTEX_PROJECT is explicitly set
     if (provider === AIProviderName.VERTEX || provider === AIProviderName.AUTO) {
-        return analyzeVideoWithVertexAI(frames, options);
+        return analyzeVideoWithVertexAI(messages, options);
     }
     // Gemini API — when GOOGLE_AI_API_KEY is set
     if (provider === AIProviderName.GOOGLE_AI && process.env.GOOGLE_AI_API_KEY) {
-        return analyzeVideoWithGeminiAPI(frames, options);
+        return analyzeVideoWithGeminiAPI(messages, options);
     }
     throw new Error("No valid provider configuration found. " +
         "Set GOOGLE_VERTEX_PROJECT for Vertex AI or GOOGLE_AI_API_KEY for Gemini API.");

package/dist/core/baseProvider.js

@@ -86,6 +86,20 @@ export class BaseProvider {
             temperature: options.temperature,
             timestamp: Date.now(),
         });
+        // ===== EARLY MULTIMODAL DETECTION =====
+        const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
+        if (hasFileInput) {
+            // ===== VIDEO ANALYSIS DETECTION =====
+            // Check if video frames are present and handle with fake streaming
+            const messages = await this.buildMessagesForStream(options);
+            if (hasVideoFrames(messages)) {
+                logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
+                    provider: this.providerName,
+                    model: this.modelName,
+                });
+                return await this.executeFakeStreaming(options, analysisSchema);
+            }
+        }
         // 🔧 CRITICAL: Image generation models don't support real streaming
         // Force fake streaming for image models to ensure image output is yielded
         const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));

package/dist/lib/adapters/video/videoAnalyzer.d.ts

@@ -8,16 +8,16 @@
  */
 import { AIProviderName } from "../../constants/enums.js";
 import type { CoreMessage } from "ai";
-export declare function analyzeVideoWithVertexAI(frames: CoreMessage, options?: {
+export declare function analyzeVideoWithVertexAI(messages: CoreMessage[], options?: {
     project?: string;
     location?: string;
     model?: string;
 }): Promise<string>;
-export declare function analyzeVideoWithGeminiAPI(frames: CoreMessage, options?: {
+export declare function analyzeVideoWithGeminiAPI(messages: CoreMessage[], options?: {
     apiKey?: string;
     model?: string;
 }): Promise<string>;
-export declare function analyzeVideo(frames: CoreMessage, options?: {
+export declare function analyzeVideo(messages: CoreMessage[], options?: {
     provider?: AIProviderName;
     project?: string;
     location?: string;

package/dist/lib/adapters/video/videoAnalyzer.js

@@ -9,23 +9,35 @@
 import { AIProviderName, ErrorSeverity, ErrorCategory, } from "../../constants/enums.js";
 import { logger } from "../../utils/logger.js";
 import { readFile } from "node:fs/promises";
-import { NeuroLinkError } from "../../utils/errorHandling.js";
+import { NeuroLinkError, ErrorFactory } from "../../utils/errorHandling.js";
 // ---------------------------------------------------------------------------
 // Shared config
 // ---------------------------------------------------------------------------
 const DEFAULT_MODEL = "gemini-2.0-flash";
 const DEFAULT_LOCATION = "us-central1";
+/**
+ * Extract content items from user messages
+ *
+ * @param messages - Array of CoreMessage objects
+ * @returns Flattened array of content items from user messages
+ */
+function extractUserContent(messages) {
+    const userMessages = messages.filter((msg) => msg.role === "user");
+    return userMessages.flatMap((msg) => Array.isArray(msg.content) ? msg.content : []);
+}
 /**
  * Convert CoreMessage content array to Gemini parts format
  *
- * @param contentArray - Array of content items from CoreMessage
+ * @param messages - Array of CoreMessage objects
  * @returns Array of parts in Gemini API format
  */
-function buildContentParts(frames) {
-    const contentArray = Array.isArray(frames.content) ? frames.content : [];
-    return contentArray.map((item) => {
-        if (item.type === "text" && item.text) {
-            return { text: item.text };
+function buildContentParts(messages) {
+    const allContent = extractUserContent(messages);
+    return allContent
+        .map((item) => {
+        if (item.type === "text") {
+            // Accept text parts regardless of whether text is empty
+            return { text: item.text || "" };
         }
         else if (item.type === "image" && item.image) {
             let base64Data;
@@ -38,7 +50,7 @@ function buildContentParts(frames) {
                 base64Data = item.image.replace(/^data:image\/[a-z]+;base64,/, "");
             }
             else {
-                throw new Error(`Invalid image data type: expected string, Buffer, or Uint8Array, got ${typeof item.image}`);
+                throw ErrorFactory.invalidConfiguration("image data type", `expected string, Buffer, or Uint8Array, got ${typeof item.image}`, { itemType: item.type, dataType: typeof item.image });
             }
             return {
                 inlineData: {
@@ -47,8 +59,14 @@ function buildContentParts(frames) {
                 },
             };
         }
-        throw new Error(`Invalid content type: ${item.type}`);
-    });
+        else if (item.type === "file") {
+            // Skip file parts - not supported in Gemini parts format
+            return null;
+        }
+        // Return null for unsupported types
+        return null;
+    })
+        .filter((part) => part !== null);
 }
 /**
  * Configuration for video frame analysis.
@@ -88,7 +106,7 @@ Ensure the final response is fully self-sufficient and does not reference extern
 // ---------------------------------------------------------------------------
 // Vertex AI
 // ---------------------------------------------------------------------------
-export async function analyzeVideoWithVertexAI(frames, options = {}) {
+export async function analyzeVideoWithVertexAI(messages, options = {}) {
     const startTime = Date.now();
     const { GoogleGenAI } = await import("@google/genai");
     // Get default config and merge with provided options
@@ -96,9 +114,9 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
     const project = options.project ?? config.project;
     const location = options.location ?? config.location;
     const model = options.model || DEFAULT_MODEL;
-    // Extract content array from CoreMessage
-    const contentArray = Array.isArray(frames.content) ? frames.content : [];
-    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(messages);
+    const frameCount = parts.filter((part) => "inlineData" in part && part.inlineData).length;
     logger.debug("[GeminiVideoAnalyzer] Analyzing video with Vertex AI", {
         project,
         location,
@@ -106,8 +124,6 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
         frameCount,
     });
     const ai = new GoogleGenAI({ vertexai: true, project, location });
-    // Convert frames content to parts array for Gemini
-    const parts = buildContentParts(frames);
     const response = await ai.models.generateContent({
         model,
         config: buildConfig(),
@@ -129,7 +145,7 @@ export async function analyzeVideoWithVertexAI(frames, options = {}) {
 // ---------------------------------------------------------------------------
 // Gemini API (Google AI)
 // ---------------------------------------------------------------------------
-export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
+export async function analyzeVideoWithGeminiAPI(messages, options = {}) {
     const startTime = Date.now();
     const { GoogleGenAI } = await import("@google/genai");
     const apiKey = options.apiKey || process.env.GOOGLE_AI_API_KEY;
@@ -137,16 +153,14 @@ export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
     if (!apiKey) {
         throw new Error("GOOGLE_AI_API_KEY environment variable is required for Gemini API video analysis");
     }
-    // Extract content array from CoreMessage
-    const contentArray = Array.isArray(frames.content) ? frames.content : [];
-    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(messages);
+    const frameCount = parts.filter((part) => "inlineData" in part && part.inlineData).length;
     logger.debug("[GeminiVideoAnalyzer] Analyzing video with Gemini API", {
         model,
         frameCount,
     });
     const ai = new GoogleGenAI({ apiKey });
-    // Convert frames content to parts array for Gemini
-    const parts = buildContentParts(frames);
     logger.debug("[GeminiVideoAnalyzer] Generating analysis with frames");
     const response = await ai.models.generateContent({
         model,
@@ -207,15 +221,15 @@ async function getVertexConfig() {
     }
     return { project, location };
 }
-export async function analyzeVideo(frames, options = {}) {
+export async function analyzeVideo(messages, options = {}) {
     const provider = options.provider || AIProviderName.AUTO;
     // Vertex — only when GOOGLE_VERTEX_PROJECT is explicitly set
     if (provider === AIProviderName.VERTEX || provider === AIProviderName.AUTO) {
-        return analyzeVideoWithVertexAI(frames, options);
+        return analyzeVideoWithVertexAI(messages, options);
     }
     // Gemini API — when GOOGLE_AI_API_KEY is set
     if (provider === AIProviderName.GOOGLE_AI && process.env.GOOGLE_AI_API_KEY) {
-        return analyzeVideoWithGeminiAPI(frames, options);
+        return analyzeVideoWithGeminiAPI(messages, options);
     }
     throw new Error("No valid provider configuration found. " +
         "Set GOOGLE_VERTEX_PROJECT for Vertex AI or GOOGLE_AI_API_KEY for Gemini API.");

package/dist/lib/core/baseProvider.js

@@ -86,6 +86,20 @@ export class BaseProvider {
             temperature: options.temperature,
             timestamp: Date.now(),
         });
+        // ===== EARLY MULTIMODAL DETECTION =====
+        const hasFileInput = !!options.input?.files?.length || !!options.input?.videoFiles?.length;
+        if (hasFileInput) {
+            // ===== VIDEO ANALYSIS DETECTION =====
+            // Check if video frames are present and handle with fake streaming
+            const messages = await this.buildMessagesForStream(options);
+            if (hasVideoFrames(messages)) {
+                logger.info(`Video frames detected in stream, using fake streaming for video analysis`, {
+                    provider: this.providerName,
+                    model: this.modelName,
+                });
+                return await this.executeFakeStreaming(options, analysisSchema);
+            }
+        }
         // 🔧 CRITICAL: Image generation models don't support real streaming
         // Force fake streaming for image models to ensure image output is yielded
         const isImageModel = IMAGE_GENERATION_MODELS.some((m) => this.modelName.includes(m));

package/dist/lib/utils/videoAnalysisProcessor.d.ts

@@ -9,9 +9,10 @@ import type { CoreMessage } from "ai";
 import { AIProviderName } from "../constants/enums.js";
 /**
  * Check if messages contain video frames (images)
+ * Only checks user messages to match buildContentParts behavior
  *
  * @param messages - Array of CoreMessage objects
- * @returns true if video frames are present
+ * @returns true if video frames are present in user messages
  */
 export declare function hasVideoFrames(messages: CoreMessage[]): boolean;
 /**

package/dist/lib/utils/videoAnalysisProcessor.js

@@ -9,12 +9,17 @@ import { AIProviderName } from "../constants/enums.js";
 import { logger } from "./logger.js";
 /**
  * Check if messages contain video frames (images)
+ * Only checks user messages to match buildContentParts behavior
  *
  * @param messages - Array of CoreMessage objects
- * @returns true if video frames are present
+ * @returns true if video frames are present in user messages
  */
 export function hasVideoFrames(messages) {
     return messages.some((msg) => {
+        // Only check user messages to match buildContentParts behavior
+        if (msg.role !== "user") {
+            return false;
+        }
         if (Array.isArray(msg.content)) {
             return msg.content.some((part) => typeof part === "object" &&
                 part !== null &&
@@ -42,7 +47,7 @@ export async function executeVideoAnalysis(messages, options) {
             options.providerName === AIProviderName.VERTEX
             ? AIProviderName.VERTEX
             : AIProviderName.AUTO;
-    const videoAnalysisText = await analyzeVideo(messages[0], {
+    const videoAnalysisText = await analyzeVideo(messages, {
         provider: provider,
         project: options.region
             ? undefined

package/dist/utils/videoAnalysisProcessor.d.ts

@@ -9,9 +9,10 @@ import type { CoreMessage } from "ai";
 import { AIProviderName } from "../constants/enums.js";
 /**
  * Check if messages contain video frames (images)
+ * Only checks user messages to match buildContentParts behavior
  *
  * @param messages - Array of CoreMessage objects
- * @returns true if video frames are present
+ * @returns true if video frames are present in user messages
  */
 export declare function hasVideoFrames(messages: CoreMessage[]): boolean;
 /**

package/dist/utils/videoAnalysisProcessor.js

@@ -9,12 +9,17 @@ import { AIProviderName } from "../constants/enums.js";
 import { logger } from "./logger.js";
 /**
  * Check if messages contain video frames (images)
+ * Only checks user messages to match buildContentParts behavior
  *
  * @param messages - Array of CoreMessage objects
- * @returns true if video frames are present
+ * @returns true if video frames are present in user messages
  */
 export function hasVideoFrames(messages) {
     return messages.some((msg) => {
+        // Only check user messages to match buildContentParts behavior
+        if (msg.role !== "user") {
+            return false;
+        }
         if (Array.isArray(msg.content)) {
             return msg.content.some((part) => typeof part === "object" &&
                 part !== null &&
@@ -42,7 +47,7 @@ export async function executeVideoAnalysis(messages, options) {
             options.providerName === AIProviderName.VERTEX
             ? AIProviderName.VERTEX
             : AIProviderName.AUTO;
-    const videoAnalysisText = await analyzeVideo(messages[0], {
+    const videoAnalysisText = await analyzeVideo(messages, {
         provider: provider,
         project: options.region
             ? undefined

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "9.10.0",
+  "version": "9.10.1",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 13 providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",