@juspay/neurolink 9.7.0 → 9.9.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [9.9.0](https://github.com/juspay/neurolink/compare/v9.8.0...v9.9.0) (2026-02-17)
+
+### Features
+
+- **(video-analysis):** add video-analysis support in neurolink ([c35f8a8](https://github.com/juspay/neurolink/commit/c35f8a8d52cc1366e10b8701285e1bec52e27d98))
+
+## [9.8.0](https://github.com/juspay/neurolink/compare/v9.7.0...v9.8.0) (2026-02-17)
+
+### Features
+
+- **(scripts):** migrate peripheral JS/CJS/MJS files to TypeScript with tsx runner ([25f17e8](https://github.com/juspay/neurolink/commit/25f17e81c09d6184a4bf2ff15e4427c302fde55e))
+
 ## [9.7.0](https://github.com/juspay/neurolink/compare/v9.6.0...v9.7.0) (2026-02-16)
 
 ### Features

package/dist/adapters/video/videoAnalyzer.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Video Analysis Handler
+ *
+ * Provides video analysis using Google's Gemini 2.0 Flash model.
+ * Supports both Vertex AI and Gemini API providers.
+ *
+ * @module adapters/video/geminiVideoAnalyzer
+ */
+import { AIProviderName } from "../../constants/enums.js";
+import type { CoreMessage } from "ai";
+export declare function analyzeVideoWithVertexAI(frames: CoreMessage, options?: {
+    project?: string;
+    location?: string;
+    model?: string;
+}): Promise<string>;
+export declare function analyzeVideoWithGeminiAPI(frames: CoreMessage, options?: {
+    apiKey?: string;
+    model?: string;
+}): Promise<string>;
+export declare function analyzeVideo(frames: CoreMessage, options?: {
+    provider?: AIProviderName;
+    project?: string;
+    location?: string;
+    apiKey?: string;
+    model?: string;
+}): Promise<string>;

package/dist/adapters/video/videoAnalyzer.js

@@ -0,0 +1,222 @@
+/**
+ * Video Analysis Handler
+ *
+ * Provides video analysis using Google's Gemini 2.0 Flash model.
+ * Supports both Vertex AI and Gemini API providers.
+ *
+ * @module adapters/video/geminiVideoAnalyzer
+ */
+import { AIProviderName, ErrorSeverity, ErrorCategory, } from "../../constants/enums.js";
+import { logger } from "../../utils/logger.js";
+import { readFile } from "node:fs/promises";
+import { NeuroLinkError } from "../../utils/errorHandling.js";
+// ---------------------------------------------------------------------------
+// Shared config
+// ---------------------------------------------------------------------------
+const DEFAULT_MODEL = "gemini-2.0-flash";
+const DEFAULT_LOCATION = "us-central1";
+/**
+ * Convert CoreMessage content array to Gemini parts format
+ *
+ * @param contentArray - Array of content items from CoreMessage
+ * @returns Array of parts in Gemini API format
+ */
+function buildContentParts(frames) {
+    const contentArray = Array.isArray(frames.content) ? frames.content : [];
+    return contentArray.map((item) => {
+        if (item.type === "text" && item.text) {
+            return { text: item.text };
+        }
+        else if (item.type === "image" && item.image) {
+            let base64Data;
+            // Handle Buffer or Uint8Array
+            if (Buffer.isBuffer(item.image) || item.image instanceof Uint8Array) {
+                base64Data = Buffer.from(item.image).toString("base64");
+            }
+            else if (typeof item.image === "string") {
+                // Strip data URI prefix if present (e.g., "data:image/jpeg;base64,")
+                base64Data = item.image.replace(/^data:image\/[a-z]+;base64,/, "");
+            }
+            else {
+                throw new Error(`Invalid image data type: expected string, Buffer, or Uint8Array, got ${typeof item.image}`);
+            }
+            return {
+                inlineData: {
+                    mimeType: "image/jpeg",
+                    data: base64Data,
+                },
+            };
+        }
+        throw new Error(`Invalid content type: ${item.type}`);
+    });
+}
+/**
+ * Configuration for video frame analysis.
+ * Generic prompt that handles both general content and technical bug reporting.
+ */
+function buildConfig() {
+    return {
+        systemInstruction: `You are a Visual Analysis Assistant.
+Your task is to analyze images or video frames provided by the user and extract structured visual features. The user may or may not provide an issue description. Your role is to understand the visual content, optionally correlate it with the provided issue, and produce a structured output that can be directly consumed by another LLM for analysis, debugging, or decision-making.
+
+Follow these rules strictly:
+- The analysis must be generic and applicable to any domain (UI, dashboards, video frames, animations, charts, documents, etc.).
+- Support both images and videos (single frame or multiple frames).
+- Extract only what is visually observable; do not assume backend behavior unless supported by visuals.
+- The JSON must be structured, consistent, and machine-readable.
+- Logs are optional and should only be included if explicitly provided.
+- The final output must be clear, concise, and actionable for an LLM.
+
+Always produce the output in the following format:
+
+Issue:
+<Refined issue description if provided, otherwise a clear description of the observed visual situation>
+
+Image/Video Patterns:
+<Structured JSON describing extracted visual features and anomalies>
+
+Steps to Reproduce:
+<Ordered steps that reliably reproduce the issue based on the visual context>
+
+[Logs: Include ONLY if provided by the user]
+
+Proof:
+<Visual evidence explaining how the image/video confirms the issue>
+Ensure the final response is fully self-sufficient and does not reference external context.`,
+    };
+}
+// ---------------------------------------------------------------------------
+// Vertex AI
+// ---------------------------------------------------------------------------
+export async function analyzeVideoWithVertexAI(frames, options = {}) {
+    const startTime = Date.now();
+    const { GoogleGenAI } = await import("@google/genai");
+    // Get default config and merge with provided options
+    const config = await getVertexConfig();
+    const project = options.project ?? config.project;
+    const location = options.location ?? config.location;
+    const model = options.model || DEFAULT_MODEL;
+    // Extract content array from CoreMessage
+    const contentArray = Array.isArray(frames.content) ? frames.content : [];
+    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    logger.debug("[GeminiVideoAnalyzer] Analyzing video with Vertex AI", {
+        project,
+        location,
+        model,
+        frameCount,
+    });
+    const ai = new GoogleGenAI({ vertexai: true, project, location });
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(frames);
+    const response = await ai.models.generateContent({
+        model,
+        config: buildConfig(),
+        contents: [
+            {
+                role: "user",
+                parts,
+            },
+        ],
+    });
+    const responseText = response.text || "";
+    const processingTime = Date.now() - startTime;
+    logger.debug("[GeminiVideoAnalyzer] Vertex response received", {
+        responseLength: responseText.length,
+        processingTime,
+    });
+    return responseText;
+}
+// ---------------------------------------------------------------------------
+// Gemini API (Google AI)
+// ---------------------------------------------------------------------------
+export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
+    const startTime = Date.now();
+    const { GoogleGenAI } = await import("@google/genai");
+    const apiKey = options.apiKey || process.env.GOOGLE_AI_API_KEY;
+    const model = options.model || DEFAULT_MODEL;
+    if (!apiKey) {
+        throw new Error("GOOGLE_AI_API_KEY environment variable is required for Gemini API video analysis");
+    }
+    // Extract content array from CoreMessage
+    const contentArray = Array.isArray(frames.content) ? frames.content : [];
+    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    logger.debug("[GeminiVideoAnalyzer] Analyzing video with Gemini API", {
+        model,
+        frameCount,
+    });
+    const ai = new GoogleGenAI({ apiKey });
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(frames);
+    logger.debug("[GeminiVideoAnalyzer] Generating analysis with frames");
+    const response = await ai.models.generateContent({
+        model,
+        config: buildConfig(),
+        contents: [
+            {
+                role: "user",
+                parts,
+            },
+        ],
+    });
+    const responseText = response.text || "";
+    const processingTime = Date.now() - startTime;
+    logger.debug("[GeminiVideoAnalyzer] Gemini API response received", {
+        responseLength: responseText.length,
+        processingTime,
+    });
+    return responseText;
+}
+async function getVertexConfig() {
+    const location = process.env.GOOGLE_VERTEX_LOCATION || DEFAULT_LOCATION;
+    // Try environment variables first
+    let project = process.env.GOOGLE_VERTEX_PROJECT ||
+        process.env.GOOGLE_CLOUD_PROJECT ||
+        process.env.GOOGLE_CLOUD_PROJECT_ID ||
+        process.env.VERTEX_PROJECT_ID;
+    // Fallback: read from ADC credentials file
+    if (!project && process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+        try {
+            const credData = JSON.parse(await readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS, "utf-8"));
+            project = credData.quota_project_id || credData.project_id;
+        }
+        catch (e) {
+            // Ignore read errors, will throw below if project still not found
+            logger.debug("Failed to read project from credentials file", {
+                error: e instanceof Error ? e.message : String(e),
+            });
+        }
+    }
+    if (!project) {
+        throw new NeuroLinkError({
+            code: "PROVIDER_NOT_CONFIGURED",
+            message: "Google Cloud project not found. Set GOOGLE_VERTEX_PROJECT or GOOGLE_CLOUD_PROJECT environment variable, or ensure ADC credentials contain project_id",
+            category: ErrorCategory.CONFIGURATION,
+            severity: ErrorSeverity.HIGH,
+            retriable: false,
+            context: {
+                missingVar: "GOOGLE_VERTEX_PROJECT",
+                feature: "video-generation",
+                checkedEnvVars: [
+                    "GOOGLE_VERTEX_PROJECT",
+                    "GOOGLE_CLOUD_PROJECT",
+                    "GOOGLE_CLOUD_PROJECT_ID",
+                    "VERTEX_PROJECT_ID",
+                ],
+            },
+        });
+    }
+    return { project, location };
+}
+export async function analyzeVideo(frames, options = {}) {
+    const provider = options.provider || AIProviderName.AUTO;
+    // Vertex — only when GOOGLE_VERTEX_PROJECT is explicitly set
+    if (provider === AIProviderName.VERTEX || provider === AIProviderName.AUTO) {
+        return analyzeVideoWithVertexAI(frames, options);
+    }
+    // Gemini API — when GOOGLE_AI_API_KEY is set
+    if (provider === AIProviderName.GOOGLE_AI && process.env.GOOGLE_AI_API_KEY) {
+        return analyzeVideoWithGeminiAPI(frames, options);
+    }
+    throw new Error("No valid provider configuration found. " +
+        "Set GOOGLE_VERTEX_PROJECT for Vertex AI or GOOGLE_AI_API_KEY for Gemini API.");
+}

package/dist/core/baseProvider.js

@@ -7,6 +7,7 @@ import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../
 import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
 import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
 import { TTSProcessor } from "../utils/ttsProcessor.js";
+import { hasVideoFrames, executeVideoAnalysis, } from "../utils/videoAnalysisProcessor.js";
 import { GenerationHandler } from "./modules/GenerationHandler.js";
 // Import modules for composition
 import { MessageBuilder } from "./modules/MessageBuilder.js";
@@ -473,6 +474,25 @@ export class BaseProvider {
             // ===== Normal AI Generation Flow =====
             const { tools, model } = await this.prepareGenerationContext(options);
             const messages = await this.buildMessages(options);
+            // ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
+            // Check if video files are present in messages content array
+            // If video analysis is needed, perform it and return early to avoid running generation
+            if (hasVideoFrames(messages)) {
+                const videoAnalysisResult = await executeVideoAnalysis(messages, {
+                    provider: options.provider,
+                    providerName: this.providerName,
+                    region: options.region,
+                    model: options.model,
+                });
+                // Return video analysis result directly without running generation
+                const videoResult = {
+                    content: videoAnalysisResult,
+                    provider: options.provider ?? this.providerName,
+                    model: this.modelName,
+                    usage: { input: 0, output: 0, total: 0 }, // Video analysis doesn't use standard token counting
+                };
+                return await this.enhanceResult(videoResult, options, startTime);
+            }
             // Compose timeout signal with user-provided abort signal (mirrors stream path)
             const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
             const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);

package/dist/lib/adapters/video/videoAnalyzer.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Video Analysis Handler
+ *
+ * Provides video analysis using Google's Gemini 2.0 Flash model.
+ * Supports both Vertex AI and Gemini API providers.
+ *
+ * @module adapters/video/geminiVideoAnalyzer
+ */
+import { AIProviderName } from "../../constants/enums.js";
+import type { CoreMessage } from "ai";
+export declare function analyzeVideoWithVertexAI(frames: CoreMessage, options?: {
+    project?: string;
+    location?: string;
+    model?: string;
+}): Promise<string>;
+export declare function analyzeVideoWithGeminiAPI(frames: CoreMessage, options?: {
+    apiKey?: string;
+    model?: string;
+}): Promise<string>;
+export declare function analyzeVideo(frames: CoreMessage, options?: {
+    provider?: AIProviderName;
+    project?: string;
+    location?: string;
+    apiKey?: string;
+    model?: string;
+}): Promise<string>;

package/dist/lib/adapters/video/videoAnalyzer.js

@@ -0,0 +1,223 @@
+/**
+ * Video Analysis Handler
+ *
+ * Provides video analysis using Google's Gemini 2.0 Flash model.
+ * Supports both Vertex AI and Gemini API providers.
+ *
+ * @module adapters/video/geminiVideoAnalyzer
+ */
+import { AIProviderName, ErrorSeverity, ErrorCategory, } from "../../constants/enums.js";
+import { logger } from "../../utils/logger.js";
+import { readFile } from "node:fs/promises";
+import { NeuroLinkError } from "../../utils/errorHandling.js";
+// ---------------------------------------------------------------------------
+// Shared config
+// ---------------------------------------------------------------------------
+const DEFAULT_MODEL = "gemini-2.0-flash";
+const DEFAULT_LOCATION = "us-central1";
+/**
+ * Convert CoreMessage content array to Gemini parts format
+ *
+ * @param contentArray - Array of content items from CoreMessage
+ * @returns Array of parts in Gemini API format
+ */
+function buildContentParts(frames) {
+    const contentArray = Array.isArray(frames.content) ? frames.content : [];
+    return contentArray.map((item) => {
+        if (item.type === "text" && item.text) {
+            return { text: item.text };
+        }
+        else if (item.type === "image" && item.image) {
+            let base64Data;
+            // Handle Buffer or Uint8Array
+            if (Buffer.isBuffer(item.image) || item.image instanceof Uint8Array) {
+                base64Data = Buffer.from(item.image).toString("base64");
+            }
+            else if (typeof item.image === "string") {
+                // Strip data URI prefix if present (e.g., "data:image/jpeg;base64,")
+                base64Data = item.image.replace(/^data:image\/[a-z]+;base64,/, "");
+            }
+            else {
+                throw new Error(`Invalid image data type: expected string, Buffer, or Uint8Array, got ${typeof item.image}`);
+            }
+            return {
+                inlineData: {
+                    mimeType: "image/jpeg",
+                    data: base64Data,
+                },
+            };
+        }
+        throw new Error(`Invalid content type: ${item.type}`);
+    });
+}
+/**
+ * Configuration for video frame analysis.
+ * Generic prompt that handles both general content and technical bug reporting.
+ */
+function buildConfig() {
+    return {
+        systemInstruction: `You are a Visual Analysis Assistant.
+Your task is to analyze images or video frames provided by the user and extract structured visual features. The user may or may not provide an issue description. Your role is to understand the visual content, optionally correlate it with the provided issue, and produce a structured output that can be directly consumed by another LLM for analysis, debugging, or decision-making.
+
+Follow these rules strictly:
+- The analysis must be generic and applicable to any domain (UI, dashboards, video frames, animations, charts, documents, etc.).
+- Support both images and videos (single frame or multiple frames).
+- Extract only what is visually observable; do not assume backend behavior unless supported by visuals.
+- The JSON must be structured, consistent, and machine-readable.
+- Logs are optional and should only be included if explicitly provided.
+- The final output must be clear, concise, and actionable for an LLM.
+
+Always produce the output in the following format:
+
+Issue:
+<Refined issue description if provided, otherwise a clear description of the observed visual situation>
+
+Image/Video Patterns:
+<Structured JSON describing extracted visual features and anomalies>
+
+Steps to Reproduce:
+<Ordered steps that reliably reproduce the issue based on the visual context>
+
+[Logs: Include ONLY if provided by the user]
+
+Proof:
+<Visual evidence explaining how the image/video confirms the issue>
+Ensure the final response is fully self-sufficient and does not reference external context.`,
+    };
+}
+// ---------------------------------------------------------------------------
+// Vertex AI
+// ---------------------------------------------------------------------------
+export async function analyzeVideoWithVertexAI(frames, options = {}) {
+    const startTime = Date.now();
+    const { GoogleGenAI } = await import("@google/genai");
+    // Get default config and merge with provided options
+    const config = await getVertexConfig();
+    const project = options.project ?? config.project;
+    const location = options.location ?? config.location;
+    const model = options.model || DEFAULT_MODEL;
+    // Extract content array from CoreMessage
+    const contentArray = Array.isArray(frames.content) ? frames.content : [];
+    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    logger.debug("[GeminiVideoAnalyzer] Analyzing video with Vertex AI", {
+        project,
+        location,
+        model,
+        frameCount,
+    });
+    const ai = new GoogleGenAI({ vertexai: true, project, location });
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(frames);
+    const response = await ai.models.generateContent({
+        model,
+        config: buildConfig(),
+        contents: [
+            {
+                role: "user",
+                parts,
+            },
+        ],
+    });
+    const responseText = response.text || "";
+    const processingTime = Date.now() - startTime;
+    logger.debug("[GeminiVideoAnalyzer] Vertex response received", {
+        responseLength: responseText.length,
+        processingTime,
+    });
+    return responseText;
+}
+// ---------------------------------------------------------------------------
+// Gemini API (Google AI)
+// ---------------------------------------------------------------------------
+export async function analyzeVideoWithGeminiAPI(frames, options = {}) {
+    const startTime = Date.now();
+    const { GoogleGenAI } = await import("@google/genai");
+    const apiKey = options.apiKey || process.env.GOOGLE_AI_API_KEY;
+    const model = options.model || DEFAULT_MODEL;
+    if (!apiKey) {
+        throw new Error("GOOGLE_AI_API_KEY environment variable is required for Gemini API video analysis");
+    }
+    // Extract content array from CoreMessage
+    const contentArray = Array.isArray(frames.content) ? frames.content : [];
+    const frameCount = contentArray.filter((item) => item.type === "image").length;
+    logger.debug("[GeminiVideoAnalyzer] Analyzing video with Gemini API", {
+        model,
+        frameCount,
+    });
+    const ai = new GoogleGenAI({ apiKey });
+    // Convert frames content to parts array for Gemini
+    const parts = buildContentParts(frames);
+    logger.debug("[GeminiVideoAnalyzer] Generating analysis with frames");
+    const response = await ai.models.generateContent({
+        model,
+        config: buildConfig(),
+        contents: [
+            {
+                role: "user",
+                parts,
+            },
+        ],
+    });
+    const responseText = response.text || "";
+    const processingTime = Date.now() - startTime;
+    logger.debug("[GeminiVideoAnalyzer] Gemini API response received", {
+        responseLength: responseText.length,
+        processingTime,
+    });
+    return responseText;
+}
+async function getVertexConfig() {
+    const location = process.env.GOOGLE_VERTEX_LOCATION || DEFAULT_LOCATION;
+    // Try environment variables first
+    let project = process.env.GOOGLE_VERTEX_PROJECT ||
+        process.env.GOOGLE_CLOUD_PROJECT ||
+        process.env.GOOGLE_CLOUD_PROJECT_ID ||
+        process.env.VERTEX_PROJECT_ID;
+    // Fallback: read from ADC credentials file
+    if (!project && process.env.GOOGLE_APPLICATION_CREDENTIALS) {
+        try {
+            const credData = JSON.parse(await readFile(process.env.GOOGLE_APPLICATION_CREDENTIALS, "utf-8"));
+            project = credData.quota_project_id || credData.project_id;
+        }
+        catch (e) {
+            // Ignore read errors, will throw below if project still not found
+            logger.debug("Failed to read project from credentials file", {
+                error: e instanceof Error ? e.message : String(e),
+            });
+        }
+    }
+    if (!project) {
+        throw new NeuroLinkError({
+            code: "PROVIDER_NOT_CONFIGURED",
+            message: "Google Cloud project not found. Set GOOGLE_VERTEX_PROJECT or GOOGLE_CLOUD_PROJECT environment variable, or ensure ADC credentials contain project_id",
+            category: ErrorCategory.CONFIGURATION,
+            severity: ErrorSeverity.HIGH,
+            retriable: false,
+            context: {
+                missingVar: "GOOGLE_VERTEX_PROJECT",
+                feature: "video-generation",
+                checkedEnvVars: [
+                    "GOOGLE_VERTEX_PROJECT",
+                    "GOOGLE_CLOUD_PROJECT",
+                    "GOOGLE_CLOUD_PROJECT_ID",
+                    "VERTEX_PROJECT_ID",
+                ],
+            },
+        });
+    }
+    return { project, location };
+}
+export async function analyzeVideo(frames, options = {}) {
+    const provider = options.provider || AIProviderName.AUTO;
+    // Vertex — only when GOOGLE_VERTEX_PROJECT is explicitly set
+    if (provider === AIProviderName.VERTEX || provider === AIProviderName.AUTO) {
+        return analyzeVideoWithVertexAI(frames, options);
+    }
+    // Gemini API — when GOOGLE_AI_API_KEY is set
+    if (provider === AIProviderName.GOOGLE_AI && process.env.GOOGLE_AI_API_KEY) {
+        return analyzeVideoWithGeminiAPI(frames, options);
+    }
+    throw new Error("No valid provider configuration found. " +
+        "Set GOOGLE_VERTEX_PROJECT for Vertex AI or GOOGLE_AI_API_KEY for Gemini API.");
+}
+//# sourceMappingURL=videoAnalyzer.js.map

package/dist/lib/core/baseProvider.js

@@ -7,6 +7,7 @@ import { composeAbortSignals, createTimeoutController, TimeoutError, } from "../
 import { shouldDisableBuiltinTools } from "../utils/toolUtils.js";
 import { getKeyCount, getKeysAsString } from "../utils/transformationUtils.js";
 import { TTSProcessor } from "../utils/ttsProcessor.js";
+import { hasVideoFrames, executeVideoAnalysis, } from "../utils/videoAnalysisProcessor.js";
 import { GenerationHandler } from "./modules/GenerationHandler.js";
 // Import modules for composition
 import { MessageBuilder } from "./modules/MessageBuilder.js";
@@ -473,6 +474,25 @@ export class BaseProvider {
             // ===== Normal AI Generation Flow =====
             const { tools, model } = await this.prepareGenerationContext(options);
             const messages = await this.buildMessages(options);
+            // ===== VIDEO ANALYSIS FROM MESSAGES CONTENT =====
+            // Check if video files are present in messages content array
+            // If video analysis is needed, perform it and return early to avoid running generation
+            if (hasVideoFrames(messages)) {
+                const videoAnalysisResult = await executeVideoAnalysis(messages, {
+                    provider: options.provider,
+                    providerName: this.providerName,
+                    region: options.region,
+                    model: options.model,
+                });
+                // Return video analysis result directly without running generation
+                const videoResult = {
+                    content: videoAnalysisResult,
+                    provider: options.provider ?? this.providerName,
+                    model: this.modelName,
+                    usage: { input: 0, output: 0, total: 0 }, // Video analysis doesn't use standard token counting
+                };
+                return await this.enhanceResult(videoResult, options, startTime);
+            }
             // Compose timeout signal with user-provided abort signal (mirrors stream path)
             const timeoutController = createTimeoutController(options.timeout, this.providerName, "generate");
             const composedSignal = composeAbortSignals(options.abortSignal, timeoutController?.controller.signal);

package/dist/lib/processors/media/VideoProcessor.js

@@ -529,7 +529,7 @@ export class VideoProcessor extends BaseFileProcessor {
         // Extract frames using ffmpeg
         const framesDir = join(tempDir, "frames");
         await fs.mkdir(framesDir, { recursive: true });
-        await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps);
+        await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps, intervalSec);
         // Read extracted frames and resize with sharp
         const keyframes = [];
         for (let i = 0; i < timestamps.length; i++) {
@@ -563,15 +563,11 @@ export class VideoProcessor extends BaseFileProcessor {
      * @param outputDir - Directory to write frame files
      * @param timestamps - Array of timestamps in seconds
      */
-    runFfmpegFrameExtraction(videoPath, outputDir, timestamps) {
+    runFfmpegFrameExtraction(videoPath, outputDir, timestamps, intervalSec) {
         return new Promise((resolve, reject) => {
-            // Build select filter expression: select='eq(n,0)+eq(n,250)+...'
-            // Instead, use fps filter for simpler approach - extract at regular intervals
-            // using -vf fps=1/interval approach which is more reliable
-            // Build timestamp-based filter
-            const selectExpr = timestamps
-                .map((t) => `gte(t\\,${t})*lt(t\\,${t + 0.5})`)
-                .join("+");
+            // Improved select expression to pick exactly one frame per interval
+            // instead of multiple frames within a 0.5s window.
+            const selectExpr = `isnan(prev_selected_t)+gte(t-prev_selected_t,${intervalSec}-0.001)`;
             const timeoutId = setTimeout(() => {
                 reject(new Error(`ffmpeg frame extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
             }, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
@@ -861,19 +857,20 @@ export class VideoProcessor extends BaseFileProcessor {
             }
             const clampedCount = Math.min(frameCount, VIDEO_CONFIG.MAX_FRAMES);
             const timestamps = [];
+            let interval = duration;
             if (clampedCount === 1) {
                 timestamps.push(startSec);
             }
             else {
-                const step = duration / (clampedCount - 1);
+                interval = duration / (clampedCount - 1);
                 for (let i = 0; i < clampedCount; i++) {
-                    timestamps.push(startSec + step * i);
+                    timestamps.push(startSec + interval * i);
                 }
             }
             // Extract frames
             const framesDir = join(tempDir, "frames");
             await fs.mkdir(framesDir, { recursive: true });
-            await this.runFfmpegFrameExtraction(tempVideoPath, framesDir, timestamps);
+            await this.runFfmpegFrameExtraction(tempVideoPath, framesDir, timestamps, interval);
             // Read and resize frames
             const keyframes = [];
             for (let i = 0; i < timestamps.length; i++) {

package/dist/lib/types/generateTypes.d.ts

@@ -521,6 +521,7 @@ export type TextGenerationOptions = {
          */
         images?: Array<Buffer | string | import("./content.js").ImageWithAltText>;
         pdfFiles?: Array<Buffer | string>;
+        files?: Array<Buffer | string | import("./fileTypes.js").FileWithMetadata>;
     };
     provider?: AIProviderName;
     model?: string;

package/dist/lib/utils/videoAnalysisProcessor.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Video Analysis Processor
+ *
+ * Formats video analysis results into human-readable text
+ *
+ * @module utils/videoAnalysisProcessor
+ */
+import type { CoreMessage } from "ai";
+import { AIProviderName } from "../constants/enums.js";
+/**
+ * Check if messages contain video frames (images)
+ *
+ * @param messages - Array of CoreMessage objects
+ * @returns true if video frames are present
+ */
+export declare function hasVideoFrames(messages: CoreMessage[]): boolean;
+/**
+ * Execute video analysis on messages containing video frames
+ *
+ * @param messages - Array of CoreMessage objects with video frames
+ * @param options - Video analysis options
+ * @returns Video analysis text result
+ * @throws Error if analysis fails
+ */
+export declare function executeVideoAnalysis(messages: CoreMessage[], options: {
+    provider?: AIProviderName | string;
+    providerName?: AIProviderName;
+    region?: string;
+    model?: string;
+}): Promise<string>;

package/dist/lib/utils/videoAnalysisProcessor.js

@@ -0,0 +1,59 @@
+/**
+ * Video Analysis Processor
+ *
+ * Formats video analysis results into human-readable text
+ *
+ * @module utils/videoAnalysisProcessor
+ */
+import { AIProviderName } from "../constants/enums.js";
+import { logger } from "./logger.js";
+/**
+ * Check if messages contain video frames (images)
+ *
+ * @param messages - Array of CoreMessage objects
+ * @returns true if video frames are present
+ */
+export function hasVideoFrames(messages) {
+    return messages.some((msg) => {
+        if (Array.isArray(msg.content)) {
+            return msg.content.some((part) => typeof part === "object" &&
+                part !== null &&
+                "type" in part &&
+                part.type === "image");
+        }
+        return false;
+    });
+}
+/**
+ * Execute video analysis on messages containing video frames
+ *
+ * @param messages - Array of CoreMessage objects with video frames
+ * @param options - Video analysis options
+ * @returns Video analysis text result
+ * @throws Error if analysis fails
+ */
+export async function executeVideoAnalysis(messages, options) {
+    logger.debug("[VideoAnalysisProcessor] Video frames detected, triggering analysis");
+    const { analyzeVideo } = await import("../adapters/video/videoAnalyzer.js");
+    const provider = options.provider === AIProviderName.GOOGLE_AI ||
+        (options.provider === AIProviderName.AUTO && process.env.GOOGLE_AI_API_KEY)
+        ? AIProviderName.GOOGLE_AI
+        : options.provider === AIProviderName.VERTEX ||
+            options.providerName === AIProviderName.VERTEX
+            ? AIProviderName.VERTEX
+            : AIProviderName.AUTO;
+    const videoAnalysisText = await analyzeVideo(messages[0], {
+        provider: provider,
+        project: options.region
+            ? undefined
+            : process.env.GOOGLE_VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT,
+        location: options.region || process.env.GOOGLE_VERTEX_LOCATION,
+        model: options.model || "gemini-2.0-flash",
+    });
+    logger.debug("[VideoAnalysisProcessor] Video analysis completed", {
+        hasResult: !!videoAnalysisText,
+        resultLength: videoAnalysisText?.length,
+    });
+    return videoAnalysisText;
+}
+//# sourceMappingURL=videoAnalysisProcessor.js.map

package/dist/processors/media/VideoProcessor.js

@@ -529,7 +529,7 @@ export class VideoProcessor extends BaseFileProcessor {
         // Extract frames using ffmpeg
         const framesDir = join(tempDir, "frames");
         await fs.mkdir(framesDir, { recursive: true });
-        await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps);
+        await this.runFfmpegFrameExtraction(videoPath, framesDir, timestamps, intervalSec);
         // Read extracted frames and resize with sharp
         const keyframes = [];
         for (let i = 0; i < timestamps.length; i++) {
@@ -563,15 +563,11 @@ export class VideoProcessor extends BaseFileProcessor {
      * @param outputDir - Directory to write frame files
      * @param timestamps - Array of timestamps in seconds
      */
-    runFfmpegFrameExtraction(videoPath, outputDir, timestamps) {
+    runFfmpegFrameExtraction(videoPath, outputDir, timestamps, intervalSec) {
         return new Promise((resolve, reject) => {
-            // Build select filter expression: select='eq(n,0)+eq(n,250)+...'
-            // Instead, use fps filter for simpler approach - extract at regular intervals
-            // using -vf fps=1/interval approach which is more reliable
-            // Build timestamp-based filter
-            const selectExpr = timestamps
-                .map((t) => `gte(t\\,${t})*lt(t\\,${t + 0.5})`)
-                .join("+");
+            // Improved select expression to pick exactly one frame per interval
+            // instead of multiple frames within a 0.5s window.
+            const selectExpr = `isnan(prev_selected_t)+gte(t-prev_selected_t,${intervalSec}-0.001)`;
             const timeoutId = setTimeout(() => {
                 reject(new Error(`ffmpeg frame extraction timed out after ${VIDEO_CONFIG.FFMPEG_TIMEOUT_MS}ms`));
             }, VIDEO_CONFIG.FFMPEG_TIMEOUT_MS);
@@ -861,19 +857,20 @@ export class VideoProcessor extends BaseFileProcessor {
             }
             const clampedCount = Math.min(frameCount, VIDEO_CONFIG.MAX_FRAMES);
             const timestamps = [];
+            let interval = duration;
             if (clampedCount === 1) {
                 timestamps.push(startSec);
             }
             else {
-                const step = duration / (clampedCount - 1);
+                interval = duration / (clampedCount - 1);
                 for (let i = 0; i < clampedCount; i++) {
-                    timestamps.push(startSec + step * i);
+                    timestamps.push(startSec + interval * i);
                 }
             }
             // Extract frames
             const framesDir = join(tempDir, "frames");
             await fs.mkdir(framesDir, { recursive: true });
-            await this.runFfmpegFrameExtraction(tempVideoPath, framesDir, timestamps);
+            await this.runFfmpegFrameExtraction(tempVideoPath, framesDir, timestamps, interval);
             // Read and resize frames
             const keyframes = [];
             for (let i = 0; i < timestamps.length; i++) {

package/dist/types/generateTypes.d.ts

@@ -521,6 +521,7 @@ export type TextGenerationOptions = {
          */
         images?: Array<Buffer | string | import("./content.js").ImageWithAltText>;
         pdfFiles?: Array<Buffer | string>;
+        files?: Array<Buffer | string | import("./fileTypes.js").FileWithMetadata>;
     };
     provider?: AIProviderName;
     model?: string;

package/dist/utils/videoAnalysisProcessor.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Video Analysis Processor
+ *
+ * Formats video analysis results into human-readable text
+ *
+ * @module utils/videoAnalysisProcessor
+ */
+import type { CoreMessage } from "ai";
+import { AIProviderName } from "../constants/enums.js";
+/**
+ * Check if messages contain video frames (images)
+ *
+ * @param messages - Array of CoreMessage objects
+ * @returns true if video frames are present
+ */
+export declare function hasVideoFrames(messages: CoreMessage[]): boolean;
+/**
+ * Execute video analysis on messages containing video frames
+ *
+ * @param messages - Array of CoreMessage objects with video frames
+ * @param options - Video analysis options
+ * @returns Video analysis text result
+ * @throws Error if analysis fails
+ */
+export declare function executeVideoAnalysis(messages: CoreMessage[], options: {
+    provider?: AIProviderName | string;
+    providerName?: AIProviderName;
+    region?: string;
+    model?: string;
+}): Promise<string>;

package/dist/utils/videoAnalysisProcessor.js

@@ -0,0 +1,58 @@
+/**
+ * Video Analysis Processor
+ *
+ * Formats video analysis results into human-readable text
+ *
+ * @module utils/videoAnalysisProcessor
+ */
+import { AIProviderName } from "../constants/enums.js";
+import { logger } from "./logger.js";
+/**
+ * Check if messages contain video frames (images)
+ *
+ * @param messages - Array of CoreMessage objects
+ * @returns true if video frames are present
+ */
+export function hasVideoFrames(messages) {
+    return messages.some((msg) => {
+        if (Array.isArray(msg.content)) {
+            return msg.content.some((part) => typeof part === "object" &&
+                part !== null &&
+                "type" in part &&
+                part.type === "image");
+        }
+        return false;
+    });
+}
+/**
+ * Execute video analysis on messages containing video frames
+ *
+ * @param messages - Array of CoreMessage objects with video frames
+ * @param options - Video analysis options
+ * @returns Video analysis text result
+ * @throws Error if analysis fails
+ */
+export async function executeVideoAnalysis(messages, options) {
+    logger.debug("[VideoAnalysisProcessor] Video frames detected, triggering analysis");
+    const { analyzeVideo } = await import("../adapters/video/videoAnalyzer.js");
+    const provider = options.provider === AIProviderName.GOOGLE_AI ||
+        (options.provider === AIProviderName.AUTO && process.env.GOOGLE_AI_API_KEY)
+        ? AIProviderName.GOOGLE_AI
+        : options.provider === AIProviderName.VERTEX ||
+            options.providerName === AIProviderName.VERTEX
+            ? AIProviderName.VERTEX
+            : AIProviderName.AUTO;
+    const videoAnalysisText = await analyzeVideo(messages[0], {
+        provider: provider,
+        project: options.region
+            ? undefined
+            : process.env.GOOGLE_VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT,
+        location: options.region || process.env.GOOGLE_VERTEX_LOCATION,
+        model: options.model || "gemini-2.0-flash",
+    });
+    logger.debug("[VideoAnalysisProcessor] Video analysis completed", {
+        hasResult: !!videoAnalysisText,
+        resultLength: videoAnalysisText?.length,
+    });
+    return videoAnalysisText;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "9.7.0",
+  "version": "9.9.0",
   "description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 13 providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
   "author": {
     "name": "Juspay Technologies",
@@ -37,28 +37,28 @@
     "prepack": "svelte-kit sync && svelte-package && pnpm run build:cli && publint",
     "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json && tsc --noEmit --strict",
     "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
-    "modelServer": "node scripts/modelServer.js",
+    "modelServer": "tsx scripts/modelServer.ts",
     "lint": "prettier --check . && eslint .",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "format:staged": "node scripts/format-staged.cjs",
+    "format:staged": "tsx scripts/format-staged.ts",
     "changeset": "changeset",
     "changeset:version": "changeset version && git add --all",
-    "format:changelog": "node scripts/format-changelog.cjs",
+    "format:changelog": "tsx scripts/format-changelog.ts",
     "// ===== NEUROLINK DEVELOPER EXPERIENCE ENHANCEMENT 2.0 =====": "",
     "// Environment & Setup (pnpm-first)": "",
-    "setup": "pnpm install && node tools/automation/environmentManager.js",
+    "setup": "pnpm install && tsx tools/setup.ts",
     "setup:complete": "pnpm run setup && pnpm run project:organize && pnpm run env:validate",
-    "env:setup": "node tools/automation/environmentManager.js",
-    "env:validate": "node tools/automation/environmentManager.js --validate",
-    "env:backup": "node tools/automation/environmentManager.js --backup",
-    "env:list-backups": "node tools/automation/environmentManager.js --list-backups",
+    "env:setup": "tsx tools/automation/environmentManager.ts",
+    "env:validate": "tsx tools/automation/environmentManager.ts --validate",
+    "env:backup": "tsx tools/automation/environmentManager.ts --backup",
+    "env:list-backups": "tsx tools/automation/environmentManager.ts --list-backups",
     "// Project Management & Analysis": "",
-    "project:organize": "node tools/automation/projectOrganizer.js",
-    "project:health": "node tools/development/healthMonitor.js",
+    "project:organize": "tsx tools/automation/projectOrganizer.ts",
+    "project:health": "tsx tools/development/healthMonitor.ts",
     "// Shell Script Conversion": "",
-    "convert:shell-scripts": "node tools/automation/shellConverter.js",
-    "convert:specific": "node tools/automation/shellConverter.js --specific",
+    "convert:shell-scripts": "tsx tools/automation/shellConverter.ts",
+    "convert:specific": "tsx tools/automation/shellConverter.ts --specific",
     "// Testing (Enhanced Vitest Framework)": "",
     "test": "vitest run",
     "test:watch": "vitest",
@@ -71,13 +71,13 @@
     "test:e2e": "vitest run test/e2e",
     "test:ci": "vitest run --coverage --reporter=junit --reporter=verbose",
     "test:debug": "vitest --inspect-brk",
-    "test:performance": "node tools/testing/performanceMonitor.js",
+    "test:performance": "tsx tools/testing/performanceMonitor.ts",
     "// Legacy Testing Support (during transition)": "",
     "test:legacy": "npx tsx test/continuous-test-suite.ts",
     "test:comparison": "pnpm run test && pnpm run test:legacy",
     "// Content Generation (Cross-platform JS)": "",
-    "content:videos": "node tools/converted-scripts/generateAllVideos.js",
-    "content:cleanup": "node tools/converted-scripts/cleanupHashNamedVideos.js",
+    "content:videos": "tsx tools/converted-scripts/generateAllVideos.ts",
+    "content:cleanup": "tsx tools/converted-scripts/cleanupHashNamedVideos.ts",
     "content:all": "pnpm run content:videos",
     "// Documentation Automation (Legacy MkDocs)": "",
     "docs:api": "typedoc",
@@ -85,7 +85,7 @@
     "docs:build:mkdocs": "pnpm run docs:api && bash scripts/sync-readme.sh && mkdocs build --strict --clean",
     "docs:serve:mkdocs": "bash scripts/sync-readme.sh && mkdocs serve",
     "docs:gh-deploy:mkdocs": "bash scripts/sync-readme.sh && mkdocs gh-deploy --force",
-    "docs:validate": "node tools/content/documentationSync.js --validate",
+    "docs:validate": "tsx tools/content/documentationSync.ts --validate",
     "docs:generate": "pnpm run docs:validate",
     "// Documentation (Docusaurus)": "",
     "docs:start": "pnpm --filter ./docs-site start",
@@ -93,11 +93,11 @@
     "docs:serve": "pnpm --filter ./docs-site serve",
     "docs:clear": "pnpm --filter ./docs-site clear",
     "// Development & Monitoring": "",
-    "dev:health": "node tools/development/healthMonitor.js",
+    "dev:health": "tsx tools/development/healthMonitor.ts",
     "dev:demo": "concurrently \"pnpm run dev\" \"node neurolink-demo/complete-enhanced-server.js\"",
-    "demo:voice": "pnpm build && node examples/voice-demo/server.mjs",
+    "demo:voice": "pnpm build && tsx examples/voice-demo/server.ts",
     "// Build & Deploy (Complete Pipeline)": "",
-    "build:complete": "node tools/automation/buildSystem.js",
+    "build:complete": "tsx tools/automation/buildSystem.ts",
     "// Quality & Maintenance": "",
     "quality:all": "pnpm run lint && pnpm run format && pnpm run test:ci",
     "clean": "pnpm run content:cleanup && rm -rf dist .svelte-kit node_modules/.cache",
@@ -108,12 +108,12 @@
     "test:semantic-release": "node scripts/test-semantic-release.js",
     "release:dry-run": "npx semantic-release --dry-run",
     "// Build Rule Enforcement Scripts": "",
-    "validate": "node scripts/build-validations.cjs",
-    "validate:env": "node scripts/env-validation.cjs",
-    "validate:security": "node scripts/security-check.cjs",
+    "validate": "tsx scripts/build-validations.ts",
+    "validate:env": "tsx scripts/env-validation.ts",
+    "validate:security": "tsx scripts/security-check.ts",
     "validate:all": "pnpm run validate && pnpm run validate:env && pnpm run validate:security",
-    "validate:commit": "node scripts/commit-validation.cjs",
-    "quality:metrics": "node scripts/quality-metrics.cjs",
+    "validate:commit": "tsx scripts/commit-validation.ts",
+    "quality:metrics": "tsx scripts/quality-metrics.ts",
     "quality:report": "pnpm run quality:metrics && echo 'Quality metrics saved to quality-metrics.json'",
     "pre-commit": "lint-staged",
     "pre-push": "pnpm run validate:commit && pnpm run validate:env && pnpm run validate && pnpm run test",
@@ -299,6 +299,7 @@
     "svelte": "^5.0.0",
     "svelte-check": "^4.0.0",
     "tslib": "^2.4.1",
+    "tsx": "^4.21.0",
     "typedoc": "^0.28.15",
     "typedoc-plugin-markdown": "^4.9.0",
     "typescript": "^5.0.0",