@juspay/neurolink 9.5.3 → 9.7.0

package/dist/processors/media/VideoProcessor.d.ts

@@ -0,0 +1,350 @@
+/**
+ * Video Processor
+ *
+ * Handles downloading, validating, and processing video files for AI consumption.
+ * Since LLMs cannot process raw video, this processor extracts:
+ * - Structured metadata (duration, resolution, codecs, etc.)
+ * - Keyframes at configurable intervals (resized to 768px JPEG)
+ * - Embedded subtitle tracks (if present)
+ *
+ * The extracted content is formatted as text + images that can be sent to any
+ * AI provider for analysis.
+ *
+ * Uses fluent-ffmpeg for video processing and sharp for frame resizing.
+ * Requires ffmpeg/ffprobe to be available (via ffmpeg-static or system PATH).
+ *
+ * Key features:
+ * - Adaptive keyframe extraction intervals based on video duration
+ * - Frame count capping (max 20 frames) to control token usage
+ * - JPEG quality optimization for AI vision models
+ * - Embedded subtitle extraction (SRT format)
+ * - Graceful degradation on corrupt files or missing codecs
+ * - Temp file cleanup with finally blocks
+ * - Configurable timeouts for ffmpeg and ffprobe operations
+ *
+ * @module processors/media/VideoProcessor
+ *
+ * @example
+ * ```typescript
+ * import { videoProcessor, processVideo, isVideoFile } from "./VideoProcessor.js";
+ *
+ * // Check if a file is a video file
+ * if (isVideoFile(fileInfo.mimetype, fileInfo.name)) {
+ *   const result = await processVideo(fileInfo, {
+ *     authHeaders: { Authorization: "Bearer token" },
+ *   });
+ *
+ *   if (result.success) {
+ *     console.log(`Duration: ${result.data.metadata.durationFormatted}`);
+ *     console.log(`Keyframes: ${result.data.frameCount}`);
+ *     console.log(`Text for LLM:\n${result.data.textContent}`);
+ *   }
+ * }
+ * ```
+ */
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+import type { FileInfo, FileProcessingResult, ProcessedFileBase, ProcessOptions } from "../base/types.js";
+/**
+ * Processed video result.
+ * Extends ProcessedFileBase with video-specific fields including metadata,
+ * extracted keyframes, subtitle text, and a pre-formatted textContent block
+ * suitable for sending to an LLM.
+ */
+export type ProcessedVideo = ProcessedFileBase & {
+    /** Pre-formatted text description for the LLM (metadata + subtitles summary) */
+    textContent: string;
+    /** Extracted keyframes as JPEG buffers (resized to max 768px dimension) */
+    keyframes: Buffer[];
+    /** Video metadata extracted via ffprobe */
+    metadata: {
+        /** Duration in seconds */
+        duration: number;
+        /** Human-readable duration (e.g., "2m 30s") */
+        durationFormatted: string;
+        /** Video width in pixels */
+        width: number;
+        /** Video height in pixels */
+        height: number;
+        /** Video codec name (e.g., "h264", "vp9") */
+        codec: string;
+        /** Frames per second */
+        fps: number;
+        /** Video bitrate in bits/second */
+        bitrate: number;
+        /** Audio codec name if present */
+        audioCodec?: string;
+        /** Number of audio channels */
+        audioChannels?: number;
+        /** Audio sample rate in Hz */
+        audioSampleRate?: number;
+        /** Number of subtitle tracks found */
+        subtitleTracks: number;
+        /** Original file size in bytes */
+        fileSize: number;
+    };
+    /** Extracted subtitle text (combined from all subtitle tracks) */
+    subtitleText?: string;
+    /** Whether any keyframes were successfully extracted */
+    hasKeyframes: boolean;
+    /** Number of keyframes extracted */
+    frameCount: number;
+};
+/**
+ * Video Processor - extracts metadata, keyframes, and subtitles from video files.
+ *
+ * Since LLMs cannot process raw video, this processor converts videos into
+ * a structured representation consisting of:
+ * 1. Text metadata block (duration, resolution, codecs, etc.)
+ * 2. Keyframe images (JPEG, resized to 768px max dimension)
+ * 3. Subtitle text (if embedded in the video)
+ *
+ * The processor uses a temp file approach because ffmpeg requires file paths
+ * for most operations. Temp files are always cleaned up in finally blocks.
+ *
+ * @example
+ * ```typescript
+ * const processor = new VideoProcessor();
+ * const result = await processor.processFile({
+ *   id: "video-1",
+ *   name: "presentation.mp4",
+ *   mimetype: "video/mp4",
+ *   size: 15_000_000,
+ *   buffer: videoBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   // result.data.textContent - text description for LLM
+ *   // result.data.keyframes   - array of JPEG buffers
+ *   // result.data.subtitleText - extracted subtitles (if any)
+ * }
+ * ```
+ */
+export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
+    constructor();
+    /**
+     * Build processed result stub.
+     * This is a synchronous placeholder - actual processing happens in the
+     * overridden processFile method since ffmpeg operations are asynchronous
+     * and require temp file I/O.
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Empty ProcessedVideo structure
+     */
+    protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedVideo;
+    /**
+     * Override processFile for async video processing with ffmpeg.
+     *
+     * Processing pipeline:
+     * 1. Validate file type and size
+     * 2. Get buffer (from fileInfo.buffer or download from URL)
+     * 3. Write buffer to temp file (ffmpeg requires file paths)
+     * 4. Extract metadata using ffprobe
+     * 5. Extract keyframes at calculated intervals, resize with sharp
+     * 6. Extract subtitle tracks if embedded
+     * 7. Build textContent summary for LLM
+     * 8. Clean up temp files
+     *
+     * @param fileInfo - File information with URL or buffer
+     * @param options - Optional processing options
+     * @returns Processing result with extracted video data or error
+     */
+    processFile(fileInfo: FileInfo, options?: ProcessOptions): Promise<FileProcessingResult<ProcessedVideo>>;
+    /**
+     * Probe a video file to extract metadata using ffprobe.
+     *
+     * @param filePath - Path to the video file
+     * @returns Success result with probe data or error message
+     */
+    private probeVideo;
+    /**
+     * Build a structured metadata object from ffprobe data.
+     *
+     * @param probeData - Raw ffprobe output
+     * @param fileSize - Original file size in bytes
+     * @returns Structured video metadata
+     */
+    private buildMetadata;
+    /**
+     * Extract keyframes from a video at calculated intervals.
+     *
+     * The interval between frames is determined by the video duration:
+     * - <= 10s:   every 1s  (very short clips — dense coverage)
+     * - <= 30s:   every 2s  (short bug clips)
+     * - <= 120s:  every 5s  (standard screen recordings)
+     * - <= 600s:  every 15s (longer demos)
+     * - <= 1800s: every 60s (meeting recordings)
+     * - > 1800s:  every 180s (full meetings)
+     *
+     * Results are capped at MAX_FRAMES (100) and each frame is resized
+     * to fit within 768x768px while maintaining aspect ratio.
+     * The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
+     * the interval widens to duration/MAX_FRAMES for full-video coverage.
+     *
+     * @param videoPath - Path to the video file
+     * @param tempDir - Temp directory for frame output
+     * @param durationSec - Video duration in seconds
+     * @returns Array of JPEG frame buffers
+     */
+    private extractKeyframes;
+    /**
+     * Run ffmpeg to extract frames at specified timestamps.
+     *
+     * Uses the `-vf select` filter to pick frames at exact timestamps,
+     * which is more efficient than seeking for each frame individually.
+     *
+     * @param videoPath - Path to the video file
+     * @param outputDir - Directory to write frame files
+     * @param timestamps - Array of timestamps in seconds
+     */
+    private runFfmpegFrameExtraction;
+    /**
+     * Determine the frame extraction interval based on video duration.
+     *
+     * @param durationSec - Video duration in seconds
+     * @returns Interval in seconds between extracted frames
+     */
+    private getFrameInterval;
+    /**
+     * Extract embedded subtitle text from the first subtitle track.
+     *
+     * Uses ffmpeg to convert the first subtitle stream to SRT format,
+     * then strips SRT formatting (timestamps, sequence numbers) to produce
+     * plain text.
+     *
+     * @param videoPath - Path to the video file
+     * @param tempDir - Temp directory for subtitle output
+     * @returns Extracted subtitle text, or undefined if extraction fails
+     */
+    private extractSubtitles;
+    /**
+     * Parse SRT subtitle content into plain text.
+     * Strips sequence numbers, timestamps, and blank lines.
+     *
+     * @param srt - Raw SRT content
+     * @returns Plain text from subtitles
+     */
+    private parseSrtToPlainText;
+    /**
+     * Build a structured text description of the video for LLM consumption.
+     *
+     * The output includes:
+     * - File name and basic info
+     * - Technical metadata (resolution, codec, duration, etc.)
+     * - Frame extraction summary
+     * - Subtitle text (if available)
+     *
+     * @param metadata - Extracted video metadata
+     * @param frameCount - Number of keyframes extracted
+     * @param subtitleText - Extracted subtitle text (if any)
+     * @param filename - Original filename
+     * @returns Formatted text content for the LLM
+     */
+    private buildTextContent;
+    /**
+     * Format a duration in seconds to a human-readable string.
+     *
+     * @param seconds - Duration in seconds
+     * @returns Formatted string (e.g., "1h 23m 45s")
+     */
+    private formatDuration;
+    /**
+     * Get a file extension from FileInfo, falling back to ".mp4".
+     *
+     * @param fileInfo - File information
+     * @returns File extension with leading dot
+     */
+    private getExtensionFromFileInfo;
+    /**
+     * Write a buffer to a file using streaming to handle large files efficiently.
+     *
+     * @param buffer - Buffer to write
+     * @param filePath - Destination file path
+     */
+    private writeBufferToFile;
+    /**
+     * Extract frames from a specific time range in a video.
+     *
+     * This is the on-demand extraction method called by the `extract_file_content`
+     * tool. Unlike initial keyframe extraction (which covers the full video),
+     * this targets a specific time window with configurable frame count.
+     *
+     * @param buffer - Video file buffer
+     * @param filename - Original filename (for extension detection)
+     * @param startSec - Start time in seconds
+     * @param endSec - End time in seconds
+     * @param frameCount - Number of frames to extract in the range (default: 5)
+     * @returns Array of JPEG frame buffers
+     */
+    extractFrameRange(buffer: Buffer, filename: string, startSec: number, endSec: number, frameCount?: number): Promise<Buffer[]>;
+    /**
+     * Guess file extension from filename, with fallback to .mp4.
+     */
+    private guessExtensionFromName;
+}
+/**
+ * Singleton Video processor instance.
+ * Use this for standard video processing operations.
+ *
+ * @example
+ * ```typescript
+ * import { videoProcessor } from "./VideoProcessor.js";
+ *
+ * const result = await videoProcessor.processFile(fileInfo);
+ * ```
+ */
+export declare const videoProcessor: VideoProcessor;
+/**
+ * Check if a file is a video file.
+ * Matches by MIME type or file extension.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is a supported video file
+ *
+ * @example
+ * ```typescript
+ * if (isVideoFile("video/mp4", "recording.mp4")) {
+ *   const result = await processVideo(fileInfo);
+ * }
+ *
+ * if (isVideoFile("", "clip.mkv")) {
+ *   // Also matches by extension
+ * }
+ * ```
+ */
+export declare function isVideoFile(mimetype: string, filename: string): boolean;
+/**
+ * Process a single video file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with extracted video data or error
+ *
+ * @example
+ * ```typescript
+ * import { processVideo } from "./VideoProcessor.js";
+ *
+ * const result = await processVideo({
+ *   id: "vid-123",
+ *   name: "demo.mp4",
+ *   mimetype: "video/mp4",
+ *   size: 15_000_000,
+ *   buffer: videoBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log(`Duration: ${result.data.metadata.durationFormatted}`);
+ *   console.log(`Extracted ${result.data.frameCount} keyframes`);
+ *   console.log(`Text content:\n${result.data.textContent}`);
+ *
+ *   if (result.data.subtitleText) {
+ *     console.log(`Subtitles:\n${result.data.subtitleText}`);
+ *   }
+ * } else {
+ *   console.error(`Processing failed: ${result.error?.userMessage}`);
+ * }
+ * ```
+ */
+export declare function processVideo(fileInfo: FileInfo, options?: ProcessOptions): Promise<FileProcessingResult<ProcessedVideo>>;