mcp-video-analyzer 0.2.0

package/dist/tools/analyze-video.js

@@ -0,0 +1,320 @@
+import { imageContent, UserError } from 'fastmcp';
+import { z } from 'zod';
+import { getAdapter } from '../adapters/adapter.interface.js';
+import { extractSceneFrames, extractDenseFrames, probeVideoDuration, formatTimestamp, } from '../processors/frame-extractor.js';
+import { extractBrowserFrames, generateTimestamps } from '../processors/browser-frame-extractor.js';
+import { deduplicateFrames } from '../processors/frame-dedup.js';
+import { extractTextFromFrames } from '../processors/frame-ocr.js';
+import { buildAnnotatedTimeline } from '../processors/annotated-timeline.js';
+import { optimizeFrames } from '../processors/image-optimizer.js';
+import { extractAudioTrack, transcribeAudio } from '../processors/audio-transcriber.js';
+import { createTempDir, cleanupTempDir } from '../utils/temp-files.js';
+import { AnalysisCache, cacheKey } from '../utils/cache.js';
+import { getDetailConfig } from '../config/detail-levels.js';
+import { filterAnalysisResult } from '../utils/field-filter.js';
+const cache = new AnalysisCache();
+const ANALYSIS_FIELDS = [
+    'metadata',
+    'transcript',
+    'frames',
+    'comments',
+    'chapters',
+    'ocrResults',
+    'timeline',
+    'aiSummary',
+];
+const AnalyzeOptionsSchema = z
+    .object({
+    maxFrames: z
+        .number()
+        .min(1)
+        .max(60)
+        .optional()
+        .describe('Maximum number of key frames to extract (default depends on detail level)'),
+    threshold: z
+        .number()
+        .min(0)
+        .max(1)
+        .default(0.1)
+        .optional()
+        .describe('Scene-change sensitivity 0.0-1.0 (lower = more frames, default: 0.1). Use 0.1 for screencasts/demos, 0.3 for live-action video.'),
+    returnBase64: z
+        .boolean()
+        .default(false)
+        .optional()
+        .describe('Return frames as base64 inline instead of file paths'),
+    skipFrames: z
+        .boolean()
+        .default(false)
+        .optional()
+        .describe('Skip frame extraction (transcript + metadata only)'),
+    detail: z
+        .enum(['brief', 'standard', 'detailed'])
+        .default('standard')
+        .optional()
+        .describe('Analysis depth: "brief" (metadata + truncated transcript, no frames), "standard" (default), "detailed" (dense sampling, more frames)'),
+    fields: z
+        .array(z.enum(ANALYSIS_FIELDS))
+        .optional()
+        .describe('Specific fields to return (default: all). E.g., ["metadata", "transcript"] returns only those fields.'),
+    forceRefresh: z
+        .boolean()
+        .default(false)
+        .optional()
+        .describe('Bypass cache and re-analyze the video'),
+})
+    .optional();
+const AnalyzeVideoSchema = z.object({
+    url: z.string().url().describe('Video URL (Loom share link or direct mp4/webm URL)'),
+    options: AnalyzeOptionsSchema.describe('Analysis options'),
+});
+export function registerAnalyzeVideo(server) {
+    server.addTool({
+        name: 'analyze_video',
+        description: `Analyze a video URL to extract transcript, key frames, metadata, comments, OCR text, and annotated timeline.
+
+Returns structured data about the video content:
+- Transcript with timestamps and speakers
+- Key frames extracted via scene-change detection (deduplicated, as images)
+- OCR text extracted from frames (code, error messages, UI text visible on screen)
+- Annotated timeline merging transcript + frames + OCR into a unified chronological view
+- Metadata (title, duration, platform)
+- Comments from viewers (if available)
+
+Supports: Loom (loom.com/share/...) and direct video URLs (.mp4, .webm, .mov).
+
+Detail levels:
+- "brief": metadata + truncated transcript only (fast, no video download)
+- "standard": full analysis with scene-change frames (default)
+- "detailed": dense sampling (1 frame/sec), more frames, full OCR
+
+Use options.fields to request only specific data (e.g., ["metadata", "transcript"]).
+Use options.forceRefresh to bypass the cache.`,
+        parameters: AnalyzeVideoSchema,
+        annotations: {
+            title: 'Analyze Video',
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
+        execute: async (args, { reportProgress }) => {
+            const { url, options } = args;
+            const detail = options?.detail ?? 'standard';
+            const forceRefresh = options?.forceRefresh ?? false;
+            const fields = options?.fields;
+            const threshold = options?.threshold ?? 0.1;
+            // Resolve detail config
+            const config = getDetailConfig(detail);
+            const maxFrames = options?.maxFrames ?? config.maxFrames;
+            const skipFrames = options?.skipFrames ?? !config.includeFrames;
+            // Cache check
+            const key = cacheKey(url, { detail, maxFrames, threshold });
+            if (!forceRefresh) {
+                const cached = cache.get(key);
+                if (cached) {
+                    const filtered = filterAnalysisResult(cached, fields);
+                    const textData = { ...filtered, frameCount: cached.frames.length };
+                    const content = [{ type: 'text', text: JSON.stringify(textData, null, 2) }];
+                    // Re-add frame images if included
+                    if (!fields || fields.includes('frames')) {
+                        for (const frame of cached.frames) {
+                            try {
+                                content.push(await imageContent({ path: frame.filePath }));
+                            }
+                            catch {
+                                // Frame file may have been cleaned up
+                            }
+                        }
+                    }
+                    return { content };
+                }
+            }
+            let adapter;
+            try {
+                adapter = getAdapter(url);
+            }
+            catch (error) {
+                if (error instanceof UserError)
+                    throw error;
+                throw new UserError(`Failed to detect video platform for URL: ${url}`);
+            }
+            const warnings = [];
+            let tempDir = null;
+            try {
+                await reportProgress({ progress: 0, total: 100 });
+                // Fetch metadata, transcript, comments in parallel
+                const [metadata, transcript, comments, chapters, aiSummary] = await Promise.all([
+                    adapter.getMetadata(url).catch((e) => {
+                        warnings.push(`Failed to fetch metadata: ${e instanceof Error ? e.message : String(e)}`);
+                        return {
+                            platform: adapter.name,
+                            title: 'Unknown',
+                            duration: 0,
+                            durationFormatted: '0:00',
+                            url,
+                        };
+                    }),
+                    adapter.getTranscript(url).catch((e) => {
+                        warnings.push(`Failed to fetch transcript: ${e instanceof Error ? e.message : String(e)}`);
+                        return [];
+                    }),
+                    adapter.getComments(url).catch((e) => {
+                        warnings.push(`Failed to fetch comments: ${e instanceof Error ? e.message : String(e)}`);
+                        return [];
+                    }),
+                    adapter.getChapters(url).catch(() => []),
+                    adapter.getAiSummary(url).catch(() => null),
+                ]);
+                await reportProgress({ progress: 40, total: 100 });
+                // Apply transcript limit for brief mode
+                const limitedTranscript = config.transcriptMaxEntries !== null
+                    ? transcript.slice(0, config.transcriptMaxEntries)
+                    : transcript;
+                // Frame extraction (if not skipped)
+                const result = {
+                    metadata,
+                    transcript: limitedTranscript,
+                    frames: [],
+                    comments,
+                    chapters,
+                    ocrResults: [],
+                    timeline: [],
+                    aiSummary: aiSummary ?? undefined,
+                    warnings,
+                };
+                let videoPath = null;
+                if (!skipFrames) {
+                    tempDir = await createTempDir();
+                    let framesExtracted = false;
+                    // Strategy 1: yt-dlp download + ffmpeg frame extraction
+                    if (adapter.capabilities.videoDownload) {
+                        videoPath = await adapter.downloadVideo(url, tempDir);
+                        if (videoPath) {
+                            await reportProgress({ progress: 60, total: 100 });
+                            // Probe duration if metadata didn't provide it
+                            if (metadata.duration === 0) {
+                                const duration = await probeVideoDuration(videoPath).catch(() => 0);
+                                metadata.duration = duration;
+                                metadata.durationFormatted = formatTimestamp(Math.floor(duration));
+                            }
+                            // Extract frames: dense or scene-based
+                            const rawFrames = config.denseSampling
+                                ? await extractDenseFrames(videoPath, tempDir, { maxFrames }).catch((e) => {
+                                    warnings.push(`Dense frame extraction failed: ${e instanceof Error ? e.message : String(e)}`);
+                                    return [];
+                                })
+                                : await extractSceneFrames(videoPath, tempDir, {
+                                    threshold,
+                                    maxFrames,
+                                }).catch((e) => {
+                                    warnings.push(`Frame extraction failed: ${e instanceof Error ? e.message : String(e)}`);
+                                    return [];
+                                });
+                            await reportProgress({ progress: 80, total: 100 });
+                            if (rawFrames.length > 0) {
+                                const optimizedPaths = await optimizeFrames(rawFrames.map((f) => f.filePath), tempDir).catch((e) => {
+                                    warnings.push(`Frame optimization failed: ${e instanceof Error ? e.message : String(e)}`);
+                                    return rawFrames.map((f) => f.filePath);
+                                });
+                                result.frames = rawFrames.map((frame, i) => ({
+                                    ...frame,
+                                    filePath: optimizedPaths[i] ?? frame.filePath,
+                                }));
+                                framesExtracted = true;
+                            }
+                        }
+                    }
+                    // Strategy 2: Browser-based extraction (fallback)
+                    if (!framesExtracted && metadata.duration > 0) {
+                        await reportProgress({ progress: 60, total: 100 });
+                        const timestamps = generateTimestamps(metadata.duration, maxFrames);
+                        const browserFrames = await extractBrowserFrames(url, tempDir, {
+                            timestamps,
+                        }).catch((e) => {
+                            warnings.push(`Browser frame extraction failed: ${e instanceof Error ? e.message : String(e)}`);
+                            return [];
+                        });
+                        await reportProgress({ progress: 80, total: 100 });
+                        if (browserFrames.length > 0) {
+                            result.frames = browserFrames;
+                            framesExtracted = true;
+                        }
+                    }
+                    if (!framesExtracted) {
+                        warnings.push('Frame extraction not available — returning transcript and metadata only. Install yt-dlp or Chrome/Chromium for frame extraction.');
+                    }
+                    // Post-processing: dedup, OCR, timeline
+                    if (result.frames.length > 0) {
+                        const beforeDedup = result.frames.length;
+                        result.frames = await deduplicateFrames(result.frames).catch((e) => {
+                            warnings.push(`Frame dedup failed: ${e instanceof Error ? e.message : String(e)}`);
+                            return result.frames;
+                        });
+                        if (result.frames.length < beforeDedup) {
+                            warnings.push(`Removed ${beforeDedup - result.frames.length} near-duplicate frames (${beforeDedup} → ${result.frames.length})`);
+                        }
+                        await reportProgress({ progress: 85, total: 100 });
+                        // OCR: extract text visible on screen
+                        if (config.includeOcr) {
+                            result.ocrResults = await extractTextFromFrames(result.frames).catch((e) => {
+                                warnings.push(`OCR failed: ${e instanceof Error ? e.message : String(e)}`);
+                                return [];
+                            });
+                        }
+                        await reportProgress({ progress: 95, total: 100 });
+                    }
+                    // Build annotated timeline
+                    if (config.includeTimeline) {
+                        result.timeline = buildAnnotatedTimeline(result.transcript, result.frames, result.ocrResults);
+                    }
+                }
+                else {
+                    // Even without frames, try to get the video for whisper fallback
+                    if (result.transcript.length === 0 && adapter.capabilities.videoDownload) {
+                        tempDir = tempDir ?? (await createTempDir());
+                        videoPath = await adapter.downloadVideo(url, tempDir).catch(() => null);
+                    }
+                }
+                // Whisper fallback: if no transcript and we have a video file
+                if (result.transcript.length === 0 && videoPath) {
+                    try {
+                        const audioPath = await extractAudioTrack(videoPath, tempDir ?? '');
+                        const whisperTranscript = await transcribeAudio(audioPath);
+                        if (whisperTranscript.length > 0) {
+                            result.transcript = whisperTranscript;
+                            warnings.push('Transcript generated via Whisper fallback (no native transcript available).');
+                        }
+                    }
+                    catch {
+                        // Audio extraction or transcription failed — not critical
+                    }
+                }
+                await reportProgress({ progress: 100, total: 100 });
+                // Cache the full result
+                cache.set(key, result);
+                // Apply field filter
+                const filtered = filterAnalysisResult(result, fields);
+                // Build response content
+                const textData = {
+                    ...filtered,
+                    frameCount: result.frames.length,
+                };
+                const content = [{ type: 'text', text: JSON.stringify(textData, null, 2) }];
+                // Add frame images (if not filtered out)
+                if (!fields || fields.includes('frames')) {
+                    for (const frame of result.frames) {
+                        content.push(await imageContent({ path: frame.filePath }));
+                    }
+                }
+                return { content };
+            }
+            finally {
+                if (tempDir && skipFrames) {
+                    await cleanupTempDir(tempDir).catch(() => undefined);
+                }
+            }
+        },
+    });
+}
+//# sourceMappingURL=analyze-video.js.map

package/dist/tools/analyze-video.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze-video.js","sourceRoot":"","sources":["../../src/tools/analyze-video.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAC9D,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,eAAe,GAChB,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,0CAA0C,CAAC;AACpG,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,qCAAqC,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AACxF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAIhE,MAAM,KAAK,GAAG,IAAI,aAAa,EAAE,CAAC;AAElC,MAAM,eAAe,GAAG;IACtB,UAAU;IACV,YAAY;IACZ,QAAQ;IACR,UAAU;IACV,UAAU;IACV,YAAY;IACZ,UAAU;IACV,WAAW;CACH,CAAC;AAEX,MAAM,oBAAoB,GAAG,CAAC;KAC3B,MAAM,CAAC;IACN,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CAAC,2EAA2E,CAAC;IACxF,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,CAAC,CAAC;SACN,OAAO,CAAC,GAAG,CAAC;SACZ,QAAQ,EAAE;SACV,QAAQ,CACP,iIAAiI,CAClI;IACH,YAAY,EAAE,CAAC;SACZ,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,EAAE;SACV,QAAQ,CAAC,sDAAsD,CAAC;IACnE,UAAU,EAAE,CAAC;SACV,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,EAAE;SACV,QAAQ,CAAC,oDAAoD,CAAC;IACjE,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;SACvC,OAAO,CAAC,UAAU,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CACP,sIAAsI,CACvI;IACH,MAAM,EAAE,CAAC;SACN,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;SAC9B,QAAQ,EAAE;SACV,QAAQ,CACP,uGAAuG,CACxG;IACH,YAAY,EAAE,CAAC;SACZ,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,EAAE;SACV,QAAQ,CAAC,uCAAuC,CAAC;CACrD,CAAC;KACD,QAAQ,EAAE,CAAC;AAEd,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IACpF,OAAO,EAAE,oBAAoB,CAAC,QAAQ,CAAC,kBAAkB,CAAC;CAC3D,CAAC,CAAC;AAEH,MAAM,UAAU,oBAAoB,CAAC,MAAe;IAClD,MAAM,CAAC,OAAO,CAAC;QACb,IAAI,EAAE,eAAe;QACrB,WAAW,EAAE;;;;;;;;;;;;;;;;;;8CAkB6B;QAC1C,UAAU,EAAE,kBAAkB;QAC9B,WAAW,EAAE;YACX,KAAK,EAAE,eAAe;YACtB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE;YAC1C,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;YAC9B,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,UAAU,CAAC;YAC7C,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,KAAK,CAAC;YACpD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAqC,CAAC;YAC9D,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,GAAG,CAAC;YAE5C,wBAAwB;YACxB,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;YACvC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC;YACzD,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;YAEhE,cAAc;YACd,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;YAC5D,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;oBACtD,MAAM,QAAQ,GAAG,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;oBACnE,MAAM,OAAO,GAGP,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;oBAE3E,kCAAkC;oBAClC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;wBACzC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;4BAClC,IAAI,CAAC;gCACH,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;4BAC7D,CAAC;4BAAC,MAAM,CAAC;gCACP,sCAAsC;4BACxC,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,OAAO,EAAE,OAAO,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,CAAC;YACZ,IAAI,CAAC;gBACH,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,SAAS;oBAAE,MAAM,KAAK,CAAC;gBAC5C,MAAM,IAAI,SAAS,CAAC,4CAA4C,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,IAAI,OAAO,GAAkB,IAAI,CAAC;YAElC,IAAI,CAAC;gBACH,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAElD,mDAAmD;gBACnD,MAAM,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;oBAC9E,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;wBAC5C,QAAQ,CAAC,IAAI,CACX,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC1E,CAAC;wBACF,OAAO;4BACL,QAAQ,EAAE,OAAO,CAAC,IAAqC;4BACvD,KAAK,EAAE,SAAS;4BAChB,QAAQ,EAAE,CAAC;4BACX,iBAAiB,EAAE,MAAM;4BACzB,GAAG;yBACJ,CAAC;oBACJ,CAAC,CAAC;oBACF,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;wBAC9C,QAAQ,CAAC,IAAI,CACX,+BAA+B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC5E,CAAC;wBACF,OAAO,EAAE,CAAC;oBACZ,CAAC,CAAC;oBACF,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;wBAC5C,QAAQ,CAAC,IAAI,CACX,6BAA6B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC1E,CAAC;wBACF,OAAO,EAAE,CAAC;oBACZ,CAAC,CAAC;oBACF,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC;oBACxC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;iBAC5C,CAAC,CAAC;gBAEH,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAEnD,wCAAwC;gBACxC,MAAM,iBAAiB,GACrB,MAAM,CAAC,oBAAoB,KAAK,IAAI;oBAClC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,oBAAoB,CAAC;oBAClD,CAAC,CAAC,UAAU,CAAC;gBAEjB,oCAAoC;gBACpC,MAAM,MAAM,GAAoB;oBAC9B,QAAQ;oBACR,UAAU,EAAE,iBAAiB;oBAC7B,MAAM,EAAE,EAAE;oBACV,QAAQ;oBACR,QAAQ;oBACR,UAAU,EAAE,EAAE;oBACd,QAAQ,EAAE,EAAE;oBACZ,SAAS,EAAE,SAAS,IAAI,SAAS;oBACjC,QAAQ;iBACT,CAAC;gBAEF,IAAI,SAAS,GAAkB,IAAI,CAAC;gBAEpC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;oBAChC,IAAI,eAAe,GAAG,KAAK,CAAC;oBAE5B,wDAAwD;oBACxD,IAAI,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;wBACvC,SAAS,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;wBAEtD,IAAI,SAAS,EAAE,CAAC;4BACd,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;4BAEnD,+CAA+C;4BAC/C,IAAI,QAAQ,CAAC,QAAQ,KAAK,CAAC,EAAE,CAAC;gCAC5B,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;gCACpE,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;gCAC7B,QAAQ,CAAC,iBAAiB,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;4BACrE,CAAC;4BAED,uCAAuC;4BACvC,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa;gCACpC,CAAC,CAAC,MAAM,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAC/D,CAAC,CAAU,EAAE,EAAE;oCACb,QAAQ,CAAC,IAAI,CACX,kCAAkC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC/E,CAAC;oCACF,OAAO,EAAE,CAAC;gCACZ,CAAC,CACF;gCACH,CAAC,CAAC,MAAM,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE;oCAC3C,SAAS;oCACT,SAAS;iCACV,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;oCACtB,QAAQ,CAAC,IAAI,CACX,4BAA4B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACzE,CAAC;oCACF,OAAO,EAAE,CAAC;gCACZ,CAAC,CAAC,CAAC;4BAEP,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;4BAEnD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gCACzB,MAAM,cAAc,GAAG,MAAM,cAAc,CACzC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAChC,OAAO,CACR,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;oCACrB,QAAQ,CAAC,IAAI,CACX,8BAA8B,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAC3E,CAAC;oCACF,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gCAC1C,CAAC,CAAC,CAAC;gCAEH,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oCAC3C,GAAG,KAAK;oCACR,QAAQ,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,QAAQ;iCAC9C,CAAC,CAAC,CAAC;gCACJ,eAAe,GAAG,IAAI,CAAC;4BACzB,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,kDAAkD;oBAClD,IAAI,CAAC,eAAe,IAAI,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;wBAC9C,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;wBAEnD,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;wBACpE,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE;4BAC7D,UAAU;yBACX,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;4BACtB,QAAQ,CAAC,IAAI,CACX,oCAAoC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACjF,CAAC;4BACF,OAAO,EAAE,CAAC;wBACZ,CAAC,CAAC,CAAC;wBAEH,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;wBAEnD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC7B,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC;4BAC9B,eAAe,GAAG,IAAI,CAAC;wBACzB,CAAC;oBACH,CAAC;oBAED,IAAI,CAAC,eAAe,EAAE,CAAC;wBACrB,QAAQ,CAAC,IAAI,CACX,kIAAkI,CACnI,CAAC;oBACJ,CAAC;oBAED,wCAAwC;oBACxC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;wBACzC,MAAM,CAAC,MAAM,GAAG,MAAM,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;4BAC1E,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACnF,OAAO,MAAM,CAAC,MAAM,CAAC;wBACvB,CAAC,CAAC,CAAC;wBACH,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;4BACvC,QAAQ,CAAC,IAAI,CACX,WAAW,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,2BAA2B,WAAW,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CACjH,CAAC;wBACJ,CAAC;wBAED,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;wBAEnD,sCAAsC;wBACtC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;4BACtB,MAAM,CAAC,UAAU,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;gCAClF,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gCAC3E,OAAO,EAAE,CAAC;4BACZ,CAAC,CAAC,CAAC;wBACL,CAAC;wBAED,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBACrD,CAAC;oBAED,2BAA2B;oBAC3B,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;wBAC3B,MAAM,CAAC,QAAQ,GAAG,sBAAsB,CACtC,MAAM,CAAC,UAAU,EACjB,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,CAClB,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,iEAAiE;oBACjE,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;wBACzE,OAAO,GAAG,OAAO,IAAI,CAAC,MAAM,aAAa,EAAE,CAAC,CAAC;wBAC7C,SAAS,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;oBAC1E,CAAC;gBACH,CAAC;gBAED,8DAA8D;gBAC9D,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,EAAE,CAAC;oBAChD,IAAI,CAAC;wBACH,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC;wBACpE,MAAM,iBAAiB,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;wBAC3D,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACjC,MAAM,CAAC,UAAU,GAAG,iBAAiB,CAAC;4BACtC,QAAQ,CAAC,IAAI,CACX,6EAA6E,CAC9E,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,0DAA0D;oBAC5D,CAAC;gBACH,CAAC;gBAED,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAEpD,wBAAwB;gBACxB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;gBAEvB,qBAAqB;gBACrB,MAAM,QAAQ,GAAG,oBAAoB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAEtD,yBAAyB;gBACzB,MAAM,QAAQ,GAAG;oBACf,GAAG,QAAQ;oBACX,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM;iBACjC,CAAC;gBAEF,MAAM,OAAO,GAGP,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAE3E,yCAAyC;gBACzC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACzC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;wBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,CAAC;YACrB,CAAC;oBAAS,CAAC;gBACT,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;oBAC1B,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/tools/get-frame-at.d.ts

@@ -0,0 +1,2 @@
+import type { FastMCP } from 'fastmcp';
+export declare function registerGetFrameAt(server: FastMCP): void;

package/dist/tools/get-frame-at.js

@@ -0,0 +1,88 @@
+import { imageContent, UserError } from 'fastmcp';
+import { z } from 'zod';
+import { getAdapter } from '../adapters/adapter.interface.js';
+import { extractFrameAt, parseTimestamp } from '../processors/frame-extractor.js';
+import { extractBrowserFrames } from '../processors/browser-frame-extractor.js';
+import { optimizeFrame } from '../processors/image-optimizer.js';
+import { createTempDir } from '../utils/temp-files.js';
+import { getTempFilePath } from '../utils/temp-files.js';
+const GetFrameAtSchema = z.object({
+    url: z.string().url().describe('Video URL (Loom share link or direct mp4/webm URL)'),
+    timestamp: z
+        .string()
+        .describe('Timestamp to extract frame at (e.g., "1:23", "0:05", "01:23:45")'),
+    returnBase64: z
+        .boolean()
+        .default(false)
+        .optional()
+        .describe('Return frame as base64 inline instead of file path'),
+});
+export function registerGetFrameAt(server) {
+    server.addTool({
+        name: 'get_frame_at',
+        description: `Extract a single video frame at a specific timestamp.
+
+Useful for inspecting what's on screen at a particular moment. The AI reads the transcript,
+identifies a critical moment, and requests the exact frame at that timestamp.
+
+Supports: Loom (loom.com/share/...) and direct video URLs (.mp4, .webm, .mov).
+Requires video download capability — direct URLs work best.
+
+Args:
+  - url: Video URL
+  - timestamp: Time position (e.g., "1:23", "0:05", "01:23:45")
+
+Returns: A single image of the video frame at the specified timestamp.`,
+        parameters: GetFrameAtSchema,
+        annotations: {
+            title: 'Get Frame at Timestamp',
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
+        execute: async (args, { reportProgress }) => {
+            const { url, timestamp } = args;
+            const adapter = getAdapter(url);
+            await reportProgress({ progress: 0, total: 100 });
+            const tempDir = await createTempDir();
+            // Strategy 1: Download video + ffmpeg extraction
+            if (adapter.capabilities.videoDownload) {
+                const videoPath = await adapter.downloadVideo(url, tempDir);
+                if (videoPath) {
+                    await reportProgress({ progress: 50, total: 100 });
+                    const frame = await extractFrameAt(videoPath, tempDir, timestamp);
+                    const optimizedPath = getTempFilePath(tempDir, `opt_frame_at.jpg`);
+                    await optimizeFrame(frame.filePath, optimizedPath);
+                    await reportProgress({ progress: 100, total: 100 });
+                    return {
+                        content: [
+                            { type: 'text', text: `Frame extracted at ${timestamp}` },
+                            await imageContent({ path: optimizedPath }),
+                        ],
+                    };
+                }
+            }
+            // Strategy 2: Browser-based extraction (fallback)
+            await reportProgress({ progress: 30, total: 100 });
+            const seconds = parseTimestamp(timestamp);
+            const browserFrames = await extractBrowserFrames(url, tempDir, {
+                timestamps: [seconds],
+            });
+            if (browserFrames.length > 0) {
+                await reportProgress({ progress: 100, total: 100 });
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Frame extracted at ${timestamp} (via browser)`,
+                        },
+                        await imageContent({ path: browserFrames[0].filePath }),
+                    ],
+                };
+            }
+            throw new UserError('Failed to extract frame. Install yt-dlp or Chrome/Chromium for frame extraction.');
+        },
+    });
+}
+//# sourceMappingURL=get-frame-at.js.map

package/dist/tools/get-frame-at.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-frame-at.js","sourceRoot":"","sources":["../../src/tools/get-frame-at.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAE,aAAa,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IACpF,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,CAAC,kEAAkE,CAAC;IAC/E,YAAY,EAAE,CAAC;SACZ,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,EAAE;SACV,QAAQ,CAAC,oDAAoD,CAAC;CAClE,CAAC,CAAC;AAEH,MAAM,UAAU,kBAAkB,CAAC,MAAe;IAChD,MAAM,CAAC,OAAO,CAAC;QACb,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE;;;;;;;;;;;;uEAYsD;QACnE,UAAU,EAAE,gBAAgB;QAC5B,WAAW,EAAE;YACX,KAAK,EAAE,wBAAwB;YAC/B,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE;YAC1C,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;YAEhC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAEhC,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAElD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YAEtC,iDAAiD;YACjD,IAAI,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;gBACvC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAE5D,IAAI,SAAS,EAAE,CAAC;oBACd,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBAEnD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;oBAClE,MAAM,aAAa,GAAG,eAAe,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;oBACnE,MAAM,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;oBAEnD,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBAEpD,OAAO;wBACL,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,sBAAsB,SAAS,EAAE,EAAE;4BAClE,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;yBAC5C;qBACF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,kDAAkD;YAClD,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACnD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;YAC1C,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE;gBAC7D,UAAU,EAAE,CAAC,OAAO,CAAC;aACtB,CAAC,CAAC;YAEH,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBACpD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,sBAAsB,SAAS,gBAAgB;yBACtD;wBACD,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;qBACxD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,SAAS,CACjB,kFAAkF,CACnF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/tools/get-frame-burst.d.ts

@@ -0,0 +1,2 @@
+import type { FastMCP } from 'fastmcp';
+export declare function registerGetFrameBurst(server: FastMCP): void;

package/dist/tools/get-frame-burst.js

@@ -0,0 +1,106 @@
+import { imageContent, UserError } from 'fastmcp';
+import { z } from 'zod';
+import { getAdapter } from '../adapters/adapter.interface.js';
+import { extractFrameBurst, parseTimestamp } from '../processors/frame-extractor.js';
+import { extractBrowserFrames } from '../processors/browser-frame-extractor.js';
+import { optimizeFrames } from '../processors/image-optimizer.js';
+import { createTempDir } from '../utils/temp-files.js';
+const GetFrameBurstSchema = z.object({
+    url: z.string().url().describe('Video URL (Loom share link or direct mp4/webm URL)'),
+    from: z.string().describe('Start timestamp (e.g., "0:15")'),
+    to: z.string().describe('End timestamp (e.g., "0:17")'),
+    count: z
+        .number()
+        .min(2)
+        .max(30)
+        .default(5)
+        .optional()
+        .describe('Number of frames to extract (default: 5)'),
+    returnBase64: z
+        .boolean()
+        .default(false)
+        .optional()
+        .describe('Return frames as base64 inline instead of file paths'),
+});
+export function registerGetFrameBurst(server) {
+    server.addTool({
+        name: 'get_frame_burst',
+        description: `Extract multiple frames evenly distributed across a time range.
+
+Designed for motion and vibration analysis where scene-change detection fails because
+the "scene" doesn't change — only the position/state of objects does.
+
+Example: get_frame_burst(url, "0:15", "0:17", 10) → 10 frames in 2 seconds
+- AI sees the object in different positions across frames → understands the vibration
+- Works for: shaking, flickering, animations, fast scrolling, loading spinners
+
+Supports: Loom (loom.com/share/...) and direct video URLs (.mp4, .webm, .mov).
+Requires video download capability — direct URLs work best.
+
+Args:
+  - url: Video URL
+  - from: Start timestamp (e.g., "0:15")
+  - to: End timestamp (e.g., "0:17")
+  - count: Number of frames (default: 5, max: 30)
+
+Returns: N images evenly distributed between the from and to timestamps.`,
+        parameters: GetFrameBurstSchema,
+        annotations: {
+            title: 'Get Frame Burst',
+            readOnlyHint: true,
+            destructiveHint: false,
+            idempotentHint: true,
+            openWorldHint: true,
+        },
+        execute: async (args, { reportProgress }) => {
+            const { url, from, to, count } = args;
+            const frameCount = count ?? 5;
+            const adapter = getAdapter(url);
+            await reportProgress({ progress: 0, total: 100 });
+            const tempDir = await createTempDir();
+            // Strategy 1: Download video + ffmpeg burst extraction
+            if (adapter.capabilities.videoDownload) {
+                const videoPath = await adapter.downloadVideo(url, tempDir);
+                if (videoPath) {
+                    await reportProgress({ progress: 40, total: 100 });
+                    const frames = await extractFrameBurst(videoPath, tempDir, from, to, frameCount);
+                    await reportProgress({ progress: 70, total: 100 });
+                    const optimizedPaths = await optimizeFrames(frames.map((f) => f.filePath), tempDir);
+                    await reportProgress({ progress: 100, total: 100 });
+                    const content = [
+                        {
+                            type: 'text',
+                            text: `Extracted ${optimizedPaths.length} frames from ${from} to ${to}`,
+                        },
+                    ];
+                    for (const path of optimizedPaths) {
+                        content.push(await imageContent({ path }));
+                    }
+                    return { content };
+                }
+            }
+            // Strategy 2: Browser-based extraction (fallback)
+            await reportProgress({ progress: 30, total: 100 });
+            const fromSeconds = parseTimestamp(from);
+            const toSeconds = parseTimestamp(to);
+            const interval = (toSeconds - fromSeconds) / Math.max(frameCount - 1, 1);
+            const timestamps = Array.from({ length: frameCount }, (_, i) => Math.round(fromSeconds + i * interval));
+            const browserFrames = await extractBrowserFrames(url, tempDir, { timestamps });
+            if (browserFrames.length > 0) {
+                await reportProgress({ progress: 100, total: 100 });
+                const content = [
+                    {
+                        type: 'text',
+                        text: `Extracted ${browserFrames.length} frames from ${from} to ${to} (via browser)`,
+                    },
+                ];
+                for (const frame of browserFrames) {
+                    content.push(await imageContent({ path: frame.filePath }));
+                }
+                return { content };
+            }
+            throw new UserError('Failed to extract frames. Install yt-dlp or Chrome/Chromium for frame extraction.');
+        },
+    });
+}
+//# sourceMappingURL=get-frame-burst.js.map

package/dist/tools/get-frame-burst.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-frame-burst.js","sourceRoot":"","sources":["../../src/tools/get-frame-burst.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAClD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,kCAAkC,CAAC;AAC9D,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AACrF,OAAO,EAAE,oBAAoB,EAAE,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,kCAAkC,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IACpF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,gCAAgC,CAAC;IAC3D,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACvD,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,0CAA0C,CAAC;IACvD,YAAY,EAAE,CAAC;SACZ,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,EAAE;SACV,QAAQ,CAAC,sDAAsD,CAAC;CACpE,CAAC,CAAC;AAEH,MAAM,UAAU,qBAAqB,CAAC,MAAe;IACnD,MAAM,CAAC,OAAO,CAAC;QACb,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE;;;;;;;;;;;;;;;;;;yEAkBwD;QACrE,UAAU,EAAE,mBAAmB;QAC/B,WAAW,EAAE;YACX,KAAK,EAAE,iBAAiB;YACxB,YAAY,EAAE,IAAI;YAClB,eAAe,EAAE,KAAK;YACtB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;QACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,cAAc,EAAE,EAAE,EAAE;YAC1C,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;YACtC,MAAM,UAAU,GAAG,KAAK,IAAI,CAAC,CAAC;YAE9B,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAEhC,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAElD,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;YAEtC,uDAAuD;YACvD,IAAI,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;gBACvC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAE5D,IAAI,SAAS,EAAE,CAAC;oBACd,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBAEnD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC;oBAEjF,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBAEnD,MAAM,cAAc,GAAG,MAAM,cAAc,CACzC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC7B,OAAO,CACR,CAAC;oBAEF,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBAEpD,MAAM,OAAO,GAGP;wBACJ;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,aAAa,cAAc,CAAC,MAAM,gBAAgB,IAAI,OAAO,EAAE,EAAE;yBACxE;qBACF,CAAC;oBAEF,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;wBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;oBAC7C,CAAC;oBAED,OAAO,EAAE,OAAO,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,kDAAkD;YAClD,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACnD,MAAM,WAAW,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;YACzC,MAAM,SAAS,GAAG,cAAc,CAAC,EAAE,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YACzE,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC7D,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CACvC,CAAC;YAEF,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;YAE/E,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAEpD,MAAM,OAAO,GAGP;oBACJ;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,aAAa,aAAa,CAAC,MAAM,gBAAgB,IAAI,OAAO,EAAE,gBAAgB;qBACrF;iBACF,CAAC;gBAEF,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;oBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;gBAC7D,CAAC;gBAED,OAAO,EAAE,OAAO,EAAE,CAAC;YACrB,CAAC;YAED,MAAM,IAAI,SAAS,CACjB,mFAAmF,CACpF,CAAC;QACJ,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/tools/get-frames.d.ts

@@ -0,0 +1,2 @@
+import type { FastMCP } from 'fastmcp';
+export declare function registerGetFrames(server: FastMCP): void;