@vibeframe/cli 0.55.0 → 0.55.1

package/dist/agent/tools/ai-editing.js

@@ -1,763 +0,0 @@
-/**
- * @module ai-editing
- * @description Agent tools for post-production editing (text overlay, review,
- * silence cut, jump cut, captions, noise reduction, fade, thumbnail,
- * SRT translation). FFmpeg-based and AI-assisted editing tools for agent use.
- * Most tools work without API keys (FFmpeg-only), some use Gemini or OpenAI.
- *
- * ## Tools: edit_text_overlay, analyze_review, edit_silence_cut, edit_jump_cut, edit_caption,
- *          edit_noise_reduce, edit_fade, generate_thumbnail, edit_translate_srt
- * ## Dependencies: FFmpeg, Gemini (optional), OpenAI/Whisper (optional)
- * @see MODELS.md for the Single Source of Truth (SSOT) on supported providers/models
- */
-import { resolve } from "node:path";
-import { executeTextOverlay, executeSilenceCut, executeJumpCut, executeCaption, executeNoiseReduce, executeFade, executeTranslateSrt, } from "../../commands/ai-edit.js";
-import { executeReview } from "../../commands/ai-review.js";
-import { executeThumbnailBestFrame } from "../../commands/ai-image.js";
-import { sanitizeAIResult } from "../../commands/sanitize.js";
-// ============================================================================
-// Tool Definitions
-// ============================================================================
-const textOverlayDef = {
-    name: "edit_text_overlay",
-    description: "Apply text overlays to a video using FFmpeg drawtext. Supports 4 style presets: lower-third, center-bold, subtitle, minimal. Auto-detects font and scales based on video resolution.",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to input video file",
-            },
-            texts: {
-                type: "array",
-                items: { type: "string", description: "Text line to overlay" },
-                description: "Text lines to overlay (multiple lines stack vertically)",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output video file path",
-            },
-            style: {
-                type: "string",
-                description: "Overlay style preset",
-                enum: ["lower-third", "center-bold", "subtitle", "minimal"],
-            },
-            fontSize: {
-                type: "number",
-                description: "Font size in pixels (auto-calculated if omitted)",
-            },
-            fontColor: {
-                type: "string",
-                description: "Font color (default: white)",
-            },
-            fadeDuration: {
-                type: "number",
-                description: "Fade in/out duration in seconds (default: 0.3)",
-            },
-            startTime: {
-                type: "number",
-                description: "Start time for overlay in seconds (default: 0)",
-            },
-            endTime: {
-                type: "number",
-                description: "End time for overlay in seconds (default: video duration)",
-            },
-        },
-        required: ["videoPath", "texts", "outputPath"],
-    },
-};
-const reviewDef = {
-    name: "analyze_review",
-    description: "Review video quality using Gemini AI. Analyzes pacing, color, text readability, audio-visual sync, and composition. Can auto-apply fixable corrections (color grading). Returns structured feedback with scores and recommendations.",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to video file to review",
-            },
-            storyboardPath: {
-                type: "string",
-                description: "Optional path to storyboard JSON for context",
-            },
-            autoApply: {
-                type: "boolean",
-                description: "Automatically apply fixable corrections (default: false)",
-            },
-            verify: {
-                type: "boolean",
-                description: "Run verification pass after applying fixes (default: false)",
-            },
-            model: {
-                type: "string",
-                description: "Gemini model: flash (default), flash-2.5, pro",
-                enum: ["flash", "flash-2.5", "pro"],
-            },
-            outputPath: {
-                type: "string",
-                description: "Output path for corrected video (when autoApply is true)",
-            },
-        },
-        required: ["videoPath"],
-    },
-};
-const silenceCutDef = {
-    name: "edit_silence_cut",
-    description: "Remove silent segments from a video. Default uses FFmpeg silencedetect (free, no API key). Use useGemini=true for smart context-aware detection via Gemini Video Understanding — distinguishes dead air from intentional pauses using visual+audio analysis.",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to input video file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output file path (default: <name>-cut.<ext>)",
-            },
-            noiseThreshold: {
-                type: "number",
-                description: "Silence threshold in dB (default: -30). Lower = more sensitive. FFmpeg mode only.",
-            },
-            minDuration: {
-                type: "number",
-                description: "Minimum silence duration in seconds to cut (default: 0.5)",
-            },
-            padding: {
-                type: "number",
-                description: "Padding around non-silent segments in seconds (default: 0.1)",
-            },
-            analyzeOnly: {
-                type: "boolean",
-                description: "Only detect silence without cutting (default: false)",
-            },
-            useGemini: {
-                type: "boolean",
-                description: "Use Gemini Video Understanding for context-aware silence detection (default: false). Requires GOOGLE_API_KEY.",
-            },
-            model: {
-                type: "string",
-                description: "Gemini model to use (default: flash). Options: flash, flash-2.5, pro",
-            },
-            lowRes: {
-                type: "boolean",
-                description: "Low resolution mode for longer videos (Gemini only)",
-            },
-        },
-        required: ["videoPath"],
-    },
-};
-const jumpCutDef = {
-    name: "edit_jump_cut",
-    description: "Remove filler words (um, uh, like, etc.) from video using Whisper word-level timestamps + FFmpeg concat. Requires OpenAI API key. Detects filler words, cuts them out, and stitches remaining segments with stream copy (fast, no re-encode).",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to input video file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output file path (default: <name>-jumpcut.<ext>)",
-            },
-            fillers: {
-                type: "array",
-                items: { type: "string", description: "A filler word to detect" },
-                description: "Custom filler words to detect (default: um, uh, like, you know, etc.)",
-            },
-            padding: {
-                type: "number",
-                description: "Padding around cuts in seconds (default: 0.05)",
-            },
-            language: {
-                type: "string",
-                description: "Language code for transcription (e.g., en, ko)",
-            },
-            analyzeOnly: {
-                type: "boolean",
-                description: "Only detect fillers without cutting (default: false)",
-            },
-        },
-        required: ["videoPath"],
-    },
-};
-const captionDef = {
-    name: "edit_caption",
-    description: "Transcribe video with Whisper and burn styled captions using FFmpeg. Requires OpenAI API key. 4 style presets: minimal, bold (default), outline, karaoke. Auto-sizes font based on video resolution.",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to input video file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output file path (default: <name>-captioned.<ext>)",
-            },
-            style: {
-                type: "string",
-                description: "Caption style preset",
-                enum: ["minimal", "bold", "outline", "karaoke"],
-            },
-            fontSize: {
-                type: "number",
-                description: "Font size in pixels (auto-calculated based on resolution if omitted)",
-            },
-            fontColor: {
-                type: "string",
-                description: "Font color (default: white)",
-            },
-            language: {
-                type: "string",
-                description: "Language code for transcription (e.g., en, ko)",
-            },
-            position: {
-                type: "string",
-                description: "Caption position",
-                enum: ["top", "center", "bottom"],
-            },
-        },
-        required: ["videoPath"],
-    },
-};
-const noiseReduceDef = {
-    name: "edit_noise_reduce",
-    description: "Remove background noise from audio/video using FFmpeg afftdn filter. No API key needed. Three strength presets: low, medium (default), high. High adds bandpass filtering.",
-    parameters: {
-        type: "object",
-        properties: {
-            inputPath: {
-                type: "string",
-                description: "Path to input audio or video file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output file path (default: <name>-denoised.<ext>)",
-            },
-            strength: {
-                type: "string",
-                description: "Noise reduction strength",
-                enum: ["low", "medium", "high"],
-            },
-            noiseFloor: {
-                type: "number",
-                description: "Custom noise floor in dB (overrides strength preset)",
-            },
-        },
-        required: ["inputPath"],
-    },
-};
-const fadeDef = {
-    name: "edit_fade",
-    description: "Apply fade in/out effects to video using FFmpeg. No API key needed. Supports video-only, audio-only, or both. Configurable fade durations.",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to input video file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output file path (default: <name>-faded.<ext>)",
-            },
-            fadeIn: {
-                type: "number",
-                description: "Fade-in duration in seconds (default: 1)",
-            },
-            fadeOut: {
-                type: "number",
-                description: "Fade-out duration in seconds (default: 1)",
-            },
-            audioOnly: {
-                type: "boolean",
-                description: "Apply fade to audio only (default: false)",
-            },
-            videoOnly: {
-                type: "boolean",
-                description: "Apply fade to video only (default: false)",
-            },
-        },
-        required: ["videoPath"],
-    },
-};
-const thumbnailBestFrameDef = {
-    name: "generate_thumbnail",
-    description: "Extract the best thumbnail frame from a video using Gemini AI analysis + FFmpeg frame extraction. Requires GOOGLE_API_KEY. Finds visually striking, well-composed frames.",
-    parameters: {
-        type: "object",
-        properties: {
-            videoPath: {
-                type: "string",
-                description: "Path to input video file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output image path (default: <name>-thumbnail.png)",
-            },
-            prompt: {
-                type: "string",
-                description: "Custom prompt for frame selection analysis",
-            },
-            model: {
-                type: "string",
-                description: "Gemini model to use",
-                enum: ["flash", "flash-2.5", "pro"],
-            },
-        },
-        required: ["videoPath"],
-    },
-};
-const translateSrtDef = {
-    name: "edit_translate_srt",
-    description: "Translate SRT subtitle file to another language using Claude or OpenAI. Preserves timestamps. Batches segments for efficiency.",
-    parameters: {
-        type: "object",
-        properties: {
-            srtPath: {
-                type: "string",
-                description: "Path to input SRT file",
-            },
-            outputPath: {
-                type: "string",
-                description: "Output file path (default: <name>-<target>.srt)",
-            },
-            targetLanguage: {
-                type: "string",
-                description: "Target language (e.g., ko, es, fr, ja, zh)",
-            },
-            provider: {
-                type: "string",
-                description: "Translation provider",
-                enum: ["claude", "openai"],
-            },
-            sourceLanguage: {
-                type: "string",
-                description: "Source language (auto-detected if omitted)",
-            },
-        },
-        required: ["srtPath", "targetLanguage"],
-    },
-};
-// ============================================================================
-// Tool Handlers
-// ============================================================================
-const textOverlayHandler = async (args) => {
-    const { videoPath, texts, outputPath, style, fontSize, fontColor, fadeDuration, startTime, endTime } = args;
-    if (!videoPath || !texts || texts.length === 0 || !outputPath) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: "videoPath, texts (non-empty array), and outputPath are required",
-        };
-    }
-    const result = await executeTextOverlay({
-        videoPath,
-        texts,
-        outputPath,
-        style,
-        fontSize,
-        fontColor,
-        fadeDuration,
-        startTime,
-        endTime,
-    });
-    if (!result.success) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: result.error || "Text overlay failed",
-        };
-    }
-    return {
-        toolCallId: "",
-        success: true,
-        output: `Text overlay applied: ${result.outputPath}`,
-    };
-};
-const reviewHandler = async (args) => {
-    const { videoPath, storyboardPath, autoApply, verify, model, outputPath } = args;
-    if (!videoPath) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: "videoPath is required",
-        };
-    }
-    const result = await executeReview({
-        videoPath,
-        storyboardPath,
-        autoApply,
-        verify,
-        model,
-        outputPath,
-    });
-    if (!result.success) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: result.error || "Video review failed",
-        };
-    }
-    const fb = sanitizeAIResult(result.feedback);
-    let output = `Video Review: ${fb.overallScore}/10\n`;
-    output += `Pacing: ${fb.categories.pacing.score}/10, Color: ${fb.categories.color.score}/10, `;
-    output += `Text: ${fb.categories.textReadability.score}/10, AV Sync: ${fb.categories.audioVisualSync.score}/10, `;
-    output += `Composition: ${fb.categories.composition.score}/10\n`;
-    if (result.appliedFixes && result.appliedFixes.length > 0) {
-        output += `Applied fixes: ${sanitizeAIResult(result.appliedFixes).join("; ")}\n`;
-    }
-    if (result.verificationScore !== undefined) {
-        output += `Verification score: ${result.verificationScore}/10\n`;
-    }
-    if (fb.recommendations.length > 0) {
-        output += `Recommendations: ${fb.recommendations.join("; ")}`;
-    }
-    return {
-        toolCallId: "",
-        success: true,
-        output,
-    };
-};
-const silenceCutHandler = async (args, context) => {
-    const videoPath = resolve(context.workingDirectory, args.videoPath);
-    const ext = videoPath.split(".").pop() || "mp4";
-    const name = videoPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-cut.${ext}`;
-    try {
-        const result = await executeSilenceCut({
-            videoPath,
-            outputPath,
-            noiseThreshold: args.noiseThreshold,
-            minDuration: args.minDuration,
-            padding: args.padding,
-            analyzeOnly: args.analyzeOnly,
-            useGemini: args.useGemini,
-            model: args.model,
-            lowRes: args.lowRes,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Silence cut failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Detection method: ${result.method === "gemini" ? "Gemini Video Understanding" : "FFmpeg silencedetect"}`);
-        lines.push(`Total duration: ${result.totalDuration.toFixed(1)}s`);
-        lines.push(`Silent periods: ${result.silentPeriods.length}`);
-        lines.push(`Silent duration: ${result.silentDuration.toFixed(1)}s`);
-        lines.push(`Non-silent duration: ${(result.totalDuration - result.silentDuration).toFixed(1)}s`);
-        if (result.outputPath) {
-            lines.push(`Output: ${result.outputPath}`);
-        }
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Silence cut failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-const jumpCutHandler = async (args, context) => {
-    const videoPath = resolve(context.workingDirectory, args.videoPath);
-    const ext = videoPath.split(".").pop() || "mp4";
-    const name = videoPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-jumpcut.${ext}`;
-    try {
-        const result = await executeJumpCut({
-            videoPath,
-            outputPath,
-            fillers: args.fillers,
-            padding: args.padding,
-            language: args.language,
-            analyzeOnly: args.analyzeOnly,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Jump cut failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Total duration: ${result.totalDuration.toFixed(1)}s`);
-        lines.push(`Filler words found: ${result.fillerCount}`);
-        lines.push(`Filler duration: ${result.fillerDuration.toFixed(1)}s`);
-        lines.push(`Clean duration: ${(result.totalDuration - result.fillerDuration).toFixed(1)}s`);
-        if (result.fillers && result.fillers.length > 0) {
-            lines.push("");
-            lines.push("Detected fillers:");
-            for (const filler of result.fillers) {
-                lines.push(`  "${filler.word}" at ${filler.start.toFixed(2)}s - ${filler.end.toFixed(2)}s`);
-            }
-        }
-        if (result.outputPath) {
-            lines.push(`Output: ${result.outputPath}`);
-        }
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Jump cut failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-const captionHandler = async (args, context) => {
-    const videoPath = resolve(context.workingDirectory, args.videoPath);
-    const ext = videoPath.split(".").pop() || "mp4";
-    const name = videoPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-captioned.${ext}`;
-    try {
-        const result = await executeCaption({
-            videoPath,
-            outputPath,
-            style: args.style,
-            fontSize: args.fontSize,
-            fontColor: args.fontColor,
-            language: args.language,
-            position: args.position,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Caption failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Captions applied: ${result.outputPath}`);
-        lines.push(`Segments transcribed: ${result.segmentCount}`);
-        if (result.srtPath) {
-            lines.push(`SRT file: ${result.srtPath}`);
-        }
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Caption failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-const noiseReduceHandler = async (args, context) => {
-    const inputPath = resolve(context.workingDirectory, args.inputPath);
-    const ext = inputPath.split(".").pop() || "mp4";
-    const name = inputPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-denoised.${ext}`;
-    try {
-        const result = await executeNoiseReduce({
-            inputPath,
-            outputPath,
-            strength: args.strength,
-            noiseFloor: args.noiseFloor,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Noise reduction failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Noise reduction applied: ${result.outputPath}`);
-        lines.push(`Input duration: ${result.inputDuration.toFixed(1)}s`);
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Noise reduction failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-const fadeHandler = async (args, context) => {
-    const videoPath = resolve(context.workingDirectory, args.videoPath);
-    const ext = videoPath.split(".").pop() || "mp4";
-    const name = videoPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-faded.${ext}`;
-    try {
-        const result = await executeFade({
-            videoPath,
-            outputPath,
-            fadeIn: args.fadeIn,
-            fadeOut: args.fadeOut,
-            audioOnly: args.audioOnly,
-            videoOnly: args.videoOnly,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Fade failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Fade effects applied: ${result.outputPath}`);
-        lines.push(`Total duration: ${result.totalDuration.toFixed(1)}s`);
-        if (result.fadeInApplied)
-            lines.push(`Fade-in applied`);
-        if (result.fadeOutApplied)
-            lines.push(`Fade-out applied`);
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Fade failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-const thumbnailBestFrameHandler = async (args, context) => {
-    const videoPath = resolve(context.workingDirectory, args.videoPath);
-    const name = videoPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-thumbnail.png`;
-    try {
-        const result = await executeThumbnailBestFrame({
-            videoPath,
-            outputPath,
-            prompt: args.prompt,
-            model: args.model,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Best frame extraction failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Best frame extracted: ${result.outputPath}`);
-        lines.push(`Timestamp: ${result.timestamp.toFixed(2)}s`);
-        if (result.reason)
-            lines.push(`Reason: ${sanitizeAIResult(result.reason)}`);
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Best frame extraction failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-const translateSrtHandler = async (args, context) => {
-    const srtPath = resolve(context.workingDirectory, args.srtPath);
-    const target = args.targetLanguage;
-    const ext = srtPath.split(".").pop() || "srt";
-    const name = srtPath.replace(/\.[^.]+$/, "");
-    const outputPath = args.outputPath
-        ? resolve(context.workingDirectory, args.outputPath)
-        : `${name}-${target}.${ext}`;
-    try {
-        const result = await executeTranslateSrt({
-            srtPath,
-            outputPath,
-            targetLanguage: target,
-            provider: args.provider,
-            sourceLanguage: args.sourceLanguage,
-        });
-        if (!result.success) {
-            return {
-                toolCallId: "",
-                success: false,
-                output: "",
-                error: result.error || "Translation failed",
-            };
-        }
-        const lines = [];
-        lines.push(`Translation complete: ${result.outputPath}`);
-        lines.push(`Segments translated: ${result.segmentCount}`);
-        lines.push(`Target language: ${result.targetLanguage}`);
-        return {
-            toolCallId: "",
-            success: true,
-            output: lines.join("\n"),
-        };
-    }
-    catch (error) {
-        return {
-            toolCallId: "",
-            success: false,
-            output: "",
-            error: `Translation failed: ${error instanceof Error ? error.message : String(error)}`,
-        };
-    }
-};
-// ============================================================================
-// Registration
-// ============================================================================
-export function registerEditingTools(registry) {
-    registry.register(textOverlayDef, textOverlayHandler);
-    registry.register(reviewDef, reviewHandler);
-    registry.register(silenceCutDef, silenceCutHandler);
-    registry.register(jumpCutDef, jumpCutHandler);
-    registry.register(captionDef, captionHandler);
-    registry.register(noiseReduceDef, noiseReduceHandler);
-    registry.register(fadeDef, fadeHandler);
-    registry.register(thumbnailBestFrameDef, thumbnailBestFrameHandler);
-    registry.register(translateSrtDef, translateSrtHandler);
-}
-//# sourceMappingURL=ai-editing.js.map