@vibeframe/cli 0.27.0 → 0.29.0

package/src/agent/tools/media.ts

@@ -1,697 +0,0 @@
-/**
- * Media Tools - Analyze media files (scenes, silence, beats, transcription)
- */
-
-import { readFile, writeFile } from "node:fs/promises";
-import { resolve } from "node:path";
-import type { ToolRegistry, ToolHandler } from "./index.js";
-import type { ToolDefinition, ToolResult } from "../types.js";
-import { getApiKeyFromConfig } from "../../config/index.js";
-import { execSafe, ffprobeDuration } from "../../utils/exec-safe.js";
-
-// Tool Definitions
-const mediaInfoDef: ToolDefinition = {
-  name: "media_info",
-  description: "Get information about a media file (duration, resolution, codec, etc.)",
-  parameters: {
-    type: "object",
-    properties: {
-      path: {
-        type: "string",
-        description: "Media file path",
-      },
-    },
-    required: ["path"],
-  },
-};
-
-const detectScenesDef: ToolDefinition = {
-  name: "detect_scenes",
-  description: "Detect scene changes in a video file",
-  parameters: {
-    type: "object",
-    properties: {
-      video: {
-        type: "string",
-        description: "Video file path",
-      },
-      threshold: {
-        type: "number",
-        description: "Scene change threshold (0-1, default: 0.3)",
-      },
-      output: {
-        type: "string",
-        description: "Output JSON file path for timestamps",
-      },
-    },
-    required: ["video"],
-  },
-};
-
-const detectSilenceDef: ToolDefinition = {
-  name: "detect_silence",
-  description: "Detect silence periods in audio/video",
-  parameters: {
-    type: "object",
-    properties: {
-      media: {
-        type: "string",
-        description: "Media file path",
-      },
-      noise: {
-        type: "string",
-        description: "Noise threshold in dB (default: -30)",
-      },
-      duration: {
-        type: "number",
-        description: "Minimum silence duration in seconds (default: 0.5)",
-      },
-      output: {
-        type: "string",
-        description: "Output JSON file path",
-      },
-    },
-    required: ["media"],
-  },
-};
-
-const detectBeatsDef: ToolDefinition = {
-  name: "detect_beats",
-  description: "Detect beats in audio for music sync",
-  parameters: {
-    type: "object",
-    properties: {
-      audio: {
-        type: "string",
-        description: "Audio file path",
-      },
-      output: {
-        type: "string",
-        description: "Output JSON file path",
-      },
-    },
-    required: ["audio"],
-  },
-};
-
-const transcribeDef: ToolDefinition = {
-  name: "audio_transcribe",
-  description: "Transcribe audio using Whisper AI",
-  parameters: {
-    type: "object",
-    properties: {
-      audio: {
-        type: "string",
-        description: "Audio file path",
-      },
-      language: {
-        type: "string",
-        description: "Language code (e.g., en, ko)",
-      },
-      output: {
-        type: "string",
-        description: "Output file path (supports .json, .srt, .vtt)",
-      },
-    },
-    required: ["audio"],
-  },
-};
-
-// Helper to format timestamp
-function formatTimestamp(seconds: number): string {
-  const mins = Math.floor(seconds / 60);
-  const secs = (seconds % 60).toFixed(2);
-  return `${mins.toString().padStart(2, "0")}:${secs.padStart(5, "0")}`;
-}
-
-// Tool Handlers
-const mediaInfo: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const mediaPath = args.path as string;
-
-  try {
-    const absPath = resolve(context.workingDirectory, mediaPath);
-
-    // Get detailed info using ffprobe
-    const { stdout } = await execSafe("ffprobe", ["-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", absPath], { maxBuffer: 10 * 1024 * 1024 });
-
-    const info = JSON.parse(stdout);
-    const format = info.format || {};
-    const streams = info.streams || [];
-
-    const output: string[] = [`File: ${mediaPath}`];
-
-    // General info
-    if (format.duration) {
-      output.push(`Duration: ${parseFloat(format.duration).toFixed(2)}s`);
-    }
-    if (format.size) {
-      const sizeMB = (parseInt(format.size) / (1024 * 1024)).toFixed(2);
-      output.push(`Size: ${sizeMB} MB`);
-    }
-    if (format.bit_rate) {
-      const bitrateMbps = (parseInt(format.bit_rate) / 1000000).toFixed(2);
-      output.push(`Bitrate: ${bitrateMbps} Mbps`);
-    }
-
-    // Stream info
-    for (const stream of streams) {
-      if (stream.codec_type === "video") {
-        output.push(`\nVideo:`);
-        output.push(`  Resolution: ${stream.width}x${stream.height}`);
-        output.push(`  Codec: ${stream.codec_name}`);
-        if (stream.r_frame_rate) {
-          const [num, den] = stream.r_frame_rate.split("/");
-          const fps = (parseInt(num) / parseInt(den)).toFixed(2);
-          output.push(`  Frame Rate: ${fps} fps`);
-        }
-      } else if (stream.codec_type === "audio") {
-        output.push(`\nAudio:`);
-        output.push(`  Codec: ${stream.codec_name}`);
-        output.push(`  Sample Rate: ${stream.sample_rate} Hz`);
-        output.push(`  Channels: ${stream.channels}`);
-      }
-    }
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: output.join("\n"),
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to get media info: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-const detectScenes: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const videoPath = args.video as string;
-  const threshold = (args.threshold as number) || 0.3;
-
-  try {
-    const absPath = resolve(context.workingDirectory, videoPath);
-
-    // Detect scenes using FFmpeg
-    const { stdout, stderr } = await execSafe("ffmpeg", ["-i", absPath, "-filter:v", `select='gt(scene,${threshold})',showinfo`, "-f", "null", "-"], { maxBuffer: 50 * 1024 * 1024 });
-    const output = stdout + stderr;
-
-    // Parse scene timestamps
-    const scenes: { timestamp: number; score: number }[] = [];
-    const regex = /pts_time:(\d+\.?\d*)/g;
-    let match;
-
-    scenes.push({ timestamp: 0, score: 1 });
-    while ((match = regex.exec(output)) !== null) {
-      scenes.push({ timestamp: parseFloat(match[1]), score: threshold });
-    }
-
-    // Get duration
-    const totalDuration = await ffprobeDuration(absPath);
-
-    // Format output
-    const sceneList = scenes.map((s, i) => {
-      const end = i < scenes.length - 1 ? scenes[i + 1].timestamp : totalDuration;
-      return `[${i + 1}] ${formatTimestamp(s.timestamp)} - ${formatTimestamp(end)} (${(end - s.timestamp).toFixed(1)}s)`;
-    }).join("\n");
-
-    // Save to file if requested
-    if (args.output) {
-      const outputPath = resolve(context.workingDirectory, args.output as string);
-      const result = {
-        source: absPath,
-        totalDuration,
-        threshold,
-        scenes: scenes.map((s, i) => ({
-          index: i,
-          startTime: s.timestamp,
-          endTime: i < scenes.length - 1 ? scenes[i + 1].timestamp : totalDuration,
-          duration: (i < scenes.length - 1 ? scenes[i + 1].timestamp : totalDuration) - s.timestamp,
-        })),
-      };
-      await writeFile(outputPath, JSON.stringify(result, null, 2), "utf-8");
-    }
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: `Detected ${scenes.length} scenes:\n${sceneList}`,
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to detect scenes: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-const detectSilence: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const mediaPath = args.media as string;
-  const noise = (args.noise as string) || "-30";
-  const duration = (args.duration as number) || 0.5;
-
-  try {
-    const absPath = resolve(context.workingDirectory, mediaPath);
-
-    const { stdout, stderr } = await execSafe("ffmpeg", ["-i", absPath, "-af", `silencedetect=noise=${noise}dB:d=${duration}`, "-f", "null", "-"], { maxBuffer: 50 * 1024 * 1024 });
-    const output = stdout + stderr;
-
-    // Parse silence periods
-    const silences: { start: number; end: number; duration: number }[] = [];
-    const startRegex = /silence_start: (\d+\.?\d*)/g;
-    const endRegex = /silence_end: (\d+\.?\d*) \| silence_duration: (\d+\.?\d*)/g;
-
-    const starts: number[] = [];
-    let match;
-
-    while ((match = startRegex.exec(output)) !== null) {
-      starts.push(parseFloat(match[1]));
-    }
-
-    let i = 0;
-    while ((match = endRegex.exec(output)) !== null) {
-      if (i < starts.length) {
-        silences.push({
-          start: starts[i],
-          end: parseFloat(match[1]),
-          duration: parseFloat(match[2]),
-        });
-        i++;
-      }
-    }
-
-    // Format output
-    const silenceList = silences.map((s, idx) =>
-      `[${idx + 1}] ${formatTimestamp(s.start)} - ${formatTimestamp(s.end)} (${s.duration.toFixed(1)}s)`
-    ).join("\n");
-
-    // Save to file if requested
-    if (args.output) {
-      const outputPath = resolve(context.workingDirectory, args.output as string);
-      await writeFile(
-        outputPath,
-        JSON.stringify({ source: absPath, silences }, null, 2),
-        "utf-8"
-      );
-    }
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: silences.length > 0
-        ? `Detected ${silences.length} silence periods:\n${silenceList}`
-        : "No silence periods detected.",
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to detect silence: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-const detectBeats: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const audioPath = args.audio as string;
-
-  try {
-    const absPath = resolve(context.workingDirectory, audioPath);
-
-    // Get duration
-    const totalDuration = await ffprobeDuration(absPath);
-
-    // Use interval-based detection (120 BPM default)
-    const estimatedBPM = 120;
-    const beatInterval = 60 / estimatedBPM;
-    const beats: number[] = [];
-
-    for (let t = 0; t < totalDuration; t += beatInterval) {
-      beats.push(t);
-    }
-
-    // Show first 20 beats
-    const beatList = beats.slice(0, 20).map((t, i) =>
-      `[${i + 1}] ${formatTimestamp(t)}`
-    ).join("\n");
-
-    // Save to file if requested
-    if (args.output) {
-      const outputPath = resolve(context.workingDirectory, args.output as string);
-      await writeFile(
-        outputPath,
-        JSON.stringify({ source: absPath, beatCount: beats.length, beats }, null, 2),
-        "utf-8"
-      );
-    }
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: `Detected ${beats.length} beats (${estimatedBPM} BPM):\n${beatList}${beats.length > 20 ? `\n... and ${beats.length - 20} more` : ""}`,
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to detect beats: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-const transcribe: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const audioPath = args.audio as string;
-  const language = args.language as string | undefined;
-
-  try {
-    const apiKey = await getApiKeyFromConfig("openai");
-    if (!apiKey) {
-      return {
-        toolCallId: "",
-        success: false,
-        output: "",
-        error: "OpenAI API key required for transcription. Set OPENAI_API_KEY or configure via 'vibe setup'.",
-      };
-    }
-
-    const absPath = resolve(context.workingDirectory, audioPath);
-    const audioBuffer = await readFile(absPath);
-
-    // Dynamic import of WhisperProvider
-    const { WhisperProvider } = await import("@vibeframe/ai-providers");
-    const whisper = new WhisperProvider();
-    await whisper.initialize({ apiKey });
-
-    const audioBlob = new Blob([audioBuffer]);
-    const result = await whisper.transcribe(audioBlob, language);
-
-    if (result.status === "failed") {
-      return {
-        toolCallId: "",
-        success: false,
-        output: "",
-        error: `Transcription failed: ${result.error}`,
-      };
-    }
-
-    // Format output
-    let output = `Transcript:\n${result.fullText}`;
-
-    if (result.segments && result.segments.length > 0) {
-      output += "\n\nSegments:";
-      for (const seg of result.segments.slice(0, 10)) {
-        output += `\n[${formatTimestamp(seg.startTime)} - ${formatTimestamp(seg.endTime)}] ${seg.text}`;
-      }
-      if (result.segments.length > 10) {
-        output += `\n... and ${result.segments.length - 10} more segments`;
-      }
-    }
-
-    // Save to file if requested
-    if (args.output) {
-      const outputPath = resolve(context.workingDirectory, args.output as string);
-      const { detectFormat, formatTranscript } = await import("../../utils/subtitle.js");
-      const format = detectFormat(args.output as string, undefined);
-      const content = formatTranscript(result, format);
-      await writeFile(outputPath, content, "utf-8");
-      output += `\n\nSaved to: ${args.output}`;
-    }
-
-    return {
-      toolCallId: "",
-      success: true,
-      output,
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to transcribe: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-// Media Manipulation Tools
-
-const compressDef: ToolDefinition = {
-  name: "media_compress",
-  description: "Compress a video or audio file using FFmpeg",
-  parameters: {
-    type: "object",
-    properties: {
-      input: {
-        type: "string",
-        description: "Input media file path",
-      },
-      output: {
-        type: "string",
-        description: "Output file path (default: input-compressed.ext)",
-      },
-      quality: {
-        type: "string",
-        description: "Quality preset: low, medium (default), high",
-        enum: ["low", "medium", "high"],
-      },
-      maxSize: {
-        type: "string",
-        description: "Target max file size (e.g., '10M', '100M')",
-      },
-    },
-    required: ["input"],
-  },
-};
-
-const convertDef: ToolDefinition = {
-  name: "media_convert",
-  description: "Convert media file to a different format using FFmpeg",
-  parameters: {
-    type: "object",
-    properties: {
-      input: {
-        type: "string",
-        description: "Input media file path",
-      },
-      output: {
-        type: "string",
-        description: "Output file path with desired extension (e.g., 'video.webm')",
-      },
-      codec: {
-        type: "string",
-        description: "Video codec (h264, h265, vp9, av1)",
-      },
-      audioCodec: {
-        type: "string",
-        description: "Audio codec (aac, mp3, opus)",
-      },
-    },
-    required: ["input", "output"],
-  },
-};
-
-const concatDef: ToolDefinition = {
-  name: "media_concat",
-  description: "Concatenate multiple media files into one using FFmpeg",
-  parameters: {
-    type: "object",
-    properties: {
-      inputs: {
-        type: "array",
-        items: { type: "string", description: "Input file path" },
-        description: "Array of input file paths to concatenate",
-      },
-      output: {
-        type: "string",
-        description: "Output file path",
-      },
-      reencode: {
-        type: "boolean",
-        description: "Re-encode files (slower but works with different codecs)",
-      },
-    },
-    required: ["inputs", "output"],
-  },
-};
-
-const compress: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const inputPath = args.input as string;
-  const quality = (args.quality as string) || "medium";
-  // maxSize is accepted as a parameter but not yet implemented
-  // const maxSize = args.maxSize as string | undefined;
-
-  try {
-    const absInput = resolve(context.workingDirectory, inputPath);
-    const ext = inputPath.split(".").pop() || "mp4";
-    const baseName = inputPath.replace(/\.[^/.]+$/, "");
-    const outputPath = args.output
-      ? resolve(context.workingDirectory, args.output as string)
-      : resolve(context.workingDirectory, `${baseName}-compressed.${ext}`);
-
-    // Quality presets (CRF values - lower = better quality, larger file)
-    const crfValues: Record<string, number> = {
-      low: 28,
-      medium: 23,
-      high: 18,
-    };
-    const crf = crfValues[quality] || 23;
-
-    await execSafe("ffmpeg", ["-i", absInput, "-c:v", "libx264", "-crf", String(crf), "-preset", "medium", "-c:a", "aac", "-b:a", "128k", outputPath, "-y"], { maxBuffer: 50 * 1024 * 1024 });
-
-    // Get file sizes for comparison
-    const inputStats = await readFile(absInput);
-    const outputStats = await readFile(outputPath);
-    const inputSize = inputStats.length;
-    const outputSize = outputStats.length;
-    const reduction = (((inputSize - outputSize) / inputSize) * 100).toFixed(1);
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: `Compressed: ${inputPath} → ${outputPath}\nSize: ${formatSize(inputSize)} → ${formatSize(outputSize)} (${reduction}% reduction)`,
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to compress: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-const convert: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const inputPath = args.input as string;
-  const outputPath = args.output as string;
-  const codec = args.codec as string | undefined;
-  const audioCodec = args.audioCodec as string | undefined;
-
-  try {
-    const absInput = resolve(context.workingDirectory, inputPath);
-    const absOutput = resolve(context.workingDirectory, outputPath);
-
-    // Build codec options
-    let videoCodecName = "copy";
-    let audioCodecName = "copy";
-
-    if (codec) {
-      const codecMap: Record<string, string> = {
-        h264: "libx264",
-        h265: "libx265",
-        vp9: "libvpx-vp9",
-        av1: "libaom-av1",
-      };
-      videoCodecName = codecMap[codec] || codec;
-    }
-
-    if (audioCodec) {
-      const audioCodecMap: Record<string, string> = {
-        aac: "aac",
-        mp3: "libmp3lame",
-        opus: "libopus",
-      };
-      audioCodecName = audioCodecMap[audioCodec] || audioCodec;
-    }
-
-    await execSafe("ffmpeg", ["-i", absInput, "-c:v", videoCodecName, "-c:a", audioCodecName, absOutput, "-y"], { maxBuffer: 50 * 1024 * 1024 });
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: `Converted: ${inputPath} → ${outputPath}`,
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to convert: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-const concat: ToolHandler = async (args, context): Promise<ToolResult> => {
-  const inputs = args.inputs as string[];
-  const outputPath = args.output as string;
-  const reencode = args.reencode as boolean || false;
-
-  if (!inputs || inputs.length < 2) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: "At least 2 input files required for concatenation",
-    };
-  }
-
-  try {
-    const absOutput = resolve(context.workingDirectory, outputPath);
-
-    if (reencode) {
-      // Re-encode method - works with different codecs
-      const ffmpegArgs: string[] = [];
-      for (const i of inputs) {
-        ffmpegArgs.push("-i", resolve(context.workingDirectory, i));
-      }
-      const filterComplex = inputs.map((_, i) => `[${i}:v][${i}:a]`).join("");
-      ffmpegArgs.push("-filter_complex", `${filterComplex}concat=n=${inputs.length}:v=1:a=1[outv][outa]`, "-map", "[outv]", "-map", "[outa]", absOutput, "-y");
-      await execSafe("ffmpeg", ffmpegArgs, { maxBuffer: 100 * 1024 * 1024 });
-    } else {
-      // Concat demuxer method - fast but requires same codec
-      const tempList = resolve(context.workingDirectory, `concat-list-${Date.now()}.txt`);
-      const listContent = inputs
-        .map((i) => `file '${resolve(context.workingDirectory, i)}'`)
-        .join("\n");
-      await writeFile(tempList, listContent, "utf-8");
-
-      await execSafe("ffmpeg", ["-f", "concat", "-safe", "0", "-i", tempList, "-c", "copy", absOutput, "-y"], { maxBuffer: 100 * 1024 * 1024 });
-
-      // Clean up temp file
-      const { unlink } = await import("node:fs/promises");
-      await unlink(tempList).catch(() => {});
-    }
-
-    return {
-      toolCallId: "",
-      success: true,
-      output: `Concatenated ${inputs.length} files → ${outputPath}`,
-    };
-  } catch (error) {
-    return {
-      toolCallId: "",
-      success: false,
-      output: "",
-      error: `Failed to concatenate: ${error instanceof Error ? error.message : String(error)}`,
-    };
-  }
-};
-
-// Helper function
-function formatSize(bytes: number): string {
-  if (bytes >= 1024 * 1024 * 1024) {
-    return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
-  }
-  if (bytes >= 1024 * 1024) {
-    return `${(bytes / (1024 * 1024)).toFixed(2)} MB`;
-  }
-  if (bytes >= 1024) {
-    return `${(bytes / 1024).toFixed(2)} KB`;
-  }
-  return `${bytes} B`;
-}
-
-// Registration function
-export function registerMediaTools(registry: ToolRegistry): void {
-  registry.register(mediaInfoDef, mediaInfo);
-  registry.register(detectScenesDef, detectScenes);
-  registry.register(detectSilenceDef, detectSilence);
-  registry.register(detectBeatsDef, detectBeats);
-  registry.register(transcribeDef, transcribe);
-  registry.register(compressDef, compress);
-  registry.register(convertDef, convert);
-  registry.register(concatDef, concat);
-}