@_vrsen/openswarm 0.1.0

package/video_generation_agent/tools/CombineVideos.py

@@ -0,0 +1,113 @@
+"""Tool for combining multiple videos into a single video using ffmpeg."""
+
+import os
+import subprocess
+import tempfile
+
+from pydantic import Field, field_validator
+
+from agency_swarm import BaseTool, ToolOutputText
+
+from .utils.video_utils import get_videos_dir, resolve_ffmpeg_executable
+
+
+class CombineVideos(BaseTool):
+    """Combine multiple videos into a single video using instant cut transitions (ffmpeg).
+    
+    Videos are saved to: mnt/{product_name}/generated_videos/
+    """
+
+    product_name: str = Field(
+        ...,
+        description="Name of the product these videos are for (e.g., 'Acme_Widget_Pro', 'Green_Tea_Extract'). Used to locate and save videos in product-specific folders.",
+    )
+    video_names: list[str] = Field(
+        ...,
+        description="List of video file names (without extension) to combine in order.",
+    )
+    name: str = Field(
+        ...,
+        description="The name for the combined video file (without extension)",
+    )
+
+    @field_validator("video_names")
+    @classmethod
+    def _validate_video_names(cls, value: list[str]) -> list[str]:
+        if not value:
+            raise ValueError("video_names must not be empty")
+        if len(value) < 2:
+            raise ValueError("At least 2 videos are required for combining")
+        for name in value:
+            if not name.strip():
+                raise ValueError("Video names must not be empty")
+        return value
+
+    @field_validator("name")
+    @classmethod
+    def _name_not_blank(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("name must not be empty")
+        return value
+
+    def run(self) -> list:
+        """Combine videos using ffmpeg concat demuxer."""
+        videos_dir = get_videos_dir(self.product_name)
+
+        video_paths = []
+        for video_name in self.video_names:
+            video_path = os.path.join(videos_dir, f"{video_name}.mp4")
+            if not os.path.exists(video_path):
+                raise FileNotFoundError(
+                    f"Video file not found: {video_path}. "
+                    f"Make sure the video exists in the {videos_dir} directory."
+                )
+            video_paths.append(video_path)
+
+        output_path = os.path.join(videos_dir, f"{self.name}.mp4")
+
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False, encoding='utf-8') as f:
+            for path in video_paths:
+                abs_path = os.path.abspath(path).replace('\\', '/')
+                escaped_path = abs_path.replace("'", "'\\''")
+                f.write(f"file '{escaped_path}'\n")
+            concat_file = f.name
+
+        try:
+            ffmpeg_executable = resolve_ffmpeg_executable()
+            cmd = [
+                ffmpeg_executable,
+                '-f', 'concat',
+                '-safe', '0',
+                '-i', concat_file,
+                '-c', 'copy',  # copy streams without re-encoding
+                '-y',
+                output_path
+            ]
+            result = subprocess.run(cmd, capture_output=True, text=True)
+            if result.returncode != 0:
+                raise RuntimeError(f"FFmpeg concatenation failed: {result.stderr}")
+        finally:
+            try:
+                os.unlink(concat_file)
+            except Exception:
+                pass
+
+        lines = [f"Combined {len(self.video_names)} videos:"]
+        for i, name in enumerate(self.video_names, 1):
+            lines.append(f"  {i}. {name}.mp4")
+        lines.append(f"\nOutput: {self.name}.mp4")
+        lines.append(f"Path: {output_path}")
+
+        return [ToolOutputText(type="text", text="\n".join(lines))]
+
+
+if __name__ == "__main__":
+    # Example usage
+    tool = CombineVideos(
+        product_name="Test_Product",
+        video_names=["herbaluxe_01_hook_v2","herbaluxe_02_formula","herbaluxe_03_result_consistency_fix","herbaluxe_04_cta"],
+        name="x_combine_test",
+    )
+    result = tool.run()
+    print(result)
+

package/video_generation_agent/tools/EditAudio.py

@@ -0,0 +1,297 @@
+"""Tool for combining audio from one video with visuals from another video."""
+
+import os
+import subprocess
+import asyncio
+import cv2
+from typing import Optional
+from pathlib import Path
+
+from pydantic import Field, field_validator
+from PIL import Image
+
+from agency_swarm import BaseTool, ToolOutputText
+
+from .utils.video_utils import (
+    get_videos_dir,
+    ensure_not_blank,
+    generate_spritesheet,
+    extract_last_frame,
+    create_image_output,
+    resolve_ffmpeg_executable,
+)
+
+
+class EditAudio(BaseTool):
+    """
+    Combine audio from one video with visuals from another video.
+    
+    Useful for adding b-roll footage over narration, or replacing visuals while keeping original audio.
+    Supports padding to offset video timing relative to audio.
+    
+    Videos are saved to: mnt/{product_name}/generated_videos/
+    """
+    
+    product_name: str = Field(
+        ...,
+        description="Name of the product this video is for (e.g., 'Acme_Widget_Pro', 'Green_Tea_Extract'). Used to organize files into product-specific folders.",
+    )
+    audio_source: str = Field(
+        ...,
+        description=(
+            "The video to extract audio from. Can be: "
+            "1) Video name without extension (searches generated_videos folder), "
+            "2) Full local path to video file."
+        ),
+    )
+    video_source: str = Field(
+        ...,
+        description=(
+            "The video to use for visuals/b-roll. Can be: "
+            "1) Video name without extension (searches generated_videos folder), "
+            "2) Full local path to video file."
+        ),
+    )
+    output_name: str = Field(
+        ...,
+        description="The name for the combined video file (without extension)",
+    )
+    pad_seconds: float = Field(
+        default=0.0,
+        description=(
+            "Seconds to offset the video relative to audio. "
+            "Negative = video starts before audio (e.g., -2.0 = video plays 2s before audio), "
+            "Positive = video starts after audio (e.g., 2.0 = video plays 2s after audio starts), "
+            "Zero = video and audio start together (default: 0.0)"
+        ),
+    )
+    
+    @field_validator("audio_source")
+    @classmethod
+    def _audio_not_blank(cls, value: str) -> str:
+        return ensure_not_blank(value, "audio_source")
+    
+    @field_validator("video_source")
+    @classmethod
+    def _video_not_blank(cls, value: str) -> str:
+        return ensure_not_blank(value, "video_source")
+    
+    @field_validator("output_name")
+    @classmethod
+    def _output_not_blank(cls, value: str) -> str:
+        return ensure_not_blank(value, "output_name")
+    
+    async def run(self) -> list:
+        """Mix audio and video from two different sources."""
+        audio_path = self._resolve_video_path(self.audio_source)
+        video_path = self._resolve_video_path(self.video_source)
+
+        videos_dir = get_videos_dir(self.product_name)
+        output_path = os.path.join(videos_dir, f"{self.output_name}.mp4")
+
+        loop = asyncio.get_event_loop()
+        await loop.run_in_executor(None, self._mix_audio_video_blocking, audio_path, video_path, output_path)
+
+        output = []
+
+        spritesheet_path = os.path.join(videos_dir, f"{self.output_name}_spritesheet.jpg")
+        spritesheet = await loop.run_in_executor(None, generate_spritesheet, output_path, spritesheet_path)
+        if spritesheet:
+            output.extend(create_image_output(spritesheet_path, f"{self.output_name}_spritesheet.jpg"))
+
+        thumbnail_path = os.path.join(videos_dir, f"{self.output_name}_thumbnail.jpg")
+        thumbnail = await loop.run_in_executor(None, self._extract_first_frame, output_path, thumbnail_path)
+        if thumbnail:
+            output.extend(create_image_output(thumbnail_path, f"{self.output_name}_thumbnail.jpg"))
+
+        last_frame_path = os.path.join(videos_dir, f"{self.output_name}_last_frame.jpg")
+        last_frame = await loop.run_in_executor(None, extract_last_frame, output_path, last_frame_path)
+        if last_frame:
+            output.extend(create_image_output(last_frame_path, f"{self.output_name}_last_frame.jpg"))
+
+        pad_info = ""
+        if self.pad_seconds != 0.0:
+            if self.pad_seconds < 0:
+                pad_info = f"\nPadding: Video starts {abs(self.pad_seconds)}s before audio"
+            else:
+                pad_info = f"\nPadding: Video starts {self.pad_seconds}s after audio"
+        
+        output.append(ToolOutputText(
+            type="text",
+            text=f"Audio and video mixed successfully!\nSaved to: `{self.output_name}.mp4`\nPath: {output_path}{pad_info}"
+        ))
+        
+        return output
+    
+    def _resolve_video_path(self, video_ref: str) -> str:
+        """Resolve video reference to full path."""
+        # Try as full path first
+        path = Path(video_ref).expanduser().resolve()
+        
+        if path.exists():
+            return str(path)
+        
+        # Try as video name without extension in generated_videos
+        videos_dir = get_videos_dir(self.product_name)
+        
+        for ext in [".mp4", ".mov", ".avi", ".webm"]:
+            potential_path = os.path.join(videos_dir, f"{video_ref}{ext}")
+            if os.path.exists(potential_path):
+                return potential_path
+        
+        raise FileNotFoundError(
+            f"Video '{video_ref}' not found in {videos_dir}. "
+            f"Tried extensions: .mp4, .mov, .avi, .webm"
+        )
+    
+    def _mix_audio_video_blocking(self, audio_path: str, video_path: str, output_path: str) -> None:
+        """Mix audio and video using ffmpeg (blocking operation)."""
+        cap_audio = cv2.VideoCapture(audio_path)
+        fps_audio = cap_audio.get(cv2.CAP_PROP_FPS)
+        frames_audio = int(cap_audio.get(cv2.CAP_PROP_FRAME_COUNT))
+        audio_duration = frames_audio / fps_audio if fps_audio > 0 else 0
+        cap_audio.release()
+        
+        cap_video = cv2.VideoCapture(video_path)
+        fps_video = cap_video.get(cv2.CAP_PROP_FPS)
+        frames_video = int(cap_video.get(cv2.CAP_PROP_FRAME_COUNT))
+        video_duration = frames_video / fps_video if fps_video > 0 else 0
+        cap_video.release()
+
+        ffmpeg_executable = resolve_ffmpeg_executable()
+        
+        if self.pad_seconds == 0.0:
+            # Simple case: no padding, just combine audio and video
+            ffmpeg_cmd = [
+                ffmpeg_executable,
+                "-y",  # Overwrite output file
+                "-i", video_path,  # Input 0: video source
+                "-i", audio_path,  # Input 1: audio source
+                "-map", "0:v:0",  # Use video from input 0
+                "-map", "1:a:0",  # Use audio from input 1
+                "-c:v", "libx264",  # Video codec
+                "-preset", "medium",
+                "-crf", "23",
+                "-c:a", "aac",  # Audio codec
+                "-b:a", "128k",
+                "-movflags", "+faststart",
+                "-pix_fmt", "yuv420p",
+                "-t", str(video_duration),  # Use video duration as master timeline
+                output_path
+            ]
+        else:
+            # Complex case: apply padding offset
+            # Negative pad = delay audio (video starts first)
+            # Positive pad = delay video (audio starts first)
+            
+            if self.pad_seconds < 0:
+                # Video starts before audio - delay audio track
+                audio_delay = abs(self.pad_seconds)
+                ffmpeg_cmd = [
+                    ffmpeg_executable,
+                    "-y",
+                    "-i", video_path,
+                    "-i", audio_path,
+                    "-filter_complex",
+                    f"[1:a]adelay={int(audio_delay * 1000)}|{int(audio_delay * 1000)}[a]",  # Delay audio in milliseconds
+                    "-map", "0:v:0",
+                    "-map", "[a]",
+                    "-c:v", "libx264",
+                    "-preset", "medium",
+                    "-crf", "23",
+                    "-c:a", "aac",
+                    "-b:a", "128k",
+                    "-movflags", "+faststart",
+                    "-pix_fmt", "yuv420p",
+                    "-t", str(video_duration),  # Use video duration as master timeline
+                    output_path
+                ]
+            else:
+                # Audio starts before video - delay video track
+                video_delay = self.pad_seconds
+                ffmpeg_cmd = [
+                    ffmpeg_executable,
+                    "-y",
+                    "-i", video_path,
+                    "-i", audio_path,
+                    "-filter_complex",
+                    f"[0:v]setpts=PTS+{video_delay}/TB[v]",  # Delay video
+                    "-map", "[v]",
+                    "-map", "1:a:0",
+                    "-c:v", "libx264",
+                    "-preset", "medium",
+                    "-crf", "23",
+                    "-c:a", "aac",
+                    "-b:a", "128k",
+                    "-movflags", "+faststart",
+                    "-pix_fmt", "yuv420p",
+                    "-t", str(video_duration),  # Use video duration as master timeline
+                    output_path
+                ]
+        
+        try:
+            subprocess.run(
+                ffmpeg_cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True
+            )
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(
+                f"ffmpeg failed to mix audio and video. Error: {e.stderr}"
+            )
+        except FileNotFoundError:
+            raise RuntimeError(
+                "ffmpeg not found. Please install ffmpeg and ensure it's in your PATH. "
+                "Download from: https://ffmpeg.org/download.html"
+            )
+    
+    def _extract_first_frame(self, video_path: str, output_path: str) -> Optional[object]:
+        """Extract the first frame from video as thumbnail."""
+        cap = cv2.VideoCapture(video_path)
+        ret, frame = cap.read()
+        cap.release()
+        
+        if not ret:
+            return None
+        
+        # Convert BGR to RGB
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        thumbnail_image = Image.fromarray(frame_rgb)
+        thumbnail_image.save(output_path)
+        
+        return thumbnail_image
+
+
+if __name__ == "__main__":
+    # Check if test videos exist
+    test_dir = Path(__file__).parent.parent.parent / "mnt" / "Test_Product" / "generated_videos"
+    video1 = test_dir / "test_video.mp4"
+    video2 = test_dir / "test_video_trimmed_last2s.mp4"
+    
+    if not video1.exists() or not video2.exists():
+        print("Test videos not found. Skipping test.")
+        print(f"Expected: {video1}")
+        print(f"Expected: {video2}")
+    else:
+        # Example: Combine audio from one video with b-roll from another
+        # Audio is 4s, b-roll is 26.8s - b-roll continues after audio ends
+        # Video starts 5 seconds after audio (audio pre-roll)
+        tool = EditAudio(
+            product_name="Test_Product",
+            audio_source="test_video",  # Audio from this video (4s)
+            video_source="Ad2_3seg_AI_Employee_UGC_final",  # Visuals from this video (26.8s)
+            output_name="mixed_video",
+            pad_seconds=-3.0,  # Video starts 5 seconds after audio begins
+        )
+        
+        try:
+            result = asyncio.run(tool.run())
+            print("\nMix complete!")
+            for item in result:
+                if hasattr(item, 'text'):
+                    print(item.text)
+        except Exception as exc:
+            print(f"Audio/video mixing failed: {exc}")
+

package/video_generation_agent/tools/EditImage.py

@@ -0,0 +1,144 @@
+"""Tool for editing images using Google's Gemini 2.5 Flash Image model."""
+
+import asyncio
+import os
+from typing import Literal
+
+from google import genai
+from pydantic import Field, field_validator
+
+from agency_swarm import BaseTool
+
+from .utils.image_utils import (
+    get_images_dir,
+    MODEL_NAME,
+    load_image_by_name,
+    extract_image_from_response,
+    extract_usage_metadata,
+    process_variant_result,
+    split_results_and_usage,
+    run_parallel_variants,
+    compress_image_for_base64,
+)
+
+
+class EditImage(BaseTool):
+    """Edit existing images using Google's Gemini 2.5 Flash Image (Nano Banana) model.
+    
+    Images are saved to: mnt/{product_name}/generated_images/
+    """
+
+    product_name: str = Field(
+        ...,
+        description="Name of the product this image is for (e.g., 'Acme_Widget_Pro', 'Green_Tea_Extract'). Used to organize files into product-specific folders.",
+    )
+    input_image_name: str = Field(
+        ...,
+        description="Name of the existing image file to edit (without extension).",
+    )
+    edit_prompt: str = Field(
+        ...,
+        description="Text prompt describing the edits to make to the image",
+    )
+    output_image_name: str = Field(
+        ...,
+        description="The name for the generated edited image file (without extension)",
+    )
+    num_variants: int = Field(
+        default=1,
+        description="Number of image variants to generate (1-4, default is 1)",
+    )
+    aspect_ratio: Literal["1:1", "2:3", "3:2", "3:4", "4:3", "4:5", "5:4", "9:16", "16:9", "21:9"] = Field(
+        default="1:1",
+        description="The aspect ratio of the generated image (default is 1:1)",
+    )
+
+    @field_validator("input_image_name")
+    @classmethod
+    def _input_name_not_blank(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("input_image_name must not be empty")
+        return value
+
+    @field_validator("edit_prompt")
+    @classmethod
+    def _prompt_not_blank(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("edit_prompt must not be empty")
+        return value
+
+    @field_validator("output_image_name")
+    @classmethod
+    def _output_name_not_blank(cls, value: str) -> str:
+        if not value.strip():
+            raise ValueError("output_image_name must not be empty")
+        return value
+
+    @field_validator("num_variants")
+    @classmethod
+    def _validate_num_variants(cls, value: int) -> int:
+        if value < 1 or value > 4:
+            raise ValueError("num_variants must be between 1 and 4")
+        return value
+
+    async def run(self) -> list:
+        """Edit an image using the Gemini API."""
+        api_key = os.getenv("GOOGLE_API_KEY")
+        if not api_key:
+            raise ValueError("GOOGLE_API_KEY is not set. Add it to your .env to use image editing.")
+
+        client = genai.Client(api_key=api_key)
+        images_dir = get_images_dir(self.product_name)
+
+        image, _image_path, load_error = load_image_by_name(self.input_image_name, images_dir)
+        if load_error:
+            raise FileNotFoundError(load_error)
+
+        def edit_single_variant(variant_num: int):
+            try:
+                response = client.models.generate_content(
+                    model=MODEL_NAME,
+                    contents=[self.edit_prompt, image],
+                    config=genai.types.GenerateContentConfig(
+                        image_config=genai.types.ImageConfig(aspect_ratio=self.aspect_ratio),
+                    ),
+                )
+                usage_metadata = extract_usage_metadata(response)
+                edited_image, _text = extract_image_from_response(response)
+                if edited_image is None:
+                    return None
+                result = process_variant_result(
+                    variant_num,
+                    edited_image,
+                    self.output_image_name,
+                    self.num_variants,
+                    compress_image_for_base64,
+                    images_dir,
+                )
+                result["prompt_tokens"] = float(usage_metadata.get("prompt_token_count") or 0)
+                result["candidate_tokens"] = float(usage_metadata.get("candidates_token_count") or 0)
+                return result
+            except Exception:
+                return None
+
+        raw_results = await run_parallel_variants(edit_single_variant, self.num_variants)
+        if not raw_results:
+            raise RuntimeError("No variants were successfully generated")
+
+        results, _usage = split_results_and_usage(raw_results)
+        return results
+
+if __name__ == "__main__":
+    # Example usage with Google Gemini 2.5 Flash Image
+    tool = EditImage(
+        product_name="Test_Product",
+        input_image_name="logo_image_variant_1",
+        edit_prompt="Change the logo color from red to blue",
+        output_image_name="logo_image_edited",
+        num_variants=1,
+    )
+    try:
+        result = asyncio.run(tool.run())
+        print(result)
+    except Exception as exc:
+        print(f"Image editing failed: {exc}")