@weirdfingers/baseboards 0.5.3 → 0.6.0

package/templates/api/src/boards/generators/implementations/fal/video/sync_lipsync_v2_pro.py

@@ -0,0 +1,155 @@
+"""
+fal.ai sync-lipsync v2 pro video generator.
+
+Generate high-quality realistic lipsync animations from audio while preserving
+unique details like natural teeth and unique facial features.
+
+Based on Fal AI's fal-ai/sync-lipsync/v2/pro model.
+See: https://fal.ai/models/fal-ai/sync-lipsync/v2/pro
+"""
+
+import os
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ....artifacts import AudioArtifact, VideoArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+
+
+class SyncLipsyncV2ProInput(BaseModel):
+    """Input schema for sync-lipsync v2 pro.
+
+    Artifact fields are automatically detected via type introspection
+    and resolved from generation IDs to artifact objects.
+    """
+
+    video: VideoArtifact = Field(description="Input video for lip-sync animation")
+    audio: AudioArtifact = Field(description="Audio to synchronize with the video")
+    sync_mode: Literal["cut_off", "loop", "bounce", "silence", "remap"] = Field(
+        default="cut_off",
+        description="Lipsync synchronization approach when media durations differ",
+    )
+
+
+class FalSyncLipsyncV2ProGenerator(BaseGenerator):
+    """Generator for high-quality realistic lip-synchronization animations."""
+
+    name = "fal-sync-lipsync-v2-pro"
+    description = "Fal: sync-lipsync v2 pro - High-quality lipsync preserving facial features"
+    artifact_type = "video"
+
+    def get_input_schema(self) -> type[SyncLipsyncV2ProInput]:
+        """Return the input schema for this generator."""
+        return SyncLipsyncV2ProInput
+
+    async def generate(
+        self, inputs: SyncLipsyncV2ProInput, context: GeneratorExecutionContext
+    ) -> GeneratorResult:
+        """Generate lip-synced video using fal.ai sync-lipsync/v2/pro."""
+        # Check for API key
+        if not os.getenv("FAL_KEY"):
+            raise ValueError("API configuration invalid. Missing FAL_KEY environment variable")
+
+        # Import fal_client
+        try:
+            import fal_client
+        except ImportError as e:
+            raise ImportError(
+                "fal.ai SDK is required for FalSyncLipsyncV2ProGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-fal]"
+            ) from e
+
+        # Upload video and audio artifacts to Fal's public storage
+        # Fal API requires publicly accessible URLs
+        from ..utils import upload_artifacts_to_fal
+
+        # Upload video and audio separately
+        video_urls = await upload_artifacts_to_fal([inputs.video], context)
+        audio_urls = await upload_artifacts_to_fal([inputs.audio], context)
+
+        # Prepare arguments for fal.ai API
+        arguments = {
+            "video_url": video_urls[0],
+            "audio_url": audio_urls[0],
+            "sync_mode": inputs.sync_mode,
+        }
+
+        # Submit async job
+        handler = await fal_client.submit_async(
+            "fal-ai/sync-lipsync/v2/pro",
+            arguments=arguments,
+        )
+
+        # Store external job ID
+        await context.set_external_job_id(handler.request_id)
+
+        # Stream progress updates
+        from .....progress.models import ProgressUpdate
+
+        event_count = 0
+        async for event in handler.iter_events(with_logs=True):
+            event_count += 1
+            # Sample every 3rd event to avoid spam
+            if event_count % 3 == 0:
+                # Extract logs if available
+                logs = getattr(event, "logs", None)
+                if logs:
+                    # Join log entries into a single message
+                    if isinstance(logs, list):
+                        message = " | ".join(str(log) for log in logs if log)
+                    else:
+                        message = str(logs)
+
+                    if message:
+                        await context.publish_progress(
+                            ProgressUpdate(
+                                job_id=handler.request_id,
+                                status="processing",
+                                progress=50.0,  # Approximate mid-point progress
+                                phase="processing",
+                                message=message,
+                            )
+                        )
+
+        # Get final result
+        result = await handler.get()
+
+        # Extract video from result
+        # fal.ai returns: {"video": {"url": "...", "content_type": "video/mp4", ...}}
+        video_data = result.get("video")
+
+        if not video_data:
+            raise ValueError("No video returned from fal.ai API")
+
+        video_url = video_data.get("url")
+        if not video_url:
+            raise ValueError("Video missing URL in fal.ai response")
+
+        # Extract format from content_type (e.g., "video/mp4" -> "mp4")
+        content_type = video_data.get("content_type", "video/mp4")
+        video_format = content_type.split("/")[-1] if "/" in content_type else "mp4"
+
+        # Store the video result
+        # Note: The API doesn't return width/height/duration/fps, so we use input values
+        # The actual dimensions will be the same as the input video
+        artifact = await context.store_video_result(
+            storage_url=video_url,
+            format=video_format,
+            width=inputs.video.width,
+            height=inputs.video.height,
+            duration=inputs.audio.duration,
+            fps=inputs.video.fps,
+            output_index=0,
+        )
+
+        return GeneratorResult(outputs=[artifact])
+
+    async def estimate_cost(self, inputs: SyncLipsyncV2ProInput) -> float:
+        """Estimate cost for sync-lipsync v2 pro generation in USD.
+
+        Pricing not specified in documentation, using estimate based on
+        typical video processing costs for pro models.
+        """
+        # Estimated cost per generation for pro model
+        return 0.10

package/templates/api/src/boards/generators/implementations/fal/video/veed_lipsync.py

@@ -0,0 +1,174 @@
+"""
+VEED Lipsync video generator.
+
+Generate realistic lipsync from any audio using VEED's latest model.
+This generator synchronizes lip movements in video with provided audio.
+
+Based on Fal AI's veed/lipsync model.
+See: https://fal.ai/models/veed/lipsync
+"""
+
+import os
+
+from pydantic import BaseModel, Field
+
+from ....artifacts import AudioArtifact, VideoArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+from ..utils import upload_artifacts_to_fal
+
+
+class VeedLipsyncInput(BaseModel):
+    """Input schema for VEED lipsync.
+
+    Artifact fields are automatically detected via type introspection
+    and resolved from generation IDs to artifact objects.
+    """
+
+    video_url: VideoArtifact = Field(description="Video to apply lip-sync animation to")
+    audio_url: AudioArtifact = Field(description="Audio to synchronize with the video")
+
+
+class FalVeedLipsyncGenerator(BaseGenerator):
+    """Generator for realistic lip-synchronization using VEED's model."""
+
+    name = "veed-lipsync"
+    description = "VEED: Lipsync - Generate realistic lipsync from any audio"
+    artifact_type = "video"
+
+    def get_input_schema(self) -> type[VeedLipsyncInput]:
+        """Return the input schema for this generator."""
+        return VeedLipsyncInput
+
+    async def generate(
+        self, inputs: VeedLipsyncInput, context: GeneratorExecutionContext
+    ) -> GeneratorResult:
+        """Generate lip-synced video using VEED lipsync."""
+        # Check for API key
+        if not os.getenv("FAL_KEY"):
+            raise ValueError("API configuration invalid. Missing FAL_KEY environment variable")
+
+        # Import fal_client
+        try:
+            import fal_client
+        except ImportError as e:
+            raise ImportError(
+                "fal.ai SDK is required for FalVeedLipsyncGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-fal]"
+            ) from e
+
+        # Upload video and audio artifacts to Fal's public storage
+        # Fal API requires publicly accessible URLs
+        # Upload video and audio separately
+        video_urls = await upload_artifacts_to_fal([inputs.video_url], context)
+        audio_urls = await upload_artifacts_to_fal([inputs.audio_url], context)
+
+        # Prepare arguments for fal.ai API
+        arguments = {
+            "video_url": video_urls[0],
+            "audio_url": audio_urls[0],
+        }
+
+        # Submit async job
+        handler = await fal_client.submit_async(
+            "veed/lipsync",
+            arguments=arguments,
+        )
+
+        # Store external job ID
+        await context.set_external_job_id(handler.request_id)
+
+        # Stream progress updates
+        from .....progress.models import ProgressUpdate
+
+        event_count = 0
+        async for event in handler.iter_events(with_logs=True):
+            event_count += 1
+            # Sample every 3rd event to avoid spamming progress updates
+            # This provides regular feedback without overwhelming the system
+            if event_count % 3 == 0:
+                # Extract logs if available
+                logs = getattr(event, "logs", None)
+                if logs:
+                    # Join log entries into a single message
+                    if isinstance(logs, list):
+                        message = " | ".join(str(log) for log in logs if log)
+                    else:
+                        message = str(logs)
+
+                    if message:
+                        await context.publish_progress(
+                            ProgressUpdate(
+                                job_id=handler.request_id,
+                                status="processing",
+                                # Using fixed 50% since API doesn't provide granular progress
+                                # This indicates processing is underway without false precision
+                                progress=50.0,
+                                phase="processing",
+                                message=message,
+                            )
+                        )
+
+        # Get final result
+        result = await handler.get()
+
+        # Extract video from result
+        # VEED API returns: {"video": {"url": "...", "content_type": "video/mp4", ...}}
+        video_data = result.get("video")
+
+        if not video_data:
+            raise ValueError(
+                f"No video returned from VEED API. Response structure: {list(result.keys())}"
+            )
+
+        video_url = video_data.get("url")
+        if not video_url:
+            raise ValueError(
+                f"Video missing URL in VEED response. Video data keys: {list(video_data.keys())}"
+            )
+
+        # Determine video format with fallback strategy:
+        # 1. Try to extract from URL extension (most reliable)
+        # 2. Parse content_type only if it's a video/* MIME type
+        # 3. Default to mp4 (most common format for this API)
+        video_format = "mp4"  # Default
+
+        # Try extracting extension from URL
+        if video_url:
+            url_parts = video_url.split(".")
+            if len(url_parts) > 1:
+                ext = url_parts[-1].split("?")[0].lower()  # Remove query params
+                if ext in ["mp4", "webm", "mov", "avi"]:  # Common video formats
+                    video_format = ext
+
+        # If no valid extension found, try content_type (only if it's video/*)
+        if video_format == "mp4":  # Still using default
+            content_type = video_data.get("content_type", "")
+            if content_type.startswith("video/"):
+                video_format = content_type.split("/")[-1]
+
+        # Store the video result
+        # Note: The API doesn't return width/height/duration/fps in documentation
+        # Using input video dimensions and audio duration
+        artifact = await context.store_video_result(
+            storage_url=video_url,
+            format=video_format,
+            width=inputs.video_url.width,
+            height=inputs.video_url.height,
+            duration=inputs.audio_url.duration,
+            fps=inputs.video_url.fps,
+            output_index=0,
+        )
+
+        return GeneratorResult(outputs=[artifact])
+
+    async def estimate_cost(self, inputs: VeedLipsyncInput) -> float:
+        """Estimate cost for VEED lipsync generation in USD.
+
+        Pricing not specified in documentation, using estimate based on
+        typical video lipsync processing costs.
+        """
+        # Fixed cost estimate of $0.05 per generation
+        # Based on typical AI video processing costs (~$0.03-0.07 per minute)
+        # This is a conservative estimate and should be updated when official
+        # pricing information becomes available from VEED/FAL
+        return 0.05

package/templates/api/src/boards/generators/implementations/fal/video/veo3.py

@@ -0,0 +1,194 @@
+"""
+Google Veo 3 text-to-video generator.
+
+The most advanced AI video generation model in the world by Google, capable of
+generating high-quality videos from text prompts with optional audio synthesis.
+
+Based on Fal AI's fal-ai/veo3 model.
+See: https://fal.ai/models/fal-ai/veo3
+"""
+
+import os
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+
+
+class Veo3Input(BaseModel):
+    """Input schema for Google Veo 3 text-to-video generation."""
+
+    prompt: str = Field(description="The text prompt describing the video you want to generate")
+    aspect_ratio: Literal["9:16", "16:9", "1:1"] = Field(
+        default="16:9",
+        description="Aspect ratio of the generated video. If 1:1, video will be outpainted",
+    )
+    duration: Literal["4s", "6s", "8s"] = Field(
+        default="8s",
+        description="Duration of the generated video",
+    )
+    resolution: Literal["720p", "1080p"] = Field(
+        default="720p",
+        description="Resolution of the generated video",
+    )
+    generate_audio: bool = Field(
+        default=True,
+        description="Whether to generate audio for the video. If false, 50% less credits used",
+    )
+    enhance_prompt: bool = Field(
+        default=True,
+        description="Whether to enhance video generation",
+    )
+    auto_fix: bool = Field(
+        default=True,
+        description="Automatically attempt to fix prompts that fail content policy",
+    )
+    seed: int | None = Field(
+        default=None,
+        description="Seed value for reproducible generation",
+    )
+    negative_prompt: str | None = Field(
+        default=None,
+        description="Guidance text to exclude from generation",
+    )
+
+
+class FalVeo3Generator(BaseGenerator):
+    """Generator for text-to-video using Google Veo 3."""
+
+    name = "fal-veo3"
+    description = "Fal: Veo 3 - Google's most advanced AI video generation model"
+    artifact_type = "video"
+
+    def get_input_schema(self) -> type[Veo3Input]:
+        """Return the input schema for this generator."""
+        return Veo3Input
+
+    async def generate(
+        self, inputs: Veo3Input, context: GeneratorExecutionContext
+    ) -> GeneratorResult:
+        """Generate video using fal.ai veo3."""
+        # Check for API key
+        if not os.getenv("FAL_KEY"):
+            raise ValueError("API configuration invalid. Missing FAL_KEY environment variable")
+
+        # Import fal_client
+        try:
+            import fal_client
+        except ImportError as e:
+            raise ImportError(
+                "fal.ai SDK is required for FalVeo3Generator. "
+                "Install with: pip install weirdfingers-boards[generators-fal]"
+            ) from e
+
+        # Prepare arguments for fal.ai API
+        arguments = {
+            "prompt": inputs.prompt,
+            "aspect_ratio": inputs.aspect_ratio,
+            "duration": inputs.duration,
+            "resolution": inputs.resolution,
+            "generate_audio": inputs.generate_audio,
+            "enhance_prompt": inputs.enhance_prompt,
+            "auto_fix": inputs.auto_fix,
+        }
+
+        # Add optional parameters if provided
+        if inputs.seed is not None:
+            arguments["seed"] = inputs.seed
+        if inputs.negative_prompt is not None:
+            arguments["negative_prompt"] = inputs.negative_prompt
+
+        # Submit async job
+        handler = await fal_client.submit_async(
+            "fal-ai/veo3",
+            arguments=arguments,
+        )
+
+        # Store external job ID
+        await context.set_external_job_id(handler.request_id)
+
+        # Stream progress updates
+        from .....progress.models import ProgressUpdate
+
+        event_count = 0
+        async for event in handler.iter_events(with_logs=True):
+            event_count += 1
+            # Sample every 3rd event to avoid spam
+            if event_count % 3 == 0:
+                # Extract logs if available
+                logs = getattr(event, "logs", None)
+                if logs:
+                    # Join log entries into a single message
+                    if isinstance(logs, list):
+                        message = " | ".join(str(log) for log in logs if log)
+                    else:
+                        message = str(logs)
+
+                    if message:
+                        await context.publish_progress(
+                            ProgressUpdate(
+                                job_id=handler.request_id,
+                                status="processing",
+                                progress=50.0,
+                                phase="processing",
+                                message=message,
+                            )
+                        )
+
+        # Get final result
+        result = await handler.get()
+
+        # Extract video from result
+        # Expected structure: {"video": {"url": "...", "content_type": "...", ...}}
+        video_data = result.get("video")
+        if not video_data:
+            raise ValueError("No video returned from fal.ai API")
+
+        video_url = video_data.get("url")
+        if not video_url:
+            raise ValueError("Video missing URL in fal.ai response")
+
+        # Determine video dimensions based on resolution and aspect ratio
+        if inputs.resolution == "720p":
+            if inputs.aspect_ratio == "16:9":
+                width, height = 1280, 720
+            elif inputs.aspect_ratio == "9:16":
+                width, height = 720, 1280
+            else:  # 1:1
+                width, height = 720, 720
+        else:  # 1080p
+            if inputs.aspect_ratio == "16:9":
+                width, height = 1920, 1080
+            elif inputs.aspect_ratio == "9:16":
+                width, height = 1080, 1920
+            else:  # 1:1
+                width, height = 1080, 1080
+
+        # Parse duration from "8s" format
+        duration_seconds = int(inputs.duration.rstrip("s"))
+
+        # Store video result
+        artifact = await context.store_video_result(
+            storage_url=video_url,
+            format="mp4",
+            width=width,
+            height=height,
+            duration=duration_seconds,
+            output_index=0,
+        )
+
+        return GeneratorResult(outputs=[artifact])
+
+    async def estimate_cost(self, inputs: Veo3Input) -> float:
+        """Estimate cost for this generation in USD.
+
+        Note: Pricing information not available in Fal documentation.
+        Using placeholder value that should be updated with actual pricing.
+        """
+        # TODO: Update with actual pricing from Fal when available
+        # Base cost, with 50% reduction if audio is disabled
+        base_cost = 0.15  # Placeholder estimate
+        if not inputs.generate_audio:
+            return base_cost * 0.5
+        return base_cost

package/templates/api/src/boards/generators/implementations/fal/video/veo31_first_last_frame_to_video.py

@@ -34,7 +34,7 @@ class Veo31FirstLastFrameToVideoInput(BaseModel):
     aspect_ratio: Literal["auto", "9:16", "16:9", "1:1"] = Field(
         default="auto",
         description=(
-            "Aspect ratio of the generated video. " "'auto' uses the aspect ratio from input images"
+            "Aspect ratio of the generated video. 'auto' uses the aspect ratio from input images"
         ),
     )
     resolution: Literal["720p", "1080p"] = Field(

package/templates/api/src/boards/generators/implementations/fal/video/wan_pro_image_to_video.py

@@ -0,0 +1,158 @@
+"""
+WAN-Pro 2.1 image-to-video generator.
+
+A premium image-to-video model that generates high-quality 1080p videos at 30fps
+with up to 6 seconds duration, converting static images into dynamic video content
+with exceptional motion diversity.
+
+Based on Fal AI's fal-ai/wan-pro/image-to-video model.
+See: https://fal.ai/models/fal-ai/wan-pro/image-to-video
+"""
+
+import os
+
+from pydantic import BaseModel, Field
+
+from ....artifacts import ImageArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+
+
+class WanProImageToVideoInput(BaseModel):
+    """Input schema for WAN-Pro 2.1 image-to-video generation.
+
+    Artifact fields (image) are automatically detected via type introspection
+    and resolved from generation IDs to ImageArtifact objects.
+    """
+
+    image: ImageArtifact = Field(description="The image to generate the video from")
+    prompt: str = Field(description="Text prompt describing the desired video content and motion")
+    seed: int | None = Field(
+        default=None,
+        description="Random seed for reproducibility. If not specified, a random seed will be used",
+    )
+    enable_safety_checker: bool = Field(
+        default=True,
+        description="Whether to enable the safety checker for content moderation",
+    )
+
+
+class FalWanProImageToVideoGenerator(BaseGenerator):
+    """Generator for creating videos from static images using WAN-Pro 2.1."""
+
+    name = "fal-wan-pro-image-to-video"
+    description = "Fal: WAN-Pro 2.1 - Generate high-quality 1080p videos from static images"
+    artifact_type = "video"
+
+    def get_input_schema(self) -> type[WanProImageToVideoInput]:
+        """Return the input schema for this generator."""
+        return WanProImageToVideoInput
+
+    async def generate(
+        self, inputs: WanProImageToVideoInput, context: GeneratorExecutionContext
+    ) -> GeneratorResult:
+        """Generate video using fal.ai wan-pro/image-to-video."""
+        # Check for API key
+        if not os.getenv("FAL_KEY"):
+            raise ValueError("API configuration invalid. Missing FAL_KEY environment variable")
+
+        # Import fal_client
+        try:
+            import fal_client
+        except ImportError as e:
+            raise ImportError(
+                "fal.ai SDK is required for FalWanProImageToVideoGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-fal]"
+            ) from e
+
+        # Upload image artifact to Fal's public storage
+        # Fal API requires publicly accessible URLs, but our storage_url might be:
+        # - Localhost URLs (not publicly accessible)
+        # - Private S3 buckets (not publicly accessible)
+        # So we upload to Fal's temporary storage first
+        from ..utils import upload_artifacts_to_fal
+
+        image_urls = await upload_artifacts_to_fal([inputs.image], context)
+
+        # Prepare arguments for fal.ai API
+        arguments = {
+            "image_url": image_urls[0],
+            "prompt": inputs.prompt,
+            "enable_safety_checker": inputs.enable_safety_checker,
+        }
+
+        # Only add seed if provided (allow API to use random if not specified)
+        if inputs.seed is not None:
+            arguments["seed"] = inputs.seed
+
+        # Submit async job
+        handler = await fal_client.submit_async(
+            "fal-ai/wan-pro/image-to-video",
+            arguments=arguments,
+        )
+
+        # Store external job ID
+        await context.set_external_job_id(handler.request_id)
+
+        # Stream progress updates
+        from .....progress.models import ProgressUpdate
+
+        event_count = 0
+        async for event in handler.iter_events(with_logs=True):
+            event_count += 1
+            # Sample every 3rd event to avoid spam
+            if event_count % 3 == 0:
+                # Extract logs if available
+                logs = getattr(event, "logs", None)
+                if logs:
+                    # Join log entries into a single message
+                    if isinstance(logs, list):
+                        message = " | ".join(str(log) for log in logs if log)
+                    else:
+                        message = str(logs)
+
+                    if message:
+                        await context.publish_progress(
+                            ProgressUpdate(
+                                job_id=handler.request_id,
+                                status="processing",
+                                progress=50.0,
+                                phase="processing",
+                                message=message,
+                            )
+                        )
+
+        # Get final result
+        result = await handler.get()
+
+        # Extract video from result
+        # Expected structure: {"video": {"url": "...", "content_type": "...", ...}}
+        video_data = result.get("video")
+        if not video_data:
+            raise ValueError("No video returned from fal.ai API")
+
+        video_url = video_data.get("url")
+        if not video_url:
+            raise ValueError("Video missing URL in fal.ai response")
+
+        # Store video result
+        # WAN-Pro generates 1080p videos at 30fps with up to 6 seconds duration
+        artifact = await context.store_video_result(
+            storage_url=video_url,
+            format="mp4",
+            width=1920,
+            height=1080,
+            duration=6,  # Maximum duration
+            fps=30,
+            output_index=0,
+        )
+
+        return GeneratorResult(outputs=[artifact])
+
+    async def estimate_cost(self, inputs: WanProImageToVideoInput) -> float:
+        """Estimate cost for this generation in USD.
+
+        Note: Pricing information not available in Fal documentation.
+        Using placeholder value that should be updated with actual pricing.
+        """
+        # TODO: Update with actual pricing from Fal when available
+        return 0.10  # Placeholder estimate for premium image-to-video generation