@weirdfingers/baseboards 0.2.1 → 0.4.0

package/templates/api/src/boards/generators/implementations/fal/video/sync_lipsync_v2.py

@@ -0,0 +1,167 @@
+"""
+fal.ai sync-lipsync v2 video generator.
+
+Generates realistic lip-synchronization animations from audio and video inputs
+using fal.ai's sync-lipsync/v2 model. Supports advanced audio/video duration
+mismatch handling with multiple sync modes.
+
+Based on Fal AI's fal-ai/sync-lipsync/v2 model.
+See: https://fal.ai/models/fal-ai/sync-lipsync/v2
+"""
+
+import os
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ....artifacts import AudioArtifact, VideoArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+
+
+class SyncLipsyncV2Input(BaseModel):
+    """Input schema for sync-lipsync v2.
+
+    Artifact fields are automatically detected via type introspection
+    and resolved from generation IDs to artifact objects.
+    """
+
+    video: VideoArtifact = Field(description="Input video for lip-sync animation")
+    audio: AudioArtifact = Field(description="Audio to synchronize with the video")
+    model: Literal["lipsync-2", "lipsync-2-pro"] = Field(
+        default="lipsync-2",
+        description="Model selection; pro version costs ~1.67x more",
+    )
+    sync_mode: Literal["cut_off", "loop", "bounce", "silence", "remap"] = Field(
+        default="cut_off",
+        description="Handling method when audio/video durations mismatch",
+    )
+
+
+class FalSyncLipsyncV2Generator(BaseGenerator):
+    """Generator for realistic lip-synchronization animations."""
+
+    name = "fal-sync-lipsync-v2"
+    description = "Fal: sync-lipsync v2 - Realistic lip-sync animation with audio"
+    artifact_type = "video"
+
+    def get_input_schema(self) -> type[SyncLipsyncV2Input]:
+        """Return the input schema for this generator."""
+        return SyncLipsyncV2Input
+
+    async def generate(
+        self, inputs: SyncLipsyncV2Input, context: GeneratorExecutionContext
+    ) -> GeneratorResult:
+        """Generate lip-synced video using fal.ai sync-lipsync/v2."""
+        # Check for API key
+        if not os.getenv("FAL_KEY"):
+            raise ValueError("API configuration invalid. Missing FAL_KEY environment variable")
+
+        # Import fal_client
+        try:
+            import fal_client
+        except ImportError as e:
+            raise ImportError(
+                "fal.ai SDK is required for FalSyncLipsyncV2Generator. "
+                "Install with: pip install weirdfingers-boards[generators-fal]"
+            ) from e
+
+        # Upload video and audio artifacts to Fal's public storage
+        # Fal API requires publicly accessible URLs
+        from ..utils import upload_artifacts_to_fal
+
+        # Upload video and audio separately
+        video_urls = await upload_artifacts_to_fal([inputs.video], context)
+        audio_urls = await upload_artifacts_to_fal([inputs.audio], context)
+
+        # Prepare arguments for fal.ai API
+        arguments = {
+            "video_url": video_urls[0],
+            "audio_url": audio_urls[0],
+            "model": inputs.model,
+            "sync_mode": inputs.sync_mode,
+        }
+
+        # Submit async job
+        handler = await fal_client.submit_async(
+            "fal-ai/sync-lipsync/v2",
+            arguments=arguments,
+        )
+
+        # Store external job ID
+        await context.set_external_job_id(handler.request_id)
+
+        # Stream progress updates
+        from .....progress.models import ProgressUpdate
+
+        event_count = 0
+        async for event in handler.iter_events(with_logs=True):
+            event_count += 1
+            # Sample every 3rd event to avoid spam
+            if event_count % 3 == 0:
+                # Extract logs if available
+                logs = getattr(event, "logs", None)
+                if logs:
+                    # Join log entries into a single message
+                    if isinstance(logs, list):
+                        message = " | ".join(str(log) for log in logs if log)
+                    else:
+                        message = str(logs)
+
+                    if message:
+                        await context.publish_progress(
+                            ProgressUpdate(
+                                job_id=handler.request_id,
+                                status="processing",
+                                progress=50.0,  # Approximate mid-point progress
+                                phase="processing",
+                                message=message,
+                            )
+                        )
+
+        # Get final result
+        result = await handler.get()
+
+        # Extract video from result
+        # fal.ai returns: {"video": {"url": "...", "content_type": "video/mp4", ...}}
+        video_data = result.get("video")
+
+        if not video_data:
+            raise ValueError("No video returned from fal.ai API")
+
+        video_url = video_data.get("url")
+        if not video_url:
+            raise ValueError("Video missing URL in fal.ai response")
+
+        # Extract format from content_type (e.g., "video/mp4" -> "mp4")
+        content_type = video_data.get("content_type", "video/mp4")
+        video_format = content_type.split("/")[-1] if "/" in content_type else "mp4"
+
+        # Store the video result
+        # Note: The API doesn't return width/height/duration/fps, so we use defaults
+        # The actual dimensions will be the same as the input video
+        artifact = await context.store_video_result(
+            storage_url=video_url,
+            format=video_format,
+            width=inputs.video.width,
+            height=inputs.video.height,
+            duration=inputs.audio.duration,
+            fps=inputs.video.fps,
+            output_index=0,
+        )
+
+        return GeneratorResult(outputs=[artifact])
+
+    async def estimate_cost(self, inputs: SyncLipsyncV2Input) -> float:
+        """Estimate cost for sync-lipsync v2 generation in USD.
+
+        Pricing not specified in documentation, using estimate based on
+        typical video processing costs. Pro model costs ~1.67x more.
+        """
+        # Base cost estimate per generation
+        base_cost = 0.05
+
+        # Pro model multiplier
+        if inputs.model == "lipsync-2-pro":
+            return base_cost * 1.67
+
+        return base_cost

package/templates/api/src/boards/generators/implementations/fal/video/veo31_first_last_frame_to_video.py

@@ -0,0 +1,180 @@
+"""
+Google Veo 3.1 first-last frame to video generator.
+
+Generates videos by interpolating between first and last frame images using
+Google's Veo 3.1 technology via fal.ai.
+
+Based on Fal AI's fal-ai/veo3.1/first-last-frame-to-video model.
+See: https://fal.ai/models/fal-ai/veo3.1/first-last-frame-to-video
+"""
+
+import os
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+from ....artifacts import ImageArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+
+
+class Veo31FirstLastFrameToVideoInput(BaseModel):
+    """Input schema for Veo 3.1 first-last frame to video generation.
+
+    Artifact fields (first_frame, last_frame) are automatically detected via type
+    introspection and resolved from generation IDs to ImageArtifact objects.
+    """
+
+    first_frame: ImageArtifact = Field(description="The first frame of the video (input image)")
+    last_frame: ImageArtifact = Field(description="The last frame of the video (input image)")
+    prompt: str = Field(description="Text prompt describing the desired video content and motion")
+    duration: Literal["8s"] = Field(
+        default="8s",
+        description="Duration of the generated video in seconds (currently only 8s is supported)",
+    )
+    aspect_ratio: Literal["auto", "9:16", "16:9", "1:1"] = Field(
+        default="auto",
+        description=(
+            "Aspect ratio of the generated video. " "'auto' uses the aspect ratio from input images"
+        ),
+    )
+    resolution: Literal["720p", "1080p"] = Field(
+        default="720p",
+        description="Resolution of the generated video",
+    )
+    generate_audio: bool = Field(
+        default=True,
+        description="Whether to generate audio for the video. Disabling uses 50% fewer credits",
+    )
+
+
+class FalVeo31FirstLastFrameToVideoGenerator(BaseGenerator):
+    """Generator for creating videos from first and last frame images using Google Veo 3.1."""
+
+    name = "fal-veo31-first-last-frame-to-video"
+    description = "Fal: Veo 3.1 - Generate videos by interpolating between first and last frames"
+    artifact_type = "video"
+
+    def get_input_schema(self) -> type[Veo31FirstLastFrameToVideoInput]:
+        """Return the input schema for this generator."""
+        return Veo31FirstLastFrameToVideoInput
+
+    async def generate(
+        self, inputs: Veo31FirstLastFrameToVideoInput, context: GeneratorExecutionContext
+    ) -> GeneratorResult:
+        """Generate video using fal.ai veo3.1/first-last-frame-to-video."""
+        # Check for API key
+        if not os.getenv("FAL_KEY"):
+            raise ValueError("API configuration invalid. Missing FAL_KEY environment variable")
+
+        # Import fal_client
+        try:
+            import fal_client
+        except ImportError as e:
+            raise ImportError(
+                "fal.ai SDK is required for FalVeo31FirstLastFrameToVideoGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-fal]"
+            ) from e
+
+        # Upload image artifacts to Fal's public storage
+        # Fal API requires publicly accessible URLs, but our storage_url might be:
+        # - Localhost URLs (not publicly accessible)
+        # - Private S3 buckets (not publicly accessible)
+        # So we upload to Fal's temporary storage first
+        from ..utils import upload_artifacts_to_fal
+
+        first_frame_urls = await upload_artifacts_to_fal([inputs.first_frame], context)
+        last_frame_urls = await upload_artifacts_to_fal([inputs.last_frame], context)
+
+        # Prepare arguments for fal.ai API
+        arguments = {
+            "first_frame_url": first_frame_urls[0],
+            "last_frame_url": last_frame_urls[0],
+            "prompt": inputs.prompt,
+            "duration": inputs.duration,
+            "aspect_ratio": inputs.aspect_ratio,
+            "resolution": inputs.resolution,
+            "generate_audio": inputs.generate_audio,
+        }
+
+        # Submit async job
+        handler = await fal_client.submit_async(
+            "fal-ai/veo3.1/first-last-frame-to-video",
+            arguments=arguments,
+        )
+
+        # Store external job ID
+        await context.set_external_job_id(handler.request_id)
+
+        # Stream progress updates
+        from .....progress.models import ProgressUpdate
+
+        event_count = 0
+        async for event in handler.iter_events(with_logs=True):
+            event_count += 1
+            # Sample every 3rd event to avoid spam
+            if event_count % 3 == 0:
+                # Extract logs if available
+                logs = getattr(event, "logs", None)
+                if logs:
+                    # Join log entries into a single message
+                    if isinstance(logs, list):
+                        message = " | ".join(str(log) for log in logs if log)
+                    else:
+                        message = str(logs)
+
+                    if message:
+                        await context.publish_progress(
+                            ProgressUpdate(
+                                job_id=handler.request_id,
+                                status="processing",
+                                progress=50.0,
+                                phase="processing",
+                                message=message,
+                            )
+                        )
+
+        # Get final result
+        result = await handler.get()
+
+        # Extract video from result
+        # Expected structure: {"video": {"url": "...", "content_type": "...", ...}}
+        video_data = result.get("video")
+        if not video_data:
+            raise ValueError("No video returned from fal.ai API")
+
+        video_url = video_data.get("url")
+        if not video_url:
+            raise ValueError("Video missing URL in fal.ai response")
+
+        # Store video result
+        # Note: Fal API doesn't provide video dimensions/duration in the response,
+        # so we'll use defaults based on input parameters
+        width = 1280 if inputs.resolution == "720p" else 1920
+        height = 720 if inputs.resolution == "720p" else 1080
+
+        # Parse duration from "8s" format
+        duration_seconds = int(inputs.duration.rstrip("s"))
+
+        artifact = await context.store_video_result(
+            storage_url=video_url,
+            format="mp4",
+            width=width,
+            height=height,
+            duration=duration_seconds,
+            output_index=0,
+        )
+
+        return GeneratorResult(outputs=[artifact])
+
+    async def estimate_cost(self, inputs: Veo31FirstLastFrameToVideoInput) -> float:
+        """Estimate cost for this generation in USD.
+
+        Note: Pricing information not available in Fal documentation.
+        Using placeholder value that should be updated with actual pricing.
+        """
+        # TODO: Update with actual pricing from Fal when available
+        # Base cost, with 50% reduction if audio is disabled
+        base_cost = 0.15  # Placeholder estimate
+        if not inputs.generate_audio:
+            return base_cost * 0.5
+        return base_cost

package/templates/api/src/boards/generators/implementations/openai/__init__.py

@@ -0,0 +1 @@
+"""OpenAI provider generators."""

package/templates/api/src/boards/generators/implementations/openai/audio/__init__.py

@@ -0,0 +1 @@
+"""OpenAI audio generators."""

package/templates/api/src/boards/generators/implementations/{audio → openai/audio}/whisper.py

@@ -6,8 +6,8 @@ Demonstrates audio processing generator that outputs text.
 
 from pydantic import BaseModel, Field
 
-from ...artifacts import AudioArtifact
-from ...base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+from ....artifacts import AudioArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
 
 
 class WhisperInput(BaseModel):
@@ -18,12 +18,12 @@ class WhisperInput(BaseModel):
     prompt: str = Field(default="", description="Optional prompt to guide transcription")
 
 
-class WhisperGenerator(BaseGenerator):
+class OpenAIWhisperGenerator(BaseGenerator):
     """Whisper speech-to-text transcription using OpenAI API."""
 
-    name = "whisper"
+    name = "openai-whisper"
     artifact_type = "text"
-    description = "OpenAI Whisper - speech-to-text transcription"
+    description = "OpenAI: Whisper - speech-to-text transcription"
 
     def get_input_schema(self) -> type[WhisperInput]:
         return WhisperInput
@@ -35,7 +35,10 @@ class WhisperGenerator(BaseGenerator):
         try:
             from openai import AsyncOpenAI
         except ImportError as e:
-            raise ValueError("Required dependencies not available") from e
+            raise ImportError(
+                "OpenAI SDK is required for WhisperGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-openai]"
+            ) from e
 
         client = AsyncOpenAI()
 

package/templates/api/src/boards/generators/implementations/openai/image/__init__.py

@@ -0,0 +1 @@
+"""OpenAI image generators."""

package/templates/api/src/boards/generators/implementations/{image → openai/image}/dalle3.py

@@ -8,7 +8,7 @@ import os
 
 from pydantic import BaseModel, Field
 
-from ...base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
 
 
 class DallE3Input(BaseModel):
@@ -24,12 +24,12 @@ class DallE3Input(BaseModel):
     style: str = Field(default="vivid", description="Image style", pattern="^(vivid|natural)$")
 
 
-class DallE3Generator(BaseGenerator):
+class OpenAIDallE3Generator(BaseGenerator):
     """DALL-E 3 image generator using OpenAI API."""
 
-    name = "dall-e-3"
+    name = "openai-dall-e-3"
     artifact_type = "image"
-    description = "OpenAI's DALL-E 3 - advanced text-to-image generation"
+    description = "OpenAI: DALL-E 3 - advanced text-to-image generation"
 
     def get_input_schema(self) -> type[DallE3Input]:
         return DallE3Input
@@ -46,7 +46,10 @@ class DallE3Generator(BaseGenerator):
         try:
             from openai import AsyncOpenAI
         except ImportError as e:
-            raise ValueError("Required dependencies not available") from e
+            raise ImportError(
+                "OpenAI SDK is required for DallE3Generator. "
+                "Install with: pip install weirdfingers-boards[generators-openai]"
+            ) from e
 
         client = AsyncOpenAI()
 

package/templates/api/src/boards/generators/implementations/replicate/__init__.py

@@ -0,0 +1 @@
+"""Replicate provider generators."""

package/templates/api/src/boards/generators/implementations/replicate/image/__init__.py

@@ -0,0 +1 @@
+"""Replicate image generators."""

package/templates/api/src/boards/generators/implementations/{image → replicate/image}/flux_pro.py

@@ -12,7 +12,7 @@ from collections.abc import AsyncIterator
 
 from pydantic import BaseModel, Field
 
-from ...base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
 
 
 class FluxProInput(BaseModel):
@@ -27,12 +27,12 @@ class FluxProInput(BaseModel):
     safety_tolerance: int = Field(default=2, ge=1, le=5, description="Safety tolerance level (1-5)")
 
 
-class FluxProGenerator(BaseGenerator):
+class ReplicateFluxProGenerator(BaseGenerator):
     """FLUX.1.1 Pro image generator using Replicate."""
 
-    name = "flux-pro"
+    name = "replicate-flux-pro"
     artifact_type = "image"
-    description = "FLUX.1.1 [pro] by Black Forest Labs - high-quality image generation"
+    description = "Replicate: FLUX.1.1 [pro] by Black Forest Labs - high-quality image generation"
 
     def get_input_schema(self) -> type[FluxProInput]:
         return FluxProInput
@@ -50,7 +50,10 @@ class FluxProGenerator(BaseGenerator):
             import replicate
             from replicate.helpers import FileOutput
         except ImportError as e:
-            raise ValueError("Required dependencies not available") from e
+            raise ImportError(
+                "Replicate SDK is required for FluxProGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-replicate]"
+            ) from e
 
         # Use Replicate SDK directly
         prediction: FileOutput | AsyncIterator[FileOutput] = await replicate.async_run(

package/templates/api/src/boards/generators/implementations/replicate/video/__init__.py

@@ -0,0 +1 @@
+"""Replicate video generators."""

package/templates/api/src/boards/generators/implementations/{video → replicate/video}/lipsync.py

@@ -7,8 +7,8 @@ with automatic artifact resolution.
 
 from pydantic import BaseModel, Field
 
-from ...artifacts import AudioArtifact, VideoArtifact
-from ...base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
+from ....artifacts import AudioArtifact, VideoArtifact
+from ....base import BaseGenerator, GeneratorExecutionContext, GeneratorResult
 
 
 class LipsyncInput(BaseModel):
@@ -19,12 +19,12 @@ class LipsyncInput(BaseModel):
     prompt: str | None = Field(None, description="Optional prompt for generation")
 
 
-class LipsyncGenerator(BaseGenerator):
+class ReplicateLipsyncGenerator(BaseGenerator):
     """Lipsync generator that syncs lips in video to audio."""
 
-    name = "lipsync"
+    name = "replicate-lipsync"
     artifact_type = "video"
-    description = "Sync lips in video to match audio track"
+    description = "Replicate: Sync lips in video to match audio track"
 
     def get_input_schema(self) -> type[LipsyncInput]:
         return LipsyncInput
@@ -37,7 +37,10 @@ class LipsyncGenerator(BaseGenerator):
         try:
             import replicate  # type: ignore
         except ImportError as e:
-            raise ValueError("Required dependencies not available") from e
+            raise ImportError(
+                "Replicate SDK is required for LipsyncGenerator. "
+                "Install with: pip install weirdfingers-boards[generators-replicate]"
+            ) from e
 
         # Resolve artifacts via context
         audio_file = await context.resolve_artifact(inputs.audio_source)