@weirdfingers/baseboards 0.2.1 → 0.4.0

package/templates/api/src/boards/generators/resolution.py

@@ -2,6 +2,7 @@
 Artifact resolution utilities for converting Generation references to actual files.
 """
 
+import base64
 import os
 import tempfile
 import uuid
@@ -153,6 +154,59 @@ def _get_file_extension(
     return format_ext
 
 
+def _decode_data_url(data_url: str) -> bytes:
+    """
+    Decode a data URL to bytes.
+
+    Supports data URLs in the format: data:[<mediatype>][;base64],<data>
+
+    Args:
+        data_url: Data URL string (e.g., "data:image/png;base64,iVBORw0KGgo...")
+
+    Returns:
+        bytes: Decoded content
+
+    Raises:
+        ValueError: If data URL is malformed or empty
+    """
+    if not data_url.startswith("data:"):
+        raise ValueError("Invalid data URL: must start with 'data:'")
+
+    # Split off the "data:" prefix
+    try:
+        # Format: data:[<mediatype>][;base64],<data>
+        header, data = data_url[5:].split(",", 1)
+    except ValueError as e:
+        raise ValueError("Invalid data URL format: missing comma separator") from e
+
+    if not data:
+        raise ValueError("Data URL contains no data after comma")
+
+    # Check if base64 encoded
+    is_base64 = ";base64" in header
+
+    if is_base64:
+        try:
+            decoded = base64.b64decode(data)
+        except Exception as e:
+            raise ValueError(f"Failed to decode base64 data: {e}") from e
+    else:
+        # URL-encoded data (rare for binary content)
+        from urllib.parse import unquote
+
+        decoded = unquote(data).encode("utf-8")
+
+    if len(decoded) == 0:
+        raise ValueError("Decoded data URL is empty")
+
+    logger.info(
+        "Successfully decoded data URL",
+        size_bytes=len(decoded),
+        is_base64=is_base64,
+    )
+    return decoded
+
+
 async def download_from_url(url: str) -> bytes:
     """
     Download content from a URL (typically a provider's temporary URL).
@@ -160,20 +214,26 @@ async def download_from_url(url: str) -> bytes:
     This is used to download generated content from providers like Replicate, OpenAI, etc.
     before uploading to our permanent storage.
 
+    Supports both HTTP(S) URLs and data URLs (data:mime/type;base64,...)
+
     Note: For very large files, consider using streaming downloads directly to storage
     instead of loading into memory.
 
     Args:
-        url: URL to download from
+        url: URL to download from (HTTP(S) or data URL)
 
     Returns:
         bytes: Downloaded content
 
     Raises:
         httpx.HTTPError: If download fails
-        ValueError: If downloaded content is empty
+        ValueError: If downloaded content is empty or data URL is malformed
     """
-    logger.debug("Downloading content from URL", url=url)
+    logger.debug("Downloading content from URL", url=url[:50])
+
+    # Check if this is a data URL
+    if url.startswith("data:"):
+        return _decode_data_url(url)
 
     # Stream download to avoid loading entire file into memory at once
     async with httpx.AsyncClient(timeout=60.0) as client:
@@ -244,8 +304,8 @@ async def store_image_result(
     board_id: str,
     storage_url: str,
     format: str,
-    width: int,
-    height: int,
+    width: int | None = None,
+    height: int | None = None,
 ) -> ImageArtifact:
     """
     Store an image result by downloading from provider URL and uploading to storage.
@@ -257,8 +317,8 @@ async def store_image_result(
         board_id: Board ID for organization
         storage_url: Provider's temporary URL to download from
         format: Image format (png, jpg, etc.)
-        width: Image width in pixels
-        height: Image height in pixels
+        width: Image width in pixels (optional)
+        height: Image height in pixels (optional)
 
     Returns:
         ImageArtifact with permanent storage URL
@@ -270,7 +330,7 @@ async def store_image_result(
     logger.info(
         "Storing image result",
         generation_id=generation_id,
-        provider_url=storage_url,
+        provider_url=storage_url[:50],
         format=format,
     )
 
@@ -294,7 +354,7 @@ async def store_image_result(
         "Image stored successfully",
         generation_id=generation_id,
         storage_key=artifact_ref.storage_key,
-        storage_url=artifact_ref.storage_url,
+        storage_url=artifact_ref.storage_url[:50],
     )
 
     # Return artifact with our permanent storage URL
@@ -314,8 +374,8 @@ async def store_video_result(
     board_id: str,
     storage_url: str,
     format: str,
-    width: int,
-    height: int,
+    width: int | None = None,
+    height: int | None = None,
     duration: float | None = None,
     fps: float | None = None,
 ) -> VideoArtifact:
@@ -329,8 +389,8 @@ async def store_video_result(
         board_id: Board ID for organization
         storage_url: Provider's temporary URL to download from
         format: Video format (mp4, webm, etc.)
-        width: Video width in pixels
-        height: Video height in pixels
+        width: Video width in pixels (optional)
+        height: Video height in pixels (optional)
         duration: Video duration in seconds (optional)
         fps: Frames per second (optional)
 
@@ -344,7 +404,7 @@ async def store_video_result(
     logger.info(
         "Storing video result",
         generation_id=generation_id,
-        provider_url=storage_url,
+        provider_url=storage_url[:50],
         format=format,
     )
 
@@ -368,7 +428,7 @@ async def store_video_result(
         "Video stored successfully",
         generation_id=generation_id,
         storage_key=artifact_ref.storage_key,
-        storage_url=artifact_ref.storage_url,
+        storage_url=artifact_ref.storage_url[:50],
     )
 
     # Return artifact with our permanent storage URL
@@ -418,7 +478,7 @@ async def store_audio_result(
     logger.info(
         "Storing audio result",
         generation_id=generation_id,
-        provider_url=storage_url,
+        provider_url=storage_url[:50],
         format=format,
     )
 
@@ -442,7 +502,7 @@ async def store_audio_result(
         "Audio stored successfully",
         generation_id=generation_id,
         storage_key=artifact_ref.storage_key,
-        storage_url=artifact_ref.storage_url,
+        storage_url=artifact_ref.storage_url[:50],
     )
 
     # Return artifact with our permanent storage URL
@@ -485,7 +545,7 @@ async def store_text_result(
     logger.info(
         "Storing text result",
         generation_id=generation_id,
-        content=content,
+        content=content[:50],
         format=format,
     )
 
@@ -503,13 +563,13 @@ async def store_text_result(
         "Text stored successfully",
         generation_id=generation_id,
         storage_key=artifact_ref.storage_key,
-        storage_url=artifact_ref.storage_url,
+        storage_url=artifact_ref.storage_url[:50],
     )
 
     # Return artifact with our permanent storage URL
     return TextArtifact(
         generation_id=generation_id,
         storage_url=artifact_ref.storage_url,
-        content=content,
+        content=content[:50],
         format=format,
     )

package/templates/api/src/boards/jobs/repository.py

@@ -102,3 +102,52 @@ async def finalize_success(
         )
     )
     await session.execute(stmt)
+
+
+async def create_batch_generation(
+    session: AsyncSession,
+    *,
+    tenant_id: UUID,
+    board_id: UUID,
+    user_id: UUID,
+    generator_name: str,
+    artifact_type: str,
+    input_params: dict,
+    batch_id: str,
+    batch_index: int,
+) -> Generations:
+    """Create a batch generation record for multi-output generators.
+
+    This creates a new generation record that is part of a batch, with
+    batch metadata stored in output_metadata.
+
+    Args:
+        session: Database session
+        tenant_id: Tenant ID
+        board_id: Board ID
+        user_id: User ID
+        generator_name: Name of the generator
+        artifact_type: Type of artifact (image, video, etc.)
+        input_params: Input parameters (same as primary generation)
+        batch_id: Unique ID for this batch
+        batch_index: Index of this output in the batch
+
+    Returns:
+        Created generation record
+    """
+    gen = Generations()
+    gen.tenant_id = tenant_id
+    gen.board_id = board_id
+    gen.user_id = user_id
+    gen.generator_name = generator_name
+    gen.artifact_type = artifact_type
+    gen.input_params = input_params
+    gen.status = "processing"
+    gen.progress = Decimal(0.0)
+    gen.output_metadata = {
+        "batch_id": batch_id,
+        "batch_index": batch_index,
+    }
+    session.add(gen)
+    await session.flush()
+    return gen

package/templates/api/src/boards/storage/factory.py

@@ -47,7 +47,10 @@ try:
 except ImportError:
     SupabaseStorageProvider = None
     _supabase_available = False
-    logger.warning("Supabase storage not available - install supabase-py to enable")
+    logger.warning(
+        "Supabase storage not available. "
+        "Install with: pip install weirdfingers-boards[storage-supabase]"
+    )
 
 try:
     from .implementations.s3 import S3StorageProvider
@@ -56,7 +59,9 @@ try:
 except ImportError:
     S3StorageProvider = None
     _s3_available = False
-    logger.warning("S3 storage not available - install boto3 and aioboto3 to enable")
+    logger.warning(
+        "S3 storage not available. " "Install with: pip install weirdfingers-boards[storage-s3]"
+    )
 
 try:
     from .implementations.gcs import GCSStorageProvider
@@ -65,7 +70,9 @@ try:
 except ImportError:
     GCSStorageProvider = None
     _gcs_available = False
-    logger.warning("GCS storage not available - install google-cloud-storage to enable")
+    logger.warning(
+        "GCS storage not available. " "Install with: pip install weirdfingers-boards[storage-gcs]"
+    )
 
 
 def create_storage_provider(provider_type: str, config: dict[str, Any]) -> StorageProvider:
@@ -88,19 +95,22 @@ def create_storage_provider(provider_type: str, config: dict[str, Any]) -> Stora
     elif provider_type == "supabase":
         if not _supabase_available:
             raise ImportError(
-                "Supabase storage requires supabase-py. Install with: pip install supabase"
+                "Supabase storage requires additional dependencies. "
+                "Install with: pip install weirdfingers-boards[storage-supabase]"
             )
         return _create_supabase_provider(config)
     elif provider_type == "s3":
         if not _s3_available:
             raise ImportError(
-                "S3 storage requires boto3 and aioboto3. Install with: pip install boto3 aioboto3"
+                "S3 storage requires additional dependencies. "
+                "Install with: pip install weirdfingers-boards[storage-s3]"
             )
         return _create_s3_provider(config)
     elif provider_type == "gcs":
         if not _gcs_available:
             raise ImportError(
-                "GCS storage package required. Install with: pip install google-cloud-storage"
+                "GCS storage requires additional dependencies. "
+                "Install with: pip install weirdfingers-boards[storage-gcs]"
             )
         return _create_gcs_provider(config)
     else:

package/templates/api/src/boards/workers/actors.py

@@ -75,6 +75,8 @@ async def process_generation(generation_id: str) -> None:
             gen_id = gen.id
             tenant_id = gen.tenant_id
             board_id = gen.board_id
+            user_id = gen.user_id
+            artifact_type = gen.artifact_type
 
         # Initialize storage manager
         # This will use the default storage configuration from environment/config
@@ -88,18 +90,39 @@ async def process_generation(generation_id: str) -> None:
             raise RuntimeError(f"Unknown generator: {generator_name}")
 
         # Build and validate typed inputs
+        # First resolve any artifact fields (generation IDs -> artifact objects)
+        # This happens automatically via type introspection
         try:
             input_schema = generator.get_input_schema()
-            typed_inputs = input_schema.model_validate(input_params)
+
+            # Automatically resolve generation IDs to artifacts before validation
+            from ..generators.artifact_resolution import resolve_input_artifacts
+
+            async with get_async_session() as session:
+                resolved_params = await resolve_input_artifacts(
+                    input_params,
+                    input_schema,  # Schema is introspected to find artifact fields
+                    session,
+                    tenant_id,
+                )
+            typed_inputs = input_schema.model_validate(resolved_params)
         except Exception as e:
             error_msg = "Invalid input parameters"
             logger.error(error_msg, generation_id=generation_id, error=str(e))
             raise ValueError(f"Invalid input parameters: {e}") from e
 
         # Build context and run generator
-        # TODO(generators): make a way for a generator to add additional generations
-        # based on eg outputs=4, or similar.
-        context = GeneratorExecutionContext(gen_id, publisher, storage_manager, tenant_id, board_id)
+        context = GeneratorExecutionContext(
+            gen_id,
+            publisher,
+            storage_manager,
+            tenant_id,
+            board_id,
+            user_id,
+            generator_name,
+            artifact_type,
+            input_params,
+        )
 
         await publisher.publish_progress(
             generation_id,
@@ -125,9 +148,10 @@ async def process_generation(generation_id: str) -> None:
             "Generator completed successfully",
             generator_name=generator_name,
             generation_id=generation_id,
+            artifact_count=len(output.outputs),
         )
 
-        # Find the artifact with matching generation_id
+        # Find the artifact with matching generation_id (primary generation)
         # Generators should return exactly one artifact with the matching generation_id
         matching_artifacts = [art for art in output.outputs if art.generation_id == generation_id]
 
@@ -150,6 +174,18 @@ async def process_generation(generation_id: str) -> None:
         storage_url = artifact.storage_url
         output_metadata = artifact.model_dump()
 
+        # If this was a batch generation, add batch metadata to primary generation
+        if context._batch_id is not None:
+            output_metadata["batch_id"] = context._batch_id
+            output_metadata["batch_index"] = 0
+            output_metadata["batch_size"] = len(output.outputs)
+            logger.info(
+                "Primary generation is part of batch",
+                generation_id=generation_id,
+                batch_id=context._batch_id,
+                batch_size=len(output.outputs),
+            )
+
         # Finalize DB with storage URL and output metadata
         async with get_async_session() as session:
             await jobs_repo.finalize_success(
@@ -159,6 +195,34 @@ async def process_generation(generation_id: str) -> None:
                 output_metadata=output_metadata,
             )
 
+        # Finalize all batch generation records (if any)
+        if context._batch_id is not None:
+            batch_artifacts = [art for art in output.outputs if art.generation_id != generation_id]
+            logger.info(
+                "Finalizing batch generation records",
+                batch_id=context._batch_id,
+                batch_count=len(batch_artifacts),
+            )
+            for batch_artifact in batch_artifacts:
+                async with get_async_session() as session:
+                    batch_metadata = batch_artifact.model_dump()
+                    # Add batch metadata to each batch generation
+                    batch_metadata["batch_id"] = context._batch_id
+                    # batch_index was set during generation creation via create_batch_generation()
+                    batch_metadata["batch_size"] = len(output.outputs)
+
+                    await jobs_repo.finalize_success(
+                        session,
+                        batch_artifact.generation_id,
+                        storage_url=batch_artifact.storage_url,
+                        output_metadata=batch_metadata,
+                    )
+                    logger.info(
+                        "Batch generation finalized",
+                        batch_generation_id=batch_artifact.generation_id,
+                        batch_id=context._batch_id,
+                    )
+
         logger.info("Job finalized successfully", generation_id=generation_id)
 
         # Publish completion (DB already updated by finalize_success)