@weirdfingers/baseboards 0.2.0

package/templates/api/src/boards/storage/implementations/gcs.py

@@ -0,0 +1,340 @@
+"""Google Cloud Storage provider with IAM auth and CDN support."""
+
+import json
+import os
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from google.cloud import storage
+
+try:
+    import asyncio
+
+    from google.auth import default
+    from google.auth.exceptions import DefaultCredentialsError
+    from google.cloud import storage
+    from google.cloud.exceptions import GoogleCloudError, NotFound
+
+    _gcs_available = True
+except ImportError:
+    storage = None
+    NotFound = None
+    GoogleCloudError = None
+    default = None
+    DefaultCredentialsError = None
+    _gcs_available = False
+
+from ...logging import get_logger
+from ..base import StorageException, StorageProvider
+
+logger = get_logger(__name__)
+
+
+class GCSStorageProvider(StorageProvider):
+    """Google Cloud Storage with IAM auth, Cloud CDN, and proper async patterns."""
+
+    def __init__(
+        self,
+        bucket: str,
+        project_id: str | None = None,
+        credentials_path: str | None = None,
+        credentials_json: str | None = None,
+        cdn_domain: str | None = None,
+        upload_config: dict[str, Any] | None = None,
+    ):
+        if not _gcs_available:
+            raise ImportError(
+                "google-cloud-storage is required for GCSStorageProvider. "
+                "Install with: pip install google-cloud-storage"
+            )
+
+        self.bucket_name = bucket
+        self.project_id = project_id
+        self.credentials_path = credentials_path
+        self.credentials_json = credentials_json
+        self.cdn_domain = cdn_domain
+
+        # Default upload configuration
+        self.upload_config = {
+            "cache_control": "public, max-age=3600",
+            "predefined_acl": None,  # Use bucket's default ACL
+            **(upload_config or {}),
+        }
+
+        self._client: Any | None = None
+        self._bucket: Any | None = None
+
+        # Client will be initialized lazily on first use
+
+    def _get_client(self) -> Any:
+        """Get or create the GCS client with proper authentication."""
+        if self._client is None:
+            if storage is None:
+                raise ImportError("google-cloud-storage is required for GCSStorageProvider")
+
+            try:
+                if self.credentials_json:
+                    # Use JSON credentials string
+                    credentials_info = json.loads(self.credentials_json)
+                    from google.oauth2 import service_account
+
+                    credentials = service_account.Credentials.from_service_account_info(
+                        credentials_info,
+                        scopes=["https://www.googleapis.com/auth/cloud-platform"],
+                    )
+                    self._client = storage.Client(credentials=credentials, project=self.project_id)
+                elif self.credentials_path:
+                    # Use service account file
+                    credentials_path = Path(self.credentials_path)
+                    if not credentials_path.exists():
+                        raise FileNotFoundError(
+                            f"Credentials file not found: {self.credentials_path}"
+                        )
+
+                    os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(credentials_path)
+                    self._client = storage.Client(project=self.project_id)
+                else:
+                    # Use default credentials (environment variables, gcloud, etc.)
+                    self._client = storage.Client(project=self.project_id)
+
+                # Get bucket reference
+                self._bucket = self._client.bucket(self.bucket_name)
+
+            except Exception as e:
+                logger.error(f"Failed to initialize GCS client: {e}")
+                raise StorageException(f"GCS client initialization failed: {e}") from e
+
+        return self._client
+
+    async def _run_sync(self, func, *args, **kwargs) -> Any:
+        """Run synchronous GCS operations in thread pool."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(None, func, *args, **kwargs)
+
+    async def upload(
+        self,
+        key: str,
+        content: bytes | AsyncIterator[bytes],
+        content_type: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> str:
+        """Upload content to GCS."""
+        try:
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            # Create blob object
+            blob = bucket.blob(key)
+
+            # Set content type
+            blob.content_type = content_type
+
+            # Set cache control and other configuration
+            if self.upload_config.get("cache_control"):
+                blob.cache_control = self.upload_config["cache_control"]
+
+            # Add custom metadata
+            if metadata:
+                # GCS metadata keys must be lowercase and can contain only letters,
+                # numbers, and underscores
+                gcs_metadata = {}
+                for k, v in metadata.items():
+                    # Convert key to lowercase and replace invalid characters
+                    clean_key = k.lower().replace("-", "_").replace(" ", "_")
+                    gcs_metadata[clean_key] = str(v)
+                blob.metadata = gcs_metadata
+
+            # Handle streaming content for large files
+            if isinstance(content, bytes):
+                file_content = content
+            else:
+                # Collect streaming content into memory for upload
+                # For very large files, consider using resumable uploads
+                chunks = []
+                total_size = 0
+                async for chunk in content:
+                    chunks.append(chunk)
+                    total_size += len(chunk)
+                    # For files larger than 100MB, we could implement resumable upload
+                    if total_size > 100 * 1024 * 1024:
+                        logger.warning(
+                            f"Large file upload ({total_size} bytes) - "
+                            f"consider implementing resumable upload for key: {key}"
+                        )
+
+                file_content = b"".join(chunks)
+
+            # Upload using thread pool to avoid blocking
+            await self._run_sync(blob.upload_from_string, file_content, content_type=content_type)
+
+            # Return the CDN URL if configured, otherwise public GCS URL
+            if self.cdn_domain:
+                return f"https://{self.cdn_domain}/{key}"
+            else:
+                return f"https://storage.googleapis.com/{self.bucket_name}/{key}"
+
+        except Exception as e:
+            if isinstance(e, StorageException):
+                raise
+            logger.error(f"Unexpected error uploading {key} to GCS: {e}")
+            raise StorageException(f"GCS upload failed: {e}") from e
+
+    async def download(self, key: str) -> bytes:
+        """Download file content from GCS."""
+        try:
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            blob = bucket.blob(key)
+
+            # Download using thread pool to avoid blocking
+            content = await self._run_sync(blob.download_as_bytes)
+            return content
+
+        except Exception as e:
+            if isinstance(e, StorageException):
+                raise
+            logger.error(f"Failed to download {key} from GCS: {e}")
+            raise StorageException(f"GCS download failed: {e}") from e
+
+    async def get_presigned_upload_url(
+        self,
+        key: str,
+        content_type: str,
+        expires_in: timedelta | None = None,
+    ) -> dict[str, Any]:
+        """Generate presigned URL for direct client uploads."""
+        if expires_in is None:
+            expires_in = timedelta(hours=1)
+
+        try:
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            blob = bucket.blob(key)
+
+            # Generate signed URL for PUT operations
+            url = await self._run_sync(
+                blob.generate_signed_url,
+                version="v4",
+                expiration=expires_in,
+                method="PUT",
+                content_type=content_type,
+                headers={"Content-Type": content_type},
+            )
+
+            return {
+                "url": url,
+                "method": "PUT",
+                "headers": {"Content-Type": content_type},
+                "expires_at": (datetime.now(UTC) + expires_in).isoformat(),
+            }
+
+        except Exception as e:
+            if isinstance(e, StorageException):
+                raise
+            logger.error(f"Failed to create presigned upload URL for {key}: {e}")
+            raise StorageException(f"GCS presigned URL creation failed: {e}") from e
+
+    async def get_presigned_download_url(
+        self, key: str, expires_in: timedelta | None = None
+    ) -> str:
+        """Generate presigned URL for secure downloads."""
+        if expires_in is None:
+            expires_in = timedelta(hours=1)
+
+        try:
+            # Always use GCS native signed URLs for security
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            blob = bucket.blob(key)
+
+            # Generate signed URL for GET operations
+            url = await self._run_sync(
+                blob.generate_signed_url,
+                version="v4",
+                expiration=expires_in,
+                method="GET",
+            )
+
+            return url
+
+        except Exception as e:
+            if isinstance(e, StorageException):
+                raise
+            logger.error(f"Failed to create presigned download URL for {key}: {e}")
+            raise StorageException(f"GCS presigned download URL creation failed: {e}") from e
+
+    async def delete(self, key: str) -> bool:
+        """Delete file by storage key."""
+        try:
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            blob = bucket.blob(key)
+            await self._run_sync(blob.delete)
+            return True
+
+        except Exception as e:
+            logger.error(f"Unexpected error deleting {key} from GCS: {e}")
+            raise StorageException(f"GCS delete failed: {e}") from e
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists."""
+        try:
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            blob = bucket.blob(key)
+            exists = await self._run_sync(blob.exists)
+            return exists
+
+        except Exception:
+            return False
+
+    async def get_metadata(self, key: str) -> dict[str, Any]:
+        """Get file metadata (size, modified date, etc.)."""
+        try:
+            # Get client (initializes on first use)
+            client = self._get_client()
+            bucket = client.bucket(self.bucket_name)
+
+            blob = bucket.blob(key)
+
+            # Reload blob to get latest metadata
+            await self._run_sync(blob.reload)
+
+            result = {
+                "size": blob.size or 0,
+                "last_modified": blob.updated,
+                "content_type": blob.content_type,
+                "etag": blob.etag,
+                "generation": blob.generation,
+                "storage_class": blob.storage_class,
+                "cache_control": blob.cache_control,
+                "content_encoding": blob.content_encoding,
+                "content_disposition": blob.content_disposition,
+                "content_language": blob.content_language,
+            }
+
+            # Add custom metadata
+            if blob.metadata:
+                result["custom_metadata"] = blob.metadata
+
+            return result
+
+        except Exception as e:
+            if isinstance(e, StorageException):
+                raise
+            logger.error(f"Failed to get metadata for {key} from GCS: {e}")
+            raise StorageException(f"GCS get metadata failed: {e}") from e

package/templates/api/src/boards/storage/implementations/local.py

@@ -0,0 +1,201 @@
+"""Local filesystem storage provider for development and self-hosted deployments."""
+
+import json
+from collections.abc import AsyncIterable
+from datetime import timedelta
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+import aiofiles
+
+from ...logging import get_logger
+from ..base import SecurityException, StorageException, StorageProvider
+
+logger = get_logger(__name__)
+
+
+class LocalStorageProvider(StorageProvider):
+    """Local filesystem storage for development and self-hosted with security."""
+
+    def __init__(self, base_path: Path, public_url_base: str | None = None):
+        self.base_path = Path(base_path).resolve()  # Resolve to absolute path
+        self.public_url_base = public_url_base
+        self.base_path.mkdir(parents=True, exist_ok=True)
+
+    def _get_safe_file_path(self, key: str) -> Path:
+        """Get file path with security validation."""
+        # Ensure the resolved path is within base_path
+        file_path = (self.base_path / key).resolve()
+
+        # Check that resolved path is within base directory
+        try:
+            file_path.relative_to(self.base_path)
+        except ValueError as e:
+            raise SecurityException(f"Path traversal detected: {key}") from e
+
+        return file_path
+
+    async def upload(
+        self,
+        key: str,
+        content: bytes | bytearray | memoryview | AsyncIterable[bytes],
+        content_type: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> str:
+        logger.info("Uploading file", key=key, content_type=content_type, metadata=metadata)
+        try:
+            file_path = self._get_safe_file_path(key)
+            file_path.parent.mkdir(parents=True, exist_ok=True)
+
+            # Handle both bytes-like and async iterable content
+            if isinstance(content, bytes | bytearray | memoryview):
+                # aiofiles accepts bytes-like objects directly
+                async with aiofiles.open(file_path, "wb") as f:
+                    await f.write(content)
+            else:  # isinstance(content, AsyncIterable):
+                async with aiofiles.open(file_path, "wb") as f:
+                    async for chunk in content:
+                        # Just write the chunk directly - aiofiles accepts bytes-like objects
+                        # It will raise an error if chunk is not bytes-like
+                        await f.write(chunk)
+
+            # Store metadata atomically
+            if metadata:
+                try:
+                    metadata_path = file_path.with_suffix(file_path.suffix + ".meta")
+                    metadata_json = json.dumps(metadata, indent=2)
+
+                    async with aiofiles.open(metadata_path, "w") as f:
+                        await f.write(metadata_json)
+                except Exception as e:
+                    logger.warning(f"Failed to write metadata for {key}: {e}")
+                    # Continue - metadata failure shouldn't fail the upload
+
+            logger.debug(f"Successfully uploaded {key} to local storage")
+            return self._get_public_url(key)
+
+        except OSError as e:
+            logger.error(f"File system error uploading {key}: {e}")
+            raise StorageException(f"Failed to write file: {e}") from e
+        except Exception as e:
+            logger.error(f"Unexpected error uploading {key}: {e}")
+            raise StorageException(f"Upload failed: {e}") from e
+
+    def _get_public_url(self, key: str) -> str:
+        """Generate public URL for the stored file."""
+        if self.public_url_base:
+            # URL-encode the key for safety
+            encoded_key = quote(key, safe="/")
+            return f"{self.public_url_base.rstrip('/')}/{encoded_key}"
+        else:
+            return f"file://{self.base_path / key}"
+
+    async def download(self, key: str) -> bytes:
+        """Download file content from local storage."""
+        try:
+            file_path = self._get_safe_file_path(key)
+
+            if not file_path.exists():
+                raise StorageException(f"File not found: {key}")
+
+            async with aiofiles.open(file_path, "rb") as f:
+                return await f.read()
+
+        except OSError as e:
+            logger.error(f"File system error downloading {key}: {e}")
+            raise StorageException(f"Failed to read file: {e}") from e
+        except Exception as e:
+            logger.error(f"Unexpected error downloading {key}: {e}")
+            raise StorageException(f"Download failed: {e}") from e
+
+    async def get_presigned_upload_url(
+        self, key: str, content_type: str, expires_in: timedelta | None = None
+    ) -> dict[str, Any]:
+        """Local storage doesn't support presigned URLs - return direct upload info."""
+        # For local storage, we can't really do presigned URLs
+        # This would be handled by the web server (e.g., FastAPI endpoint)
+        return {
+            "url": f"/api/storage/upload/{quote(key, safe='/')}",
+            "fields": {"content-type": content_type},
+            "method": "PUT",
+            "expires_at": None,  # Handled by server session
+        }
+
+    async def get_presigned_download_url(
+        self, key: str, expires_in: timedelta | None = None
+    ) -> str:
+        """Return the public URL for local storage."""
+        return self._get_public_url(key)
+
+    async def delete(self, key: str) -> bool:
+        """Delete file by storage key."""
+        try:
+            file_path = self._get_safe_file_path(key)
+
+            if not file_path.exists():
+                return False
+
+            # Delete the main file
+            file_path.unlink()
+
+            # Delete metadata file if it exists
+            metadata_path = file_path.with_suffix(file_path.suffix + ".meta")
+            if metadata_path.exists():
+                metadata_path.unlink()
+
+            logger.debug(f"Successfully deleted {key} from local storage")
+            return True
+
+        except OSError as e:
+            logger.error(f"File system error deleting {key}: {e}")
+            raise StorageException(f"Failed to delete file: {e}") from e
+        except Exception as e:
+            logger.error(f"Unexpected error deleting {key}: {e}")
+            raise StorageException(f"Delete failed: {e}") from e
+
+    async def exists(self, key: str) -> bool:
+        """Check if file exists."""
+        try:
+            file_path = self._get_safe_file_path(key)
+            return file_path.exists()
+        except SecurityException:
+            return False
+        except Exception as e:
+            logger.warning(f"Error checking existence of {key}: {e}")
+            return False
+
+    async def get_metadata(self, key: str) -> dict[str, Any]:
+        """Get file metadata (size, modified date, etc.)."""
+        try:
+            file_path = self._get_safe_file_path(key)
+
+            if not file_path.exists():
+                raise StorageException(f"File not found: {key}")
+
+            stat = file_path.stat()
+
+            # Try to load stored metadata
+            stored_metadata = {}
+            metadata_path = file_path.with_suffix(file_path.suffix + ".meta")
+            if metadata_path.exists():
+                try:
+                    async with aiofiles.open(metadata_path) as f:
+                        metadata_content = await f.read()
+                        stored_metadata = json.loads(metadata_content)
+                except Exception as e:
+                    logger.warning(f"Failed to load metadata for {key}: {e}")
+
+            return {
+                "size": stat.st_size,
+                "modified_time": stat.st_mtime,
+                "created_time": stat.st_ctime,
+                **stored_metadata,
+            }
+
+        except OSError as e:
+            logger.error(f"File system error getting metadata for {key}: {e}")
+            raise StorageException(f"Failed to get metadata: {e}") from e
+        except Exception as e:
+            logger.error(f"Unexpected error getting metadata for {key}: {e}")
+            raise StorageException(f"Get metadata failed: {e}") from e