draft-board 0.1.0-beta.0

package/app/backend/app/utils/db_retry.py

@@ -0,0 +1,136 @@
+"""Database retry decorator for SQLite BUSY errors.
+
+SQLite has concurrency limitations and can throw BUSY errors when multiple
+processes/threads access the database. This module provides retry logic
+with exponential backoff.
+"""
+
+import asyncio
+import logging
+import sqlite3
+from collections.abc import Callable
+from functools import wraps
+from typing import TypeVar
+
+from sqlalchemy.exc import OperationalError
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+
+def with_db_retry(
+    max_retries: int = 3,
+    base_backoff: float = 0.1,
+    max_backoff: float = 2.0,
+):
+    """Decorator to retry async DB operations on SQLite BUSY errors.
+
+    Args:
+        max_retries: Maximum number of retry attempts
+        base_backoff: Base backoff time in seconds
+        max_backoff: Maximum backoff time in seconds
+
+    Usage:
+        @with_db_retry(max_retries=3)
+        async def my_db_operation(self, ...):
+            # ... database operations ...
+            await self.db.commit()
+    """
+
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            last_exception = None
+
+            for attempt in range(max_retries):
+                try:
+                    return await func(*args, **kwargs)
+                except (sqlite3.OperationalError, OperationalError) as e:
+                    last_exception = e
+                    error_msg = str(e).lower()
+
+                    # Only retry on BUSY/locked errors
+                    if "database is locked" in error_msg or "busy" in error_msg:
+                        if attempt < max_retries - 1:
+                            # Exponential backoff with cap
+                            wait = min(base_backoff * (2**attempt), max_backoff)
+                            logger.warning(
+                                f"DB locked in {func.__name__}, "
+                                f"retry {attempt + 1}/{max_retries} after {wait:.2f}s"
+                            )
+                            await asyncio.sleep(wait)
+                            continue
+                        else:
+                            logger.error(
+                                f"DB locked in {func.__name__}, "
+                                f"exhausted {max_retries} retries"
+                            )
+                    # For other OperationalErrors, don't retry
+                    raise
+
+            # If we get here, all retries exhausted
+            raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+def with_db_retry_sync(
+    max_retries: int = 3,
+    base_backoff: float = 0.1,
+    max_backoff: float = 2.0,
+):
+    """Decorator to retry sync DB operations on SQLite BUSY errors.
+
+    Args:
+        max_retries: Maximum number of retry attempts
+        base_backoff: Base backoff time in seconds
+        max_backoff: Maximum backoff time in seconds
+
+    Usage:
+        @with_db_retry_sync(max_retries=3)
+        def my_db_operation(self, ...):
+            # ... database operations ...
+            db.commit()
+    """
+    import time
+
+    def decorator(func: Callable[..., T]) -> Callable[..., T]:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+
+            for attempt in range(max_retries):
+                try:
+                    return func(*args, **kwargs)
+                except (sqlite3.OperationalError, OperationalError) as e:
+                    last_exception = e
+                    error_msg = str(e).lower()
+
+                    # Only retry on BUSY/locked errors
+                    if "database is locked" in error_msg or "busy" in error_msg:
+                        if attempt < max_retries - 1:
+                            # Exponential backoff with cap
+                            wait = min(base_backoff * (2**attempt), max_backoff)
+                            logger.warning(
+                                f"DB locked in {func.__name__}, "
+                                f"retry {attempt + 1}/{max_retries} after {wait:.2f}s"
+                            )
+                            time.sleep(wait)
+                            continue
+                        else:
+                            logger.error(
+                                f"DB locked in {func.__name__}, "
+                                f"exhausted {max_retries} retries"
+                            )
+                    # For other OperationalErrors, don't retry
+                    raise
+
+            # If we get here, all retries exhausted
+            raise last_exception
+
+        return wrapper
+
+    return decorator

package/app/backend/app/utils/ignored_fields.py

@@ -0,0 +1,123 @@
+"""Utility for tracking and logging ignored request fields.
+
+SECURITY: Only echo KNOWN deprecated fields in X-Ignored-Fields header.
+Arbitrary unknown fields are logged internally but NOT echoed to client
+(prevents using this as an echo channel).
+
+When deprecated fields are sent:
+1. Add X-Ignored-Fields header with ONLY known deprecated fields
+2. Log once per client_id per day (avoid spam)
+3. Unknown fields: log internally only, do NOT echo
+"""
+
+import logging
+from datetime import date
+from typing import Any
+
+from fastapi import Request, Response
+
+logger = logging.getLogger(__name__)
+
+# Track which clients have been warned today: {(client_id, field): date}
+_warned_today: dict[tuple[str, str], date] = {}
+
+# ALLOWLIST: Only these deprecated fields are echoed in X-Ignored-Fields
+# This prevents using the header as an echo channel for arbitrary data
+KNOWN_DEPRECATED_FIELDS = frozenset(
+    {
+        "workspace_path",  # Removed for security - use board.repo_root instead
+        "repo_path",  # Alias for workspace_path
+    }
+)
+
+
+def get_client_id(request: Request) -> str:
+    """Get client identifier from request."""
+    client_id = request.headers.get("X-Client-ID")
+    if client_id and len(client_id) <= 64:
+        return client_id
+    forwarded = request.headers.get("X-Forwarded-For")
+    if forwarded:
+        return f"ip:{forwarded.split(',')[0].strip()}"
+    return f"ip:{request.client.host if request.client else 'unknown'}"
+
+
+def check_ignored_fields(
+    request: Request,
+    raw_body: dict[str, Any],
+    allowed_fields: set[str],
+) -> list[str]:
+    """Check for ignored/deprecated fields in request body.
+
+    SECURITY: Only KNOWN_DEPRECATED_FIELDS are returned for echoing.
+    Unknown extra fields are logged internally but NOT returned.
+
+    Args:
+        request: The FastAPI request
+        raw_body: Parsed request body dict
+        allowed_fields: Fields that are actually used by the endpoint
+
+    Returns:
+        List of KNOWN deprecated field names that were sent (safe to echo)
+    """
+    if not raw_body:
+        return []
+
+    sent_fields = set(raw_body.keys())
+    all_ignored = sent_fields - allowed_fields
+
+    if not all_ignored:
+        return []
+
+    client_id = get_client_id(request)
+    today = date.today()
+
+    # Split into known deprecated (safe to echo) and unknown (log only)
+    known_deprecated_sent = all_ignored & KNOWN_DEPRECATED_FIELDS
+    unknown_sent = all_ignored - KNOWN_DEPRECATED_FIELDS
+
+    # Log known deprecated fields (once per client per day)
+    for field in known_deprecated_sent:
+        cache_key = (client_id, field)
+        if _warned_today.get(cache_key) != today:
+            logger.warning(
+                f"Client {client_id} sent deprecated field '{field}' - "
+                f"this field is ignored for security. "
+                f"Please remove it from your requests."
+            )
+            _warned_today[cache_key] = today
+
+    # Log unknown fields internally only (do NOT echo to client)
+    if unknown_sent:
+        # Only log once per client per day to avoid spam
+        cache_key = (client_id, "__unknown_fields__")
+        if _warned_today.get(cache_key) != today:
+            logger.info(
+                f"Client {client_id} sent unknown fields: {sorted(unknown_sent)} - "
+                f"these are silently ignored."
+            )
+            _warned_today[cache_key] = today
+
+    # Return ONLY known deprecated fields (safe to echo)
+    return sorted(known_deprecated_sent)
+
+
+def add_ignored_fields_header(response: Response, ignored_fields: list[str]) -> None:
+    """Add X-Ignored-Fields header to response.
+
+    SECURITY: Only adds header if ignored_fields contains known deprecated fields.
+    The ignored_fields list should come from check_ignored_fields() which already
+    filters to KNOWN_DEPRECATED_FIELDS only.
+    """
+    if ignored_fields:
+        # Double-check: only include known deprecated fields
+        safe_fields = [f for f in ignored_fields if f in KNOWN_DEPRECATED_FIELDS]
+        if safe_fields:
+            response.headers["X-Ignored-Fields"] = ", ".join(safe_fields)
+
+
+def cleanup_old_warnings() -> None:
+    """Clean up warning cache for old dates."""
+    global _warned_today
+    today = date.today()
+    _warned_today = {k: v for k, v in _warned_today.items() if v == today}

package/app/backend/app/utils/validators.py

@@ -0,0 +1,54 @@
+"""Input validation utilities for API requests."""
+
+import uuid
+
+from fastapi import HTTPException
+
+
+def validate_uuid(value: str, field_name: str = "ID") -> str:
+    """Validate that a string is a valid UUID format.
+
+    Args:
+        value: String to validate
+        field_name: Field name for error message
+
+    Returns:
+        The validated UUID string (normalized)
+
+    Raises:
+        HTTPException: 400 if not a valid UUID
+    """
+    try:
+        # Parse and normalize UUID
+        parsed = uuid.UUID(value)
+        return str(parsed)
+    except (ValueError, AttributeError, TypeError):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid {field_name}: must be a valid UUID (got: {value})",
+        )
+
+
+def validate_uuids(values: list[str], field_name: str = "IDs") -> list[str]:
+    """Validate a list of UUIDs.
+
+    Args:
+        values: List of UUID strings to validate
+        field_name: Field name for error message
+
+    Returns:
+        List of validated UUID strings
+
+    Raises:
+        HTTPException: 400 if any UUID is invalid
+    """
+    validated = []
+    for value in values:
+        try:
+            validated.append(validate_uuid(value, field_name))
+        except HTTPException:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Invalid {field_name}: '{value}' is not a valid UUID",
+            )
+    return validated

package/app/backend/app/websocket/__init__.py

@@ -0,0 +1,5 @@
+"""WebSocket infrastructure for real-time updates."""
+
+from app.websocket.manager import ConnectionManager, broadcast_sync, manager
+
+__all__ = ["ConnectionManager", "manager", "broadcast_sync"]

package/app/backend/app/websocket/manager.py

@@ -0,0 +1,179 @@
+"""WebSocket connection manager for real-time updates.
+
+This module manages WebSocket connections and provides channel-based
+broadcasting for real-time updates to clients.
+"""
+
+import asyncio
+import logging
+
+from fastapi import WebSocket
+
+logger = logging.getLogger(__name__)
+
+
+class ConnectionManager:
+    """Manage WebSocket connections for real-time updates.
+
+    Supports channel-based broadcasting where clients can subscribe to
+    specific channels (e.g., job:{job_id}, board:{board_id}) to receive
+    targeted updates.
+    """
+
+    def __init__(self):
+        # Map of channel -> set of websockets
+        self.connections: dict[str, set[WebSocket]] = {}
+        self._lock = asyncio.Lock()
+
+    async def connect(self, websocket: WebSocket, channel: str):
+        """Accept and register a new WebSocket connection to a channel.
+
+        Args:
+            websocket: The WebSocket connection to register
+            channel: The channel to subscribe to (e.g., "job:123", "board:abc")
+        """
+        await websocket.accept()
+        async with self._lock:
+            if channel not in self.connections:
+                self.connections[channel] = set()
+            self.connections[channel].add(websocket)
+            logger.info(f"WebSocket connected to channel: {channel}")
+
+    async def disconnect(self, websocket: WebSocket, channel: str):
+        """Unregister a WebSocket connection from a channel.
+
+        Args:
+            websocket: The WebSocket connection to remove
+            channel: The channel to unsubscribe from
+        """
+        async with self._lock:
+            if channel in self.connections:
+                self.connections[channel].discard(websocket)
+                if not self.connections[channel]:
+                    # Remove empty channel
+                    del self.connections[channel]
+                logger.info(f"WebSocket disconnected from channel: {channel}")
+
+    async def broadcast(self, channel: str, message: dict):
+        """Send message to all connections on a channel.
+
+        Args:
+            channel: The channel to broadcast to
+            message: The message dict to send (will be JSON serialized)
+
+        Automatically cleans up dead connections that fail to receive messages.
+        """
+        if channel not in self.connections:
+            return
+
+        dead_connections = set()
+        for connection in list(self.connections[channel]):
+            try:
+                await connection.send_json(message)
+            except Exception as e:
+                logger.warning(
+                    f"Failed to send message to WebSocket on channel {channel}: {e}"
+                )
+                dead_connections.add(connection)
+
+        # Clean up dead connections
+        if dead_connections:
+            async with self._lock:
+                if channel in self.connections:
+                    self.connections[channel] -= dead_connections
+                    if not self.connections[channel]:
+                        del self.connections[channel]
+                    logger.info(
+                        f"Cleaned up {len(dead_connections)} dead connections from {channel}"
+                    )
+
+    async def broadcast_to_all(self, message: dict):
+        """Broadcast message to all connections across all channels.
+
+        Args:
+            message: The message dict to send to all connected clients
+        """
+        for channel in list(self.connections.keys()):
+            await self.broadcast(channel, message)
+
+    def get_connection_count(self, channel: str = None) -> int:
+        """Get count of active connections.
+
+        Args:
+            channel: Optional channel to count connections for.
+                    If None, returns total across all channels.
+
+        Returns:
+            Number of active connections
+        """
+        if channel:
+            return len(self.connections.get(channel, set()))
+        return sum(len(conns) for conns in self.connections.values())
+
+    def get_channels(self) -> list[str]:
+        """Get list of all active channels.
+
+        Returns:
+            List of channel names with active connections
+        """
+        return list(self.connections.keys())
+
+    async def broadcast_board_state(self, board_id: str, board_state: dict):
+        """Broadcast board state as JSON patch to connected clients.
+
+        On first call for a board, sends a full snapshot. On subsequent calls,
+        computes and sends an RFC 6902 JSON patch. Sends nothing if no change.
+
+        Args:
+            board_id: The board ID
+            board_state: Full board state dict
+        """
+        channel = f"board:{board_id}"
+        if channel not in self.connections:
+            return
+
+        from app.websocket.state_tracker import get_tracker
+
+        tracker = get_tracker(board_id)
+
+        if not tracker.has_state:
+            message = tracker.get_snapshot_message(board_state)
+        else:
+            message = tracker.compute_patch(board_state)
+            if message is None:
+                return  # No changes
+
+        await self.broadcast(channel, message)
+
+
+# Global connection manager instance
+manager = ConnectionManager()
+
+
+# Helper functions for sync code (like Celery workers)
+def broadcast_sync(channel: str, message: dict):
+    """Broadcast message from synchronous code.
+
+    This is a helper for sync contexts (like Celery workers) that need to
+    broadcast WebSocket messages. It schedules the broadcast on the event loop.
+
+    Args:
+        channel: The channel to broadcast to
+        message: The message dict to send
+
+    Note: This uses asyncio.create_task() which requires an active event loop.
+    If called from a thread without a loop, the broadcast will be skipped.
+    """
+    try:
+        import asyncio
+
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            asyncio.create_task(manager.broadcast(channel, message))
+        else:
+            # If no loop is running, we can't broadcast
+            logger.debug(
+                f"Skipping WebSocket broadcast to {channel} - no event loop running"
+            )
+    except Exception as e:
+        logger.debug(f"Failed to broadcast WebSocket message to {channel}: {e}")

package/app/backend/app/websocket/state_tracker.py

@@ -0,0 +1,113 @@
+"""Board state tracker for computing JSON patches.
+
+Tracks the last-known board state per connection and computes RFC 6902
+JSON patches between states, enabling incremental updates over WebSocket.
+
+Protocol:
+  1. On connect: send full snapshot {"type": "snapshot", "data": ..., "seq": 0}
+  2. On change:  send patch     {"type": "patch", "ops": [...], "seq": N}
+  3. On gap:     client sends   {"type": "resync"} → server resends snapshot
+"""
+
+import copy
+import logging
+from typing import Any
+
+import jsonpatch
+
+logger = logging.getLogger(__name__)
+
+
+class BoardStateTracker:
+    """Tracks board state and computes JSON patches for incremental updates.
+
+    One tracker per board; stores the last-known state and a sequence counter.
+    Thread-safe for single-event-loop usage (async context).
+    """
+
+    def __init__(self) -> None:
+        self._state: dict[str, Any] | None = None
+        self._seq: int = 0
+
+    @property
+    def seq(self) -> int:
+        return self._seq
+
+    @property
+    def has_state(self) -> bool:
+        return self._state is not None
+
+    def set_state(self, state: dict[str, Any]) -> None:
+        """Set the current board state (used for initial snapshot)."""
+        self._state = copy.deepcopy(state)
+
+    def get_snapshot_message(self, state: dict[str, Any]) -> dict[str, Any]:
+        """Build a snapshot message and update internal state.
+
+        Args:
+            state: Full board state dict.
+
+        Returns:
+            Message dict: {"type": "snapshot", "data": ..., "seq": 0}
+        """
+        self._state = copy.deepcopy(state)
+        self._seq = 0
+        return {
+            "type": "snapshot",
+            "data": state,
+            "seq": self._seq,
+        }
+
+    def compute_patch(self, new_state: dict[str, Any]) -> dict[str, Any] | None:
+        """Compute a JSON patch between the stored state and new_state.
+
+        If the patch is empty (no changes), returns None.
+
+        Args:
+            new_state: The new board state dict.
+
+        Returns:
+            Patch message dict or None if no changes:
+            {"type": "patch", "ops": [...], "seq": N}
+        """
+        if self._state is None:
+            # No previous state → send snapshot instead
+            return self.get_snapshot_message(new_state)
+
+        try:
+            patch = jsonpatch.make_patch(self._state, new_state)
+            ops = patch.patch
+
+            if not ops:
+                return None
+
+            self._seq += 1
+            self._state = copy.deepcopy(new_state)
+
+            return {
+                "type": "patch",
+                "ops": ops,
+                "seq": self._seq,
+            }
+
+        except Exception as e:
+            logger.warning(
+                f"Failed to compute JSON patch: {e}, falling back to snapshot"
+            )
+            return self.get_snapshot_message(new_state)
+
+
+# Per-board trackers keyed by board_id
+_trackers: dict[str, BoardStateTracker] = {}
+
+
+def get_tracker(board_id: str) -> BoardStateTracker:
+    """Get or create a state tracker for a board."""
+    if board_id not in _trackers:
+        _trackers[board_id] = BoardStateTracker()
+    return _trackers[board_id]
+
+
+def remove_tracker(board_id: str) -> None:
+    """Remove a tracker when no more connections exist for a board."""
+    _trackers.pop(board_id, None)