machinaos 0.0.1

package/server/core/logging.py

@@ -0,0 +1,314 @@
+"""Modern structured logging configuration with WebSocket broadcasting."""
+
+import sys
+import asyncio
+import structlog
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, Optional
+from queue import Queue
+from threading import Thread
+from core.config import Settings
+
+
+class WebSocketLogHandler(logging.Handler):
+    """Logging handler that broadcasts logs to WebSocket clients.
+
+    Uses a thread-safe queue to bridge sync logging with async WebSocket broadcasting.
+    A background thread processes the queue and uses asyncio to broadcast.
+    """
+
+    _instance: Optional['WebSocketLogHandler'] = None
+
+    def __init__(self, level: int = logging.INFO):
+        super().__init__(level)
+        self._queue: Queue = Queue(maxsize=1000)  # Bounded queue to prevent memory issues
+        self._running = False
+        self._thread: Optional[Thread] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+
+        # Source name mapping for cleaner display
+        self._source_map = {
+            'services.workflow': 'workflow',
+            'services.ai': 'ai',
+            'services.android': 'android',
+            'routers.whatsapp': 'whatsapp',
+            'routers.android': 'android',
+            'routers.websocket': 'websocket',
+            'routers.workflow': 'workflow',
+            'services.execution': 'execution',
+            'services.deployment': 'deployment',
+            '__main__': 'main',
+            'main': 'main',
+        }
+
+    @classmethod
+    def get_instance(cls) -> Optional['WebSocketLogHandler']:
+        """Get the singleton instance."""
+        return cls._instance
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """Queue log record for async broadcasting."""
+        if not self._running:
+            return
+
+        try:
+            # Get the raw message without structlog formatting
+            message = record.getMessage()
+
+            # Map source name
+            source = record.name
+            for prefix, mapped in self._source_map.items():
+                if source.startswith(prefix):
+                    source = mapped
+                    break
+
+            # Extract structured key-value pairs from structlog
+            details = None
+            if hasattr(record, '_logger') or hasattr(record, 'positional_args'):
+                # Try to get extra kwargs from structlog
+                extra_keys = set(record.__dict__.keys()) - {
+                    'name', 'msg', 'args', 'created', 'filename', 'funcName',
+                    'levelname', 'levelno', 'lineno', 'module', 'msecs',
+                    'pathname', 'process', 'processName', 'relativeCreated',
+                    'stack_info', 'exc_info', 'exc_text', 'thread', 'threadName',
+                    'message', 'asctime', 'positional_args', '_logger'
+                }
+                if extra_keys:
+                    details = {k: record.__dict__[k] for k in extra_keys if not k.startswith('_')}
+
+            # Create log entry
+            log_data = {
+                'timestamp': datetime.now().isoformat(),
+                'level': record.levelname.lower(),
+                'message': message,
+                'source': source,
+            }
+
+            # Add details if present
+            if details:
+                log_data['details'] = details
+
+            # Non-blocking put - drop if queue is full
+            try:
+                self._queue.put_nowait(log_data)
+            except:
+                pass  # Drop log if queue is full
+
+        except Exception:
+            pass  # Never fail in log handler
+
+    def start(self, loop: asyncio.AbstractEventLoop) -> None:
+        """Start the background thread for processing logs."""
+        if self._running:
+            return
+
+        self._loop = loop
+        self._running = True
+        self._thread = Thread(target=self._process_queue, daemon=True)
+        self._thread.start()
+        WebSocketLogHandler._instance = self
+
+    def stop(self) -> None:
+        """Stop the background thread."""
+        self._running = False
+        WebSocketLogHandler._instance = None
+        if self._thread:
+            self._thread.join(timeout=1.0)
+
+    def _process_queue(self) -> None:
+        """Background thread that processes log queue and broadcasts."""
+        while self._running:
+            try:
+                # Block for up to 0.1 seconds waiting for logs
+                try:
+                    log_data = self._queue.get(timeout=0.1)
+                except:
+                    continue
+
+                # Schedule async broadcast on the event loop
+                if self._loop and self._running:
+                    asyncio.run_coroutine_threadsafe(
+                        self._broadcast(log_data),
+                        self._loop
+                    )
+
+            except Exception:
+                pass  # Never fail in background thread
+
+    async def _broadcast(self, log_data: Dict[str, Any]) -> None:
+        """Broadcast log to WebSocket clients."""
+        try:
+            from services.status_broadcaster import get_status_broadcaster
+            broadcaster = get_status_broadcaster()
+            await broadcaster.broadcast_terminal_log(log_data)
+        except Exception:
+            pass  # Don't fail if broadcaster not ready
+
+
+def configure_logging(settings: Settings) -> None:
+    """Configure structured logging based on settings."""
+
+    # Set up log file if specified
+    if settings.log_file:
+        log_path = Path(settings.log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Configure file handler
+        file_handler = logging.FileHandler(log_path)
+        file_handler.setLevel(getattr(logging, settings.log_level.upper()))
+
+        # Configure console handler
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(getattr(logging, settings.log_level.upper()))
+
+        # Configure root logger
+        logging.basicConfig(
+            level=getattr(logging, settings.log_level.upper()),
+            handlers=[console_handler, file_handler],
+            format="%(message)s"
+        )
+    else:
+        # Console only
+        logging.basicConfig(
+            format="%(message)s",
+            stream=sys.stdout,
+            level=getattr(logging, settings.log_level.upper())
+        )
+
+    # Configure structlog
+    processors = [
+        structlog.stdlib.filter_by_level,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.PositionalArgumentsFormatter(),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+        structlog.processors.UnicodeDecoder(),
+    ]
+
+    # Add appropriate renderer based on format
+    if settings.log_format == "json":
+        processors.insert(0, structlog.processors.TimeStamper(fmt="iso"))
+        processors.insert(0, structlog.stdlib.add_logger_name)
+        processors.append(structlog.processors.JSONRenderer())
+    else:
+        # Console format with timestamp
+        processors.insert(0, structlog.processors.TimeStamper(fmt="%H:%M:%S"))
+        processors.append(structlog.dev.ConsoleRenderer(
+            colors=False,  # No ANSI colors for cleaner output
+            pad_event=35,
+            exception_formatter=structlog.dev.plain_traceback
+        ))
+
+    structlog.configure(
+        processors=processors,
+        context_class=dict,
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        wrapper_class=structlog.stdlib.BoundLogger,
+        cache_logger_on_first_use=True,
+    )
+
+
+def get_logger(name: str) -> structlog.BoundLogger:
+    """Get a structured logger instance."""
+    return structlog.get_logger(name)
+
+
+def log_execution_time(logger: structlog.BoundLogger, operation: str,
+                      start_time: float, end_time: float, **kwargs) -> None:
+    """Log execution time with additional context."""
+    execution_time = end_time - start_time
+    logger.info(
+        "Operation completed",
+        operation=operation,
+        execution_time_seconds=round(execution_time, 4),
+        **kwargs
+    )
+
+
+def log_api_call(logger: structlog.BoundLogger, provider: str, model: str,
+                operation: str, success: bool, **kwargs) -> None:
+    """Log API calls with standardized format."""
+    logger.info(
+        "API call completed",
+        provider=provider,
+        model=model,
+        operation=operation,
+        success=success,
+        **kwargs
+    )
+
+
+def log_cache_operation(logger: structlog.BoundLogger, operation: str,
+                       key: str, hit: bool = None, **kwargs) -> None:
+    """Log cache operations."""
+    log_data = {
+        "operation": operation,
+        "cache_key": key,
+        **kwargs
+    }
+
+    if hit is not None:
+        log_data["cache_hit"] = hit
+
+    logger.debug("Cache operation", **log_data)
+
+
+# Global WebSocket log handler instance
+_ws_log_handler: Optional[WebSocketLogHandler] = None
+
+
+def setup_websocket_logging(loop: asyncio.AbstractEventLoop, level: int = logging.INFO) -> WebSocketLogHandler:
+    """Setup and start the WebSocket log handler.
+
+    Should be called during application startup after the event loop is running.
+
+    Args:
+        loop: The asyncio event loop to use for broadcasting
+        level: Minimum log level to broadcast (default: INFO)
+
+    Returns:
+        The WebSocket log handler instance
+    """
+    global _ws_log_handler
+
+    if _ws_log_handler is not None:
+        return _ws_log_handler
+
+    # Create handler
+    _ws_log_handler = WebSocketLogHandler(level=level)
+
+    # Add to root logger
+    root_logger = logging.getLogger()
+    root_logger.addHandler(_ws_log_handler)
+
+    # Start the background processing thread
+    _ws_log_handler.start(loop)
+
+    return _ws_log_handler
+
+
+def shutdown_websocket_logging() -> None:
+    """Shutdown the WebSocket log handler.
+
+    Should be called during application shutdown.
+    """
+    global _ws_log_handler
+
+    if _ws_log_handler is None:
+        return
+
+    # Stop the handler
+    _ws_log_handler.stop()
+
+    # Remove from root logger
+    root_logger = logging.getLogger()
+    root_logger.removeHandler(_ws_log_handler)
+
+    _ws_log_handler = None
+
+
+def get_websocket_log_handler() -> Optional[WebSocketLogHandler]:
+    """Get the current WebSocket log handler instance."""
+    return _ws_log_handler

package/server/main.py

@@ -0,0 +1,289 @@
+"""
+Modern FastAPI backend for React Flow workflow automation platform.
+
+Refactored with dependency injection, modular services, and clean architecture.
+"""
+
+# Performance: Install uvloop if available (Linux/macOS only)
+try:
+    import uvloop
+    uvloop.install()
+except ImportError:
+    pass  # Windows - uvloop not available, use default asyncio
+
+import os
+from datetime import datetime
+from contextlib import asynccontextmanager
+
+# Note: We don't register custom signal handlers.
+# uvicorn already handles SIGINT (Ctrl+C) and SIGTERM (docker stop) gracefully.
+# Adding custom handlers that raise KeyboardInterrupt causes cascading errors
+# during async operations (WebSocket handlers, logging, etc.).
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import ORJSONResponse
+
+from core.container import container
+from core.config import Settings
+from core.logging import configure_logging, get_logger, setup_websocket_logging, shutdown_websocket_logging
+from routers import workflow, database, maps, nodejs_compat, android, websocket, webhook, auth
+
+# Initialize settings and logging
+settings = Settings()
+configure_logging(settings)
+logger = get_logger(__name__)
+
+# Suppress noisy loggers
+import logging
+logging.getLogger("apscheduler").setLevel(logging.WARNING)
+logging.getLogger("apscheduler.scheduler").setLevel(logging.WARNING)
+logging.getLogger("apscheduler.executors").setLevel(logging.WARNING)
+logging.getLogger("uvicorn").setLevel(logging.WARNING)
+logging.getLogger("uvicorn.access").setLevel(logging.WARNING)
+logging.getLogger("uvicorn.error").setLevel(logging.WARNING)
+logging.getLogger("watchfiles").setLevel(logging.WARNING)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan management."""
+    # Startup
+    logger.info("Starting React Flow Python Services")
+
+    # Wire dependency injection
+    container.wire(modules=[
+        "routers.workflow",
+        "routers.database",
+        "routers.maps",
+        "routers.nodejs_compat",
+        "routers.android",
+        "routers.websocket",
+        "routers.webhook",
+        "routers.auth",
+        "middleware.auth"
+    ])
+
+    # Start services
+    await container.database().startup()
+    await container.cache().startup()
+
+    # Initialize event waiter with cache service for Redis Streams support
+    from services import event_waiter
+    event_waiter.set_cache_service(container.cache())
+
+    # Start APScheduler for cron jobs
+    from services.scheduler import start_scheduler, shutdown_scheduler
+    start_scheduler()
+
+    # Initialize execution engine recovery sweeper
+    from services.execution import (
+        ExecutionCache,
+        RecoverySweeper,
+        set_recovery_sweeper,
+    )
+    execution_cache = ExecutionCache(container.cache())
+    recovery_sweeper = RecoverySweeper(execution_cache)
+    set_recovery_sweeper(recovery_sweeper)
+
+    # Scan for incomplete executions on startup
+    if settings.redis_enabled:
+        incomplete = await recovery_sweeper.scan_on_startup()
+        if incomplete:
+            logger.info("Found incomplete executions on startup",
+                       count=len(incomplete),
+                       execution_ids=incomplete)
+
+        # Start background recovery sweeper
+        await recovery_sweeper.start()
+        logger.info("Execution recovery sweeper started")
+
+    # Start WebSocket logging handler to broadcast logs to frontend
+    import asyncio
+    loop = asyncio.get_running_loop()
+    setup_websocket_logging(loop)
+    logger.info("WebSocket logging handler started")
+
+    # Initialize Temporal if enabled
+    temporal_worker_manager = None
+    if settings.temporal_enabled:
+        try:
+            from services.temporal import TemporalClientWrapper, TemporalExecutor
+            from services.temporal.worker import TemporalWorkerManager
+
+            logger.info(
+                "Initializing Temporal integration",
+                server_address=settings.temporal_server_address,
+                namespace=settings.temporal_namespace,
+                task_queue=settings.temporal_task_queue,
+            )
+
+            # Connect Temporal client
+            temporal_client_wrapper = container.temporal_client()
+            temporal_client = await temporal_client_wrapper.connect()
+
+            # Create and set the Temporal executor on WorkflowService
+            temporal_executor = TemporalExecutor(
+                client=temporal_client,
+                task_queue=settings.temporal_task_queue,
+            )
+            container.workflow_service().set_temporal_executor(temporal_executor)
+
+            # Start embedded Temporal worker
+            temporal_worker_manager = TemporalWorkerManager(
+                client=temporal_client,
+                task_queue=settings.temporal_task_queue,
+            )
+            await temporal_worker_manager.start()
+
+            logger.info("Temporal integration initialized successfully")
+
+        except Exception as e:
+            logger.error(f"Failed to initialize Temporal: {str(e)}")
+            logger.warning("Falling back to Redis/sequential execution")
+
+    logger.info("Services started successfully")
+    yield
+
+    # Shutdown
+    # Stop WebSocket logging handler
+    shutdown_websocket_logging()
+
+    # Stop Temporal worker if running
+    if temporal_worker_manager is not None:
+        await temporal_worker_manager.stop()
+        logger.info("Temporal worker stopped")
+
+    # Disconnect Temporal client if connected
+    if settings.temporal_enabled:
+        try:
+            await container.temporal_client().disconnect()
+        except Exception:
+            pass
+
+    # Close Android relay client (prevents "Unclosed client session" warning)
+    from services.android.manager import close_relay_client
+    await close_relay_client(clear_stored_session=False)
+
+    # Stop recovery sweeper first
+    if settings.redis_enabled:
+        await recovery_sweeper.stop()
+        logger.info("Execution recovery sweeper stopped")
+
+    shutdown_scheduler()  # Stop APScheduler
+    await container.cache().shutdown()
+    await container.database().shutdown()
+    logger.info("Services shutdown complete")
+
+
+# Create FastAPI app
+app = FastAPI(
+    title="React Flow Python Services",
+    version="3.0.0",
+    description="Modern workflow automation backend with AI and Maps integration",
+    lifespan=lifespan,
+    default_response_class=ORJSONResponse
+)
+
+# Add exception handler middleware BEFORE CORS to catch all errors
+from fastapi import Request, status
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+
+class CatchAllExceptionsMiddleware(BaseHTTPMiddleware):
+    async def dispatch(self, request: Request, call_next):
+        try:
+            return await call_next(request)
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            logger.error(f"Unhandled exception: {type(e).__name__}: {str(e)}", exc_info=True)
+            return JSONResponse(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                content={
+                    "success": False,
+                    "error": f"{type(e).__name__}: {str(e)}",
+                    "detail": "Internal server error"
+                }
+            )
+
+app.add_middleware(CatchAllExceptionsMiddleware)
+
+# Add Auth middleware (checks JWT cookie for protected routes)
+from middleware.auth import AuthMiddleware
+app.add_middleware(AuthMiddleware)
+
+# Add CORS middleware (must be AFTER exception middleware)
+logger.info("Configuring CORS middleware",
+           origins_count=len(settings.cors_origins),
+           origins=settings.cors_origins)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+
+# Include routers
+app.include_router(auth.router)  # Auth routes (login, register, logout, status)
+app.include_router(nodejs_compat.router)  # Node.js compatibility (includes root endpoints)
+app.include_router(workflow.router)
+app.include_router(database.router)
+app.include_router(maps.router)
+app.include_router(android.router)
+app.include_router(websocket.router)
+app.include_router(webhook.router)
+
+
+@app.get("/health")
+async def health_check():
+    """Detailed health check."""
+    from services import event_waiter
+    from services.execution import get_recovery_sweeper
+
+    sweeper = get_recovery_sweeper()
+
+    # Check Temporal status
+    temporal_status = {
+        "enabled": settings.temporal_enabled,
+        "connected": False,
+    }
+    if settings.temporal_enabled:
+        try:
+            temporal_status["connected"] = container.temporal_client().is_connected
+            temporal_status["server_address"] = settings.temporal_server_address
+            temporal_status["task_queue"] = settings.temporal_task_queue
+        except Exception:
+            pass
+
+    return {
+        "status": "OK",
+        "service": "python",
+        "version": "3.2.0",  # Bumped for Temporal integration
+        "environment": "development" if settings.debug else "production",
+        "redis_enabled": settings.redis_enabled,
+        "event_waiter_mode": event_waiter.get_backend_mode(),
+        "execution_engine": {
+            "enabled": settings.redis_enabled,
+            "recovery_sweeper": sweeper is not None and sweeper._running,
+        },
+        "temporal": temporal_status,
+        "timestamp": datetime.now().isoformat()
+    }
+
+
+if __name__ == "__main__":
+    import uvicorn
+    logger.info("Starting React Flow Python Services",
+               host=settings.host, port=settings.port, debug=settings.debug)
+    uvicorn.run(
+        "main:app",
+        host=settings.host,
+        port=settings.port,
+        reload=settings.debug,
+        reload_dirs=["."] if settings.debug else None,
+        reload_excludes=["*.pyc", "__pycache__", "*.log", "*.db"] if settings.debug else None,
+        workers=1 if settings.debug else settings.workers
+    )

package/server/middleware/__init__.py

@@ -0,0 +1,5 @@
+"""Middleware package for FastAPI application."""
+
+from middleware.auth import AuthMiddleware
+
+__all__ = ["AuthMiddleware"]

package/server/middleware/auth.py

@@ -0,0 +1,89 @@
+"""Authentication middleware for route protection."""
+
+import logging
+from fastapi import Request, HTTPException
+from fastapi.responses import JSONResponse
+from starlette.middleware.base import BaseHTTPMiddleware
+
+from core.container import container
+
+logger = logging.getLogger(__name__)
+
+# Public routes that don't require authentication
+PUBLIC_PATHS = frozenset([
+    "/health",
+    "/docs",
+    "/openapi.json",
+    "/redoc",
+    "/api/auth/status",
+    "/api/auth/login",
+    "/api/auth/register",
+    "/api/auth/logout",
+    "/ws/internal",  # Internal WebSocket for Temporal workers
+])
+
+# Path prefixes that are public
+PUBLIC_PREFIXES = (
+    "/webhook/",
+)
+
+
+class AuthMiddleware(BaseHTTPMiddleware):
+    """Middleware to protect routes requiring authentication."""
+
+    async def dispatch(self, request: Request, call_next):
+        path = request.url.path
+
+        # Allow public paths
+        if self._is_public_path(path):
+            return await call_next(request)
+
+        # Get settings
+        settings = container.settings()
+
+        # Check if auth is disabled (VITE_AUTH_ENABLED=false)
+        if settings.vite_auth_enabled and settings.vite_auth_enabled.lower() == 'false':
+            # Auth disabled - set anonymous user and allow request
+            request.state.user_id = 0
+            request.state.user_email = 'anonymous'
+            request.state.is_owner = True
+            return await call_next(request)
+
+        # Auth enabled - check token
+        token = request.cookies.get(settings.jwt_cookie_name)
+
+        if not token:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Not authenticated"}
+            )
+
+        # Verify token
+        user_auth = container.user_auth_service()
+        payload = user_auth.verify_token(token)
+
+        if not payload:
+            return JSONResponse(
+                status_code=401,
+                content={"detail": "Invalid or expired session"}
+            )
+
+        # Attach user info to request state for downstream handlers
+        request.state.user_id = payload.get("sub")
+        request.state.user_email = payload.get("email")
+        request.state.is_owner = payload.get("is_owner", False)
+
+        return await call_next(request)
+
+    def _is_public_path(self, path: str) -> bool:
+        """Check if path is public (no auth required)."""
+        # Exact match
+        if path in PUBLIC_PATHS:
+            return True
+
+        # Prefix match
+        for prefix in PUBLIC_PREFIXES:
+            if path.startswith(prefix):
+                return True
+
+        return False

package/server/models/__init__.py

@@ -0,0 +1 @@
+# Database models