machinaos 0.0.1

package/server/core/cache.py

@@ -0,0 +1,461 @@
+"""Cache service with Redis (production) or SQLite (development) backend.
+
+Follows n8n pattern where SQLite is sufficient for single-process deployments,
+with Redis used only for distributed queue mode or high-performance needs.
+"""
+
+import json
+import asyncio
+from typing import Any, Dict, Optional, List, TYPE_CHECKING
+from datetime import timedelta
+
+try:
+    import redis.asyncio as redis
+    REDIS_AVAILABLE = True
+except ImportError:
+    redis = None
+    REDIS_AVAILABLE = False
+
+from core.config import Settings
+from core.logging import get_logger, log_cache_operation
+
+if TYPE_CHECKING:
+    from core.database import Database
+
+logger = get_logger(__name__)
+
+
+class CacheService:
+    """Async cache service with Redis or SQLite backend.
+
+    Backend selection:
+    - Redis: When REDIS_ENABLED=true and Redis is available (production)
+    - SQLite: When Redis disabled or unavailable (development)
+    - Memory: Temporary fallback if both fail
+    """
+
+    def __init__(self, settings: Settings, database: Optional["Database"] = None):
+        self.settings = settings
+        self.database = database  # SQLite backend
+        self.redis: Optional[redis.Redis] = None
+        self.memory_cache: Dict[str, Any] = {}  # Emergency fallback only
+        self.use_redis = settings.redis_enabled and REDIS_AVAILABLE
+        self.use_sqlite = not self.use_redis and database is not None
+        self._streams_available = False  # Checked during startup
+
+    async def startup(self):
+        """Initialize cache connection."""
+        if self.use_redis and self.settings.redis_url:
+            try:
+                self.redis = redis.from_url(
+                    self.settings.redis_url,
+                    encoding="utf-8",
+                    decode_responses=True,
+                    socket_timeout=5,
+                    socket_connect_timeout=5,
+                    retry_on_timeout=True
+                )
+
+                # Test connection
+                await self.redis.ping()
+                logger.info("Redis cache initialized", url=self.settings.redis_url)
+
+                # Test Redis Streams availability (required for trigger nodes)
+                await self._check_streams_support()
+
+            except Exception as e:
+                logger.warning("Redis connection failed, falling back", error=str(e))
+                self.use_redis = False
+                self.redis = None
+                # Try SQLite fallback
+                if self.database:
+                    self.use_sqlite = True
+                    logger.info("Using SQLite cache (Redis fallback)")
+        else:
+            if self.use_sqlite:
+                logger.info("Using SQLite cache (n8n pattern - no Redis required for single-process)")
+            else:
+                logger.info("Using in-memory cache",
+                           redis_enabled=self.settings.redis_enabled,
+                           redis_available=REDIS_AVAILABLE)
+
+    async def _check_streams_support(self):
+        """Check if Redis supports Streams (XADD/XREAD commands).
+
+        Some Redis-compatible services (e.g., certain cloud providers) don't support Streams.
+        We test this at startup to avoid runtime failures in trigger nodes.
+        """
+        if not self.redis:
+            self._streams_available = False
+            return
+
+        test_stream = "_machina_streams_test"
+        try:
+            # Try XADD - this will fail if Streams aren't supported
+            msg_id = await self.redis.xadd(test_stream, {"test": "1"}, maxlen=1)
+            if msg_id:
+                # Clean up test stream
+                await self.redis.delete(test_stream)
+                self._streams_available = True
+                logger.info("Redis Streams available - trigger nodes will use Redis persistence")
+            else:
+                self._streams_available = False
+                logger.warning("Redis Streams test failed - trigger nodes will use memory mode")
+        except Exception as e:
+            self._streams_available = False
+            error_str = str(e).lower()
+            if "unknown command" in error_str:
+                logger.warning("Redis Streams not supported by server - trigger nodes will use memory mode")
+            else:
+                logger.warning(f"Redis Streams check failed: {e} - trigger nodes will use memory mode")
+
+    async def shutdown(self):
+        """Close cache connections."""
+        if self.redis:
+            await self.redis.close()
+            logger.info("Redis cache connections closed")
+
+        # Clear memory cache
+        self.memory_cache.clear()
+
+    async def get(self, key: str) -> Optional[Any]:
+        """Get value from cache."""
+        try:
+            if self.use_redis and self.redis:
+                value = await self.redis.get(key)
+                if value:
+                    log_cache_operation(logger, "get", key, hit=True)
+                    return json.loads(value)
+                else:
+                    log_cache_operation(logger, "get", key, hit=False)
+                    return None
+            elif self.use_sqlite and self.database:
+                # SQLite cache
+                value = await self.database.get_cache_entry(key)
+                if value:
+                    log_cache_operation(logger, "get", key, hit=True)
+                    return json.loads(value)
+                else:
+                    log_cache_operation(logger, "get", key, hit=False)
+                    return None
+            else:
+                # Memory cache fallback
+                value = self.memory_cache.get(key)
+                log_cache_operation(logger, "get", key, hit=value is not None)
+                return value
+
+        except Exception as e:
+            logger.error("Cache get failed", key=key, error=str(e))
+            return None
+
+    async def set(self, key: str, value: Any, ttl: Optional[int] = None) -> bool:
+        """Set value in cache with optional TTL."""
+        try:
+            ttl = ttl or self.settings.cache_ttl
+
+            if self.use_redis and self.redis:
+                serialized = json.dumps(value, default=str)
+                await self.redis.setex(key, ttl, serialized)
+                log_cache_operation(logger, "set", key, ttl=ttl)
+                return True
+            elif self.use_sqlite and self.database:
+                # SQLite cache with TTL support
+                serialized = json.dumps(value, default=str)
+                await self.database.set_cache_entry(key, serialized, ttl)
+                log_cache_operation(logger, "set", key, ttl=ttl)
+                return True
+            else:
+                # Memory cache fallback (no TTL)
+                self.memory_cache[key] = value
+                log_cache_operation(logger, "set", key, ttl=ttl)
+                return True
+
+        except Exception as e:
+            logger.error("Cache set failed", key=key, error=str(e))
+            return False
+
+    async def delete(self, key: str) -> bool:
+        """Delete value from cache."""
+        try:
+            if self.use_redis and self.redis:
+                deleted = await self.redis.delete(key)
+                log_cache_operation(logger, "delete", key, deleted=bool(deleted))
+                return bool(deleted)
+            elif self.use_sqlite and self.database:
+                # SQLite cache
+                deleted = await self.database.delete_cache_entry(key)
+                log_cache_operation(logger, "delete", key, deleted=deleted)
+                return deleted
+            else:
+                # Memory cache fallback
+                deleted = key in self.memory_cache
+                if deleted:
+                    del self.memory_cache[key]
+                log_cache_operation(logger, "delete", key, deleted=deleted)
+                return deleted
+
+        except Exception as e:
+            logger.error("Cache delete failed", key=key, error=str(e))
+            return False
+
+    async def exists(self, key: str) -> bool:
+        """Check if key exists in cache."""
+        try:
+            if self.use_redis and self.redis:
+                exists = await self.redis.exists(key)
+                return bool(exists)
+            elif self.use_sqlite and self.database:
+                return await self.database.cache_exists(key)
+            else:
+                return key in self.memory_cache
+
+        except Exception as e:
+            logger.error("Cache exists check failed", key=key, error=str(e))
+            return False
+
+    async def expire(self, key: str, ttl: int) -> bool:
+        """Set TTL for existing key."""
+        try:
+            if self.use_redis and self.redis:
+                return bool(await self.redis.expire(key, ttl))
+            else:
+                # Memory cache doesn't support TTL updates
+                return key in self.memory_cache
+
+        except Exception as e:
+            logger.error("Cache expire failed", key=key, error=str(e))
+            return False
+
+    async def clear_pattern(self, pattern: str) -> int:
+        """Clear keys matching pattern."""
+        try:
+            if self.use_redis and self.redis:
+                keys = await self.redis.keys(pattern)
+                if keys:
+                    deleted = await self.redis.delete(*keys)
+                    log_cache_operation(logger, "clear_pattern", pattern, deleted=deleted)
+                    return deleted
+                return 0
+            elif self.use_sqlite and self.database:
+                # SQLite cache pattern matching
+                deleted = await self.database.delete_cache_pattern(pattern)
+                log_cache_operation(logger, "clear_pattern", pattern, deleted=deleted)
+                return deleted
+            else:
+                # Memory cache pattern matching
+                keys_to_delete = [k for k in self.memory_cache.keys() if pattern.replace("*", "") in k]
+                for key in keys_to_delete:
+                    del self.memory_cache[key]
+                log_cache_operation(logger, "clear_pattern", pattern, deleted=len(keys_to_delete))
+                return len(keys_to_delete)
+
+        except Exception as e:
+            logger.error("Cache clear pattern failed", pattern=pattern, error=str(e))
+            return 0
+
+    # ============================================================================
+    # API Key Specific Cache Methods
+    # ============================================================================
+
+    async def cache_api_key(self, provider: str, session_id: str, key_data: Dict[str, Any]) -> bool:
+        """Cache API key data."""
+        cache_key = f"api_key:{provider}:{session_id}"
+        return await self.set(cache_key, key_data, self.settings.api_key_cache_ttl)
+
+    async def get_cached_api_key(self, provider: str, session_id: str) -> Optional[Dict[str, Any]]:
+        """Get cached API key data."""
+        cache_key = f"api_key:{provider}:{session_id}"
+        return await self.get(cache_key)
+
+    async def remove_cached_api_key(self, provider: str, session_id: str) -> bool:
+        """Remove cached API key."""
+        cache_key = f"api_key:{provider}:{session_id}"
+        return await self.delete(cache_key)
+
+    async def cache_models(self, provider: str, models: List[str]) -> bool:
+        """Cache available models for provider."""
+        cache_key = f"models:{provider}"
+        return await self.set(cache_key, {"models": models, "cached_at": "now"},
+                            ttl=3600)  # 1 hour
+
+    async def get_cached_models(self, provider: str) -> Optional[List[str]]:
+        """Get cached models for provider."""
+        cache_key = f"models:{provider}"
+        data = await self.get(cache_key)
+        return data.get("models") if data else None
+
+    # ============================================================================
+    # Redis Streams Methods for Event Waiting
+    # ============================================================================
+
+    async def stream_add(self, stream: str, data: Dict[str, Any], maxlen: int = 1000) -> Optional[str]:
+        """Add message to Redis Stream.
+
+        Args:
+            stream: Stream name (e.g., 'events:whatsapp_message_received')
+            data: Event data to store
+            maxlen: Maximum stream length (approximate, uses ~)
+
+        Returns:
+            Message ID if successful, None otherwise
+        """
+        try:
+            if self.use_redis and self.redis and self._streams_available:
+                # Serialize ALL values with json.dumps to preserve types
+                # This matches the pattern used in set() and ensures proper round-trip:
+                # - json.dumps(True) → "true" (lowercase, valid JSON)
+                # - json.loads("true") → True (Python bool)
+                # Using str() would break: str(True) → "True" → json.loads fails
+                serialized = {k: json.dumps(v, default=str) for k, v in data.items()}
+                msg_id = await self.redis.xadd(stream, serialized, maxlen=maxlen, approximate=True)
+                logger.debug(f"Stream add: {stream} -> {msg_id}")
+                return msg_id
+            return None
+        except Exception as e:
+            logger.error(f"Stream add failed: {stream}", error=str(e))
+            return None
+
+    async def stream_read(
+        self,
+        streams: Dict[str, str],
+        count: int = 1,
+        block: Optional[int] = None
+    ) -> Optional[List[Any]]:
+        """Read from Redis Streams.
+
+        Args:
+            streams: Dict of stream_name -> last_id (use '$' for new messages only, '0' for all)
+            count: Maximum number of messages to read
+            block: Milliseconds to block (None = no blocking, 0 = infinite)
+
+        Returns:
+            List of [stream_name, [(msg_id, data), ...]] or None
+        """
+        try:
+            if self.use_redis and self.redis:
+                result = await self.redis.xread(streams, count=count, block=block)
+                return result
+            return None
+        except Exception as e:
+            logger.error(f"Stream read failed: {streams.keys()}", error=str(e))
+            return None
+
+    async def stream_create_group(
+        self,
+        stream: str,
+        group: str,
+        start_id: str = '$'
+    ) -> bool:
+        """Create consumer group for stream.
+
+        Args:
+            stream: Stream name
+            group: Consumer group name
+            start_id: Start reading from ('$' = new only, '0' = all)
+
+        Returns:
+            True if created or already exists
+        """
+        try:
+            if self.use_redis and self.redis and self._streams_available:
+                try:
+                    await self.redis.xgroup_create(stream, group, start_id, mkstream=True)
+                    logger.info(f"Created consumer group: {group} on {stream}")
+                    return True
+                except Exception as e:
+                    if "BUSYGROUP" in str(e):
+                        # Group already exists - this is fine
+                        return True
+                    raise
+            return False
+        except Exception as e:
+            logger.error(f"Stream create group failed: {stream}/{group}", error=str(e))
+            return False
+
+    async def stream_read_group(
+        self,
+        group: str,
+        consumer: str,
+        streams: Dict[str, str],
+        count: int = 1,
+        block: Optional[int] = None
+    ) -> Optional[List[Any]]:
+        """Read from streams using consumer group.
+
+        Args:
+            group: Consumer group name
+            consumer: Consumer name (unique per worker)
+            streams: Dict of stream_name -> last_id (use '>' for new pending messages)
+            count: Maximum messages to read
+            block: Milliseconds to block
+
+        Returns:
+            List of messages or None
+        """
+        try:
+            if self.use_redis and self.redis and self._streams_available:
+                result = await self.redis.xreadgroup(
+                    group, consumer, streams,
+                    count=count, block=block
+                )
+                return result
+            return None
+        except Exception as e:
+            error_str = str(e).lower()
+            # Timeout errors are expected during blocking reads - log at debug level
+            if "timeout" in error_str or "timed out" in error_str:
+                logger.debug(f"Stream read group timeout: {group}/{consumer}", error=str(e))
+            else:
+                logger.error(f"Stream read group failed: {group}/{consumer}", error=str(e))
+            return None
+
+    async def stream_ack(self, stream: str, group: str, *msg_ids: str) -> int:
+        """Acknowledge messages in consumer group.
+
+        Args:
+            stream: Stream name
+            group: Consumer group name
+            msg_ids: Message IDs to acknowledge
+
+        Returns:
+            Number of messages acknowledged
+        """
+        try:
+            if self.use_redis and self.redis:
+                count = await self.redis.xack(stream, group, *msg_ids)
+                return count
+            return 0
+        except Exception as e:
+            logger.error(f"Stream ack failed: {stream}/{group}", error=str(e))
+            return 0
+
+    async def stream_delete(self, stream: str, *msg_ids: str) -> int:
+        """Delete messages from stream.
+
+        Args:
+            stream: Stream name
+            msg_ids: Message IDs to delete
+
+        Returns:
+            Number of messages deleted
+        """
+        try:
+            if self.use_redis and self.redis:
+                count = await self.redis.xdel(stream, *msg_ids)
+                return count
+            return 0
+        except Exception as e:
+            logger.error(f"Stream delete failed: {stream}", error=str(e))
+            return 0
+
+    def is_redis_available(self) -> bool:
+        """Check if Redis is available and connected."""
+        return self.use_redis and self.redis is not None
+
+    def is_streams_available(self) -> bool:
+        """Check if Redis Streams are available (for trigger nodes).
+
+        Returns True only if Redis is connected AND supports Streams commands.
+        This is checked once during startup to avoid runtime failures.
+        """
+        return self.use_redis and self.redis is not None and self._streams_available

package/server/core/config.py

@@ -0,0 +1,128 @@
+"""Environment-driven configuration with Pydantic v2."""
+
+from typing import List, Literal, Optional
+from pathlib import Path
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings
+
+
+class Settings(BaseSettings):
+    """Application settings driven entirely by environment variables."""
+
+    # Service Ports (used by start.js, vite, and docker-compose, not hardcoded here)
+    vite_client_port: Optional[int] = Field(default=None, env="VITE_CLIENT_PORT")
+    python_backend_port: Optional[int] = Field(default=None, env="PYTHON_BACKEND_PORT")
+    whatsapp_rpc_port: Optional[int] = Field(default=None, env="WHATSAPP_RPC_PORT")
+    redis_port: Optional[int] = Field(default=None, env="REDIS_PORT")
+
+    # Server Configuration
+    host: str = Field(env="HOST")
+    port: int = Field(env="PORT", ge=1024, le=65535)
+    debug: bool = Field(default=False, env="DEBUG")
+    workers: int = Field(default=1, env="WORKERS", ge=1, le=8)
+
+    # Authentication
+    auth_mode: Literal["single", "multi"] = Field(default="single", env="AUTH_MODE")
+    jwt_secret_key: str = Field(env="JWT_SECRET_KEY", min_length=32)
+    jwt_expire_minutes: int = Field(default=10080, env="JWT_EXPIRE_MINUTES", ge=60)  # 7 days
+    jwt_cookie_name: str = Field(default="machina_token", env="JWT_COOKIE_NAME")
+    jwt_cookie_secure: bool = Field(default=False, env="JWT_COOKIE_SECURE")  # True in production
+    jwt_cookie_samesite: Literal["lax", "strict", "none"] = Field(default="lax", env="JWT_COOKIE_SAMESITE")
+
+    # Security
+    secret_key: str = Field(env="SECRET_KEY", min_length=32)
+    cors_origins: List[str] = Field(env="CORS_ORIGINS")
+
+    # Database Configuration
+    database_url: str = Field(env="DATABASE_URL")
+    database_echo: bool = Field(default=False, env="DATABASE_ECHO")
+    database_pool_size: int = Field(default=20, env="DATABASE_POOL_SIZE", ge=5, le=100)
+    database_max_overflow: int = Field(default=30, env="DATABASE_MAX_OVERFLOW", ge=10, le=100)
+
+    # Cache Configuration
+    redis_url: Optional[str] = Field(default=None, env="REDIS_URL")
+    redis_enabled: bool = Field(default=False, env="REDIS_ENABLED")
+    cache_ttl: int = Field(default=3600, env="CACHE_TTL", ge=60)
+
+    # Execution Engine
+    dlq_enabled: bool = Field(default=False, env="DLQ_ENABLED")
+
+    # Temporal Configuration
+    temporal_enabled: bool = Field(default=False, env="TEMPORAL_ENABLED")
+    temporal_server_address: str = Field(default="localhost:7233", env="TEMPORAL_SERVER_ADDRESS")
+    temporal_namespace: str = Field(default="default", env="TEMPORAL_NAMESPACE")
+    temporal_task_queue: str = Field(default="machina-tasks", env="TEMPORAL_TASK_QUEUE")
+
+    # API Keys (all optional, injected at runtime)
+    google_maps_api_key: Optional[str] = Field(default=None, env="GOOGLE_MAPS_API_KEY")
+    openai_api_key: Optional[str] = Field(default=None, env="OPENAI_API_KEY")
+    anthropic_api_key: Optional[str] = Field(default=None, env="ANTHROPIC_API_KEY")
+    google_ai_api_key: Optional[str] = Field(default=None, env="GOOGLE_AI_API_KEY")
+
+    # WhatsApp Service URL (Flask service)
+    whatsapp_service_url: str = Field(default="http://localhost:5000", env="WHATSAPP_SERVICE_URL")
+
+    # WebSocket Configuration
+    websocket_url: str = Field(default="", env="WEBSOCKET_URL")
+    websocket_api_key: Optional[str] = Field(default=None, env="WEBSOCKET_API_KEY")
+
+    # Android Relay Configuration (passed to Vite frontend)
+    vite_android_relay_url: Optional[str] = Field(default=None, env="VITE_ANDROID_RELAY_URL")
+
+    # Frontend Auth Configuration (passed to Vite frontend)
+    vite_auth_enabled: Optional[str] = Field(default=None, env="VITE_AUTH_ENABLED")
+
+    # API Key Security
+    api_key_encryption_key: str = Field(env="API_KEY_ENCRYPTION_KEY", min_length=32)
+    api_key_cache_ttl: int = Field(default=2592000, env="API_KEY_CACHE_TTL", ge=3600)
+
+    # Logging
+    log_level: str = Field(default="INFO", env="LOG_LEVEL")
+    log_format: str = Field(default="json", env="LOG_FORMAT")
+    log_file: Optional[str] = Field(default=None, env="LOG_FILE")
+
+    # Rate Limiting
+    rate_limit_enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED")
+    rate_limit_requests: int = Field(default=100, env="RATE_LIMIT_REQUESTS", ge=10)
+    rate_limit_window: int = Field(default=60, env="RATE_LIMIT_WINDOW", ge=10)
+
+    # Service Timeouts
+    ai_timeout: int = Field(default=30, env="AI_TIMEOUT", ge=5, le=300)
+    ai_max_retries: int = Field(default=3, env="AI_MAX_RETRIES", ge=0, le=5)
+    ai_retry_delay: float = Field(default=1.0, env="AI_RETRY_DELAY", ge=0.1, le=10.0)
+
+    maps_timeout: int = Field(default=10, env="MAPS_TIMEOUT", ge=5, le=60)
+    maps_max_requests_per_second: int = Field(default=50, env="MAPS_MAX_RPS", ge=1, le=1000)
+
+    # Health Check
+    health_check_interval: int = Field(default=30, env="HEALTH_CHECK_INTERVAL", ge=10)
+
+    @field_validator("database_url")
+    @classmethod
+    def validate_database_url(cls, v):
+        """Ensure database directory exists for SQLite."""
+        if v and v.startswith("sqlite"):
+            if ":///" in v:
+                db_path = v.split("///")[1]
+                Path(db_path).parent.mkdir(parents=True, exist_ok=True)
+        return v
+
+
+    @property
+    def is_development(self) -> bool:
+        """Check if running in development mode."""
+        return self.debug
+
+    @property
+    def is_production(self) -> bool:
+        """Check if running in production mode."""
+        return not self.debug
+
+    model_config = {
+        "env_file": "../.env",
+        "env_file_encoding": "utf-8",
+        "case_sensitive": False,
+        "extra": "forbid",
+        "env_parse_none_str": "none",
+        "env_nested_delimiter": "__",
+    }

package/server/core/container.py

@@ -0,0 +1,99 @@
+"""Dependency injection container for the application."""
+
+from dependency_injector import containers, providers
+from dependency_injector.wiring import Provide, inject
+
+from core.config import Settings
+from core.database import Database
+from core.cache import CacheService
+from services.ai import AIService
+from services.maps import MapsService
+from services.workflow import WorkflowService
+from services.auth import AuthService
+from services.text import TextService
+from services.android_service import AndroidService
+from services.user_auth import UserAuthService
+from services.temporal import TemporalClientWrapper
+
+
+class Container(containers.DeclarativeContainer):
+    """Application dependency injection container."""
+
+    # Configuration
+    config = providers.Configuration()
+
+    # Settings
+    settings = providers.Singleton(
+        Settings,
+    )
+
+    # Database (needed by CacheService for SQLite fallback)
+    database = providers.Singleton(
+        Database,
+        settings=settings
+    )
+
+    # Cache service (uses Redis when available, SQLite otherwise)
+    cache = providers.Singleton(
+        CacheService,
+        settings=settings,
+        database=database
+    )
+
+    # Temporal client (lazy - only created when temporal_enabled)
+    temporal_client = providers.Singleton(
+        TemporalClientWrapper,
+        server_address=settings.provided.temporal_server_address,
+        namespace=settings.provided.temporal_namespace,
+    )
+
+    # Services
+    auth_service = providers.Factory(
+        AuthService,
+        cache=cache,
+        database=database,
+        settings=settings
+    )
+
+    user_auth_service = providers.Factory(
+        UserAuthService,
+        database=database,
+        settings=settings
+    )
+
+    ai_service = providers.Factory(
+        AIService,
+        auth_service=auth_service,
+        database=database,
+        cache=cache,
+        settings=settings
+    )
+
+    maps_service = providers.Factory(
+        MapsService,
+        auth_service=auth_service,
+        settings=settings
+    )
+
+    text_service = providers.Factory(
+        TextService
+    )
+
+    android_service = providers.Factory(
+        AndroidService
+    )
+
+    workflow_service = providers.Singleton(
+        WorkflowService,
+        database=database,
+        ai_service=ai_service,
+        maps_service=maps_service,
+        text_service=text_service,
+        android_service=android_service,
+        cache=cache,
+        settings=settings
+    )
+
+
+# Global container instance
+container = Container()