machinaos 0.0.1

package/server/services/execution/recovery.py

@@ -0,0 +1,235 @@
+"""Recovery sweeper for crash recovery.
+
+Runs as background task to:
+- Detect abandoned executions (running but no heartbeat)
+- Resume interrupted workflows
+- Clean up stale data
+"""
+
+import asyncio
+import time
+from typing import Dict, Any, List, Optional, Callable, Awaitable
+
+from core.logging import get_logger
+from .models import TaskStatus, WorkflowStatus
+from .cache import ExecutionCache
+
+logger = get_logger(__name__)
+
+
+class RecoverySweeper:
+    """Background task that recovers abandoned workflow executions.
+
+    Conductor's sweeper pattern:
+    - Periodically scans for stuck executions
+    - Detects nodes with stale heartbeats
+    - Resets stuck nodes to PENDING for retry
+    - Triggers workflow_decide to resume
+    """
+
+    def __init__(self, cache: ExecutionCache,
+                 heartbeat_timeout: int = 300,  # 5 minutes
+                 sweep_interval: int = 60,      # 1 minute
+                 max_retries: int = 3):
+        """Initialize recovery sweeper.
+
+        Args:
+            cache: ExecutionCache for Redis access
+            heartbeat_timeout: Seconds before node is considered stuck
+            sweep_interval: Seconds between sweep runs
+            max_retries: Max retry attempts per node
+        """
+        self.cache = cache
+        self.heartbeat_timeout = heartbeat_timeout
+        self.sweep_interval = sweep_interval
+        self.max_retries = max_retries
+        self._running = False
+        self._task: Optional[asyncio.Task] = None
+
+        # Recovery callbacks (set by workflow service)
+        self._on_recovery: Optional[Callable[[str], Awaitable[None]]] = None
+
+    def set_recovery_callback(self, callback: Callable[[str], Awaitable[None]]) -> None:
+        """Set callback to invoke when execution needs recovery.
+
+        Args:
+            callback: Async function that takes execution_id
+        """
+        self._on_recovery = callback
+
+    async def start(self) -> None:
+        """Start the recovery sweeper background task."""
+        if self._running:
+            logger.warning("Recovery sweeper already running")
+            return
+
+        self._running = True
+        self._task = asyncio.create_task(self._sweep_loop())
+        logger.info("Recovery sweeper started",
+                   heartbeat_timeout=self.heartbeat_timeout,
+                   sweep_interval=self.sweep_interval)
+
+    async def stop(self) -> None:
+        """Stop the recovery sweeper."""
+        self._running = False
+        if self._task:
+            self._task.cancel()
+            try:
+                await self._task
+            except asyncio.CancelledError:
+                pass
+        logger.info("Recovery sweeper stopped")
+
+    async def _sweep_loop(self) -> None:
+        """Main sweep loop - runs continuously."""
+        while self._running:
+            try:
+                await self._sweep_once()
+            except Exception as e:
+                logger.error("Sweep iteration failed", error=str(e))
+
+            # Wait before next sweep
+            await asyncio.sleep(self.sweep_interval)
+
+    async def _sweep_once(self) -> None:
+        """Single sweep iteration - check all active executions."""
+        # Get all active executions
+        active_ids = await self.cache.get_active_executions()
+
+        if not active_ids:
+            return
+
+        logger.debug("Sweeping active executions", count=len(active_ids))
+
+        for execution_id in active_ids:
+            try:
+                await self._check_execution(execution_id)
+            except Exception as e:
+                logger.error("Failed to check execution",
+                           execution_id=execution_id, error=str(e))
+
+    async def _check_execution(self, execution_id: str) -> None:
+        """Check single execution for stuck nodes.
+
+        Args:
+            execution_id: Execution to check
+        """
+        # Load execution state
+        ctx = await self.cache.load_execution_state(execution_id)
+        if not ctx:
+            # Execution not found - remove from active set
+            logger.warning("Orphan execution in active set",
+                          execution_id=execution_id)
+            if self.cache.cache.is_redis_available():
+                await self.cache.cache.redis.srem("executions:active", execution_id)
+            return
+
+        # Check if already complete
+        if ctx.status in (WorkflowStatus.COMPLETED, WorkflowStatus.FAILED,
+                          WorkflowStatus.CANCELLED):
+            # Should not be in active set
+            if self.cache.cache.is_redis_available():
+                await self.cache.cache.redis.srem("executions:active", execution_id)
+            return
+
+        # Check for stuck nodes
+        needs_recovery = False
+        current_time = time.time()
+
+        for node_id, node_exec in ctx.node_executions.items():
+            if node_exec.status == TaskStatus.RUNNING:
+                # Check heartbeat
+                last_heartbeat = await self.cache.get_heartbeat(execution_id, node_id)
+
+                if last_heartbeat is None:
+                    # No heartbeat - node is stuck
+                    stuck_duration = current_time - (node_exec.started_at or current_time)
+                    if stuck_duration > self.heartbeat_timeout:
+                        logger.warning("Node stuck (no heartbeat)",
+                                      execution_id=execution_id,
+                                      node_id=node_id,
+                                      stuck_seconds=stuck_duration)
+                        needs_recovery = True
+
+                elif current_time - last_heartbeat > self.heartbeat_timeout:
+                    # Heartbeat too old
+                    logger.warning("Node stuck (stale heartbeat)",
+                                  execution_id=execution_id,
+                                  node_id=node_id,
+                                  heartbeat_age=current_time - last_heartbeat)
+                    needs_recovery = True
+
+        if needs_recovery and self._on_recovery:
+            logger.info("Triggering recovery", execution_id=execution_id)
+            try:
+                await self._on_recovery(execution_id)
+            except Exception as e:
+                logger.error("Recovery callback failed",
+                           execution_id=execution_id, error=str(e))
+
+    async def scan_on_startup(self) -> List[str]:
+        """Scan for executions that need recovery on server startup.
+
+        Returns:
+            List of execution IDs that need recovery
+        """
+        needs_recovery = []
+
+        # Get all active executions
+        active_ids = await self.cache.get_active_executions()
+
+        logger.info("Startup scan for incomplete executions",
+                   active_count=len(active_ids))
+
+        for execution_id in active_ids:
+            ctx = await self.cache.load_execution_state(execution_id)
+            if not ctx:
+                continue
+
+            # Check if execution was interrupted
+            if ctx.status == WorkflowStatus.RUNNING:
+                # Check how long it's been stuck
+                if ctx.updated_at:
+                    age = time.time() - ctx.updated_at
+                    if age > self.heartbeat_timeout:
+                        logger.info("Found interrupted execution",
+                                   execution_id=execution_id,
+                                   age_seconds=age)
+                        needs_recovery.append(execution_id)
+
+        return needs_recovery
+
+    async def cleanup_old_executions(self, max_age_hours: int = 24) -> int:
+        """Clean up completed executions older than max_age.
+
+        Args:
+            max_age_hours: Maximum age in hours
+
+        Returns:
+            Number of executions cleaned up
+        """
+        cleaned = 0
+        max_age_seconds = max_age_hours * 3600
+        current_time = time.time()
+
+        # This would need a scan of all execution keys
+        # For now, we rely on Redis TTL for cleanup
+        logger.info("Cleanup skipped - using Redis TTL",
+                   max_age_hours=max_age_hours)
+
+        return cleaned
+
+
+# Global sweeper instance (initialized by main.py)
+_sweeper: Optional[RecoverySweeper] = None
+
+
+def get_recovery_sweeper() -> Optional[RecoverySweeper]:
+    """Get global recovery sweeper instance."""
+    return _sweeper
+
+
+def set_recovery_sweeper(sweeper: RecoverySweeper) -> None:
+    """Set global recovery sweeper instance."""
+    global _sweeper
+    _sweeper = sweeper

package/server/services/handlers/__init__.py

@@ -0,0 +1,126 @@
+"""Node handlers package - extracted from workflow.py for maintainability.
+
+This package contains all node execution handlers organized by category:
+- ai.py: AI Agent, AI Chat Model, Simple Memory
+- android.py: Android Device Setup, Android Services
+- code.py: Python Executor, JavaScript Executor
+- document.py: HTTP Scraper, File Downloader, Document Parser, Text Chunker, Embedding Generator, Vector Store
+- http.py: HTTP Request, Webhook Response
+- tools.py: Tool execution handlers for AI Agent tool calling
+- triggers.py: Generic trigger node handler
+- utility.py: Maps, Text, Chat, Cron, Start
+- whatsapp.py: WhatsApp Send, WhatsApp Connect
+- polyglot.py: Polyglot server integration (standalone, not auto-imported)
+"""
+
+# AI handlers
+from .ai import (
+    handle_ai_agent,
+    handle_chat_agent,
+    handle_ai_chat_model,
+    handle_simple_memory,
+)
+
+# Android handlers
+from .android import (
+    handle_android_device_setup,
+    handle_android_service,
+)
+
+# Code execution handlers
+from .code import (
+    handle_python_executor,
+    handle_javascript_executor,
+)
+
+# HTTP handlers
+from .http import (
+    handle_http_request,
+    handle_webhook_response,
+)
+
+# Trigger handlers
+from .triggers import (
+    handle_trigger_node,
+)
+
+# Utility handlers
+from .utility import (
+    handle_create_map,
+    handle_add_locations,
+    handle_nearby_places,
+    handle_text_generator,
+    handle_file_handler,
+    handle_chat_send,
+    handle_chat_history,
+    handle_start,
+    handle_cron_scheduler,
+    handle_timer,
+    handle_console,
+)
+
+# WhatsApp handlers
+from .whatsapp import (
+    handle_whatsapp_send,
+    handle_whatsapp_connect,
+    handle_whatsapp_db,
+)
+
+# Document processing handlers
+from .document import (
+    handle_http_scraper,
+    handle_file_downloader,
+    handle_document_parser,
+    handle_text_chunker,
+    handle_embedding_generator,
+    handle_vector_store,
+)
+
+# Tool execution handlers (for AI Agent tool calling)
+from .tools import (
+    execute_tool,
+)
+
+__all__ = [
+    # AI
+    'handle_ai_agent',
+    'handle_chat_agent',
+    'handle_ai_chat_model',
+    'handle_simple_memory',
+    # Android
+    'handle_android_device_setup',
+    'handle_android_service',
+    # Code
+    'handle_python_executor',
+    'handle_javascript_executor',
+    # HTTP
+    'handle_http_request',
+    'handle_webhook_response',
+    # Triggers
+    'handle_trigger_node',
+    # Utility
+    'handle_create_map',
+    'handle_add_locations',
+    'handle_nearby_places',
+    'handle_text_generator',
+    'handle_file_handler',
+    'handle_chat_send',
+    'handle_chat_history',
+    'handle_start',
+    'handle_cron_scheduler',
+    'handle_timer',
+    'handle_console',
+    # WhatsApp
+    'handle_whatsapp_send',
+    'handle_whatsapp_connect',
+    'handle_whatsapp_db',
+    # Document processing
+    'handle_http_scraper',
+    'handle_file_downloader',
+    'handle_document_parser',
+    'handle_text_chunker',
+    'handle_embedding_generator',
+    'handle_vector_store',
+    # Tools
+    'execute_tool',
+]