machinaos 0.0.1

package/server/services/temporal/worker.py

@@ -0,0 +1,251 @@
+"""Temporal worker for distributed node execution.
+
+Uses class-based activities with shared aiohttp session for proper
+connection pooling across concurrent activity executions.
+
+The worker polls the task queue and executes:
+- MachinaWorkflow: Orchestrates the graph, schedules node activities
+- NodeExecutionActivities: Executes individual nodes with shared session
+
+Multiple workers can be started on different machines for horizontal scaling.
+Each node activity can execute on any available worker in the cluster.
+
+References:
+- https://docs.temporal.io/develop/python/python-sdk-sync-vs-async
+- https://docs.temporal.io/develop/worker-performance
+"""
+
+import asyncio
+from typing import Optional
+
+import aiohttp
+from temporalio.client import Client
+from temporalio.runtime import Runtime, TelemetryConfig
+from temporalio.worker import Worker
+
+from core.logging import get_logger
+from .workflow import MachinaWorkflow
+print(f"[Worker Import] MachinaWorkflow loaded from: {MachinaWorkflow.__module__}")
+from .activities import (
+    NodeExecutionActivities,
+    create_shared_session,
+    execute_node_activity,
+)
+
+logger = get_logger(__name__)
+
+
+def create_runtime() -> Runtime:
+    """Create a Temporal runtime with worker heartbeating disabled.
+
+    Disables the runtime-level worker heartbeating feature to avoid
+    the warning on older Temporal server versions that don't support it.
+    """
+    return Runtime(
+        telemetry=TelemetryConfig(),
+        worker_heartbeat_interval=None,  # Disable runtime heartbeating
+    )
+
+
+class TemporalWorkerManager:
+    """Manages the Temporal worker lifecycle with shared resources.
+
+    Creates a shared aiohttp.ClientSession that is passed to the activity
+    class, following Temporal's recommended dependency injection pattern.
+    """
+
+    def __init__(
+        self,
+        client: Client,
+        task_queue: str = "machina-tasks",
+        pool_size: int = 100,
+    ):
+        """Initialize the worker manager.
+
+        Args:
+            client: Connected Temporal client
+            task_queue: Task queue name to poll
+            pool_size: Connection pool size for aiohttp session
+        """
+        self.client = client
+        self.task_queue = task_queue
+        self.pool_size = pool_size
+        self._worker: Optional[Worker] = None
+        self._worker_task: Optional[asyncio.Task] = None
+        self._session: Optional[aiohttp.ClientSession] = None
+        self._activities: Optional[NodeExecutionActivities] = None
+
+    @property
+    def is_running(self) -> bool:
+        """Check if the worker is running."""
+        return self._worker_task is not None and not self._worker_task.done()
+
+    async def start(self) -> None:
+        """Start the Temporal worker in the background."""
+        if self.is_running:
+            logger.warning("Temporal worker already running")
+            return
+
+        # Create shared aiohttp session with connection pooling
+        self._session = await create_shared_session(self.pool_size)
+
+        # Create activity instance with shared session
+        self._activities = NodeExecutionActivities(self._session)
+
+        # Create worker with class-based activity
+        # For class-based activities, pass the bound method (instance.method)
+        self._worker = Worker(
+            self.client,
+            task_queue=self.task_queue,
+            workflows=[MachinaWorkflow],
+            activities=[self._activities.execute_node_activity],  # Pass bound method
+            # Allow concurrent activity execution for parallel branches
+            max_concurrent_activities=self.pool_size,
+            max_concurrent_workflow_tasks=10,
+        )
+
+        logger.info(
+            "Starting Temporal worker",
+            task_queue=self.task_queue,
+            pool_size=self.pool_size,
+        )
+        print(f"[Worker] Starting with pool_size={self.pool_size}")
+
+        # Run worker in background task
+        self._worker_task = asyncio.create_task(
+            self._run_worker(),
+            name="temporal-worker",
+        )
+
+    async def _run_worker(self) -> None:
+        """Run the worker (background task)."""
+        try:
+            await self._worker.run()
+        except asyncio.CancelledError:
+            logger.info("Temporal worker cancelled")
+        except Exception as e:
+            logger.error(f"Temporal worker error: {str(e)}")
+            raise
+
+    async def stop(self) -> None:
+        """Stop the Temporal worker and cleanup resources."""
+        if not self.is_running:
+            return
+
+        logger.info("Stopping Temporal worker")
+
+        if self._worker_task:
+            self._worker_task.cancel()
+            try:
+                await self._worker_task
+            except asyncio.CancelledError:
+                pass
+            self._worker_task = None
+
+        # Close shared session
+        if self._session and not self._session.closed:
+            await self._session.close()
+            print("[Worker] Closed shared session")
+
+        self._worker = None
+        self._session = None
+        self._activities = None
+        logger.info("Temporal worker stopped")
+
+
+async def run_standalone_worker(
+    server_address: str = "localhost:7233",
+    namespace: str = "default",
+    task_queue: str = "machina-tasks",
+    pool_size: int = 100,
+) -> None:
+    """Run the Temporal worker as a standalone process.
+
+    This can be used for running workers separately from the main server,
+    enabling horizontal scaling across multiple machines.
+
+    Example:
+        # Start multiple workers for horizontal scaling
+        python -m services.temporal.worker
+
+    Args:
+        server_address: Temporal server address
+        namespace: Temporal namespace
+        task_queue: Task queue to poll
+        pool_size: Connection pool size
+    """
+    logger.info(
+        "Starting standalone Temporal worker",
+        server_address=server_address,
+        namespace=namespace,
+        task_queue=task_queue,
+    )
+
+    print(f"[Worker] Connecting to {server_address}")
+    print(f"[Worker] Namespace: {namespace}")
+    print(f"[Worker] Task Queue: {task_queue}")
+    print(f"[Worker] Pool Size: {pool_size}")
+
+    # Use custom runtime with heartbeating disabled to avoid warning on older servers
+    runtime = create_runtime()
+    client = await Client.connect(server_address, namespace=namespace, runtime=runtime)
+
+    # Create shared session and activities
+    session = await create_shared_session(pool_size)
+    activities = NodeExecutionActivities(session)
+
+    try:
+        worker = Worker(
+            client,
+            task_queue=task_queue,
+            workflows=[MachinaWorkflow],
+            activities=[activities.execute_node_activity],  # Pass bound method
+            max_concurrent_activities=pool_size,
+            max_concurrent_workflow_tasks=10,
+        )
+
+        print("[Worker] Running. Press Ctrl+C to stop.")
+        logger.info("Worker running. Press Ctrl+C to stop.")
+        await worker.run()
+
+    finally:
+        # Cleanup session on shutdown
+        if not session.closed:
+            await session.close()
+            print("[Worker] Session closed")
+
+
+async def create_worker(
+    client: Client,
+    task_queue: str = "machina-tasks",
+    session: Optional[aiohttp.ClientSession] = None,
+) -> Worker:
+    """Create a worker instance for use in tests or custom setups.
+
+    Args:
+        client: Connected Temporal client
+        task_queue: Task queue name
+        session: Optional shared aiohttp session (created if not provided)
+
+    Returns:
+        Configured Worker instance (not started)
+    """
+    if session is None:
+        session = await create_shared_session()
+
+    activities = NodeExecutionActivities(session)
+
+    return Worker(
+        client,
+        task_queue=task_queue,
+        workflows=[MachinaWorkflow],
+        activities=[activities.execute_node_activity],  # Pass bound method
+        max_concurrent_activities=100,
+        max_concurrent_workflow_tasks=10,
+    )
+
+
+if __name__ == "__main__":
+    # Allow running worker standalone
+    # Usage: python -m services.temporal.worker
+    asyncio.run(run_standalone_worker())

package/server/services/temporal/workflow.py

@@ -0,0 +1,355 @@
+"""Temporal workflow - Pure orchestrator for distributed node execution.
+
+The workflow ONLY orchestrates:
+- Parses graph structure
+- Filters config nodes (tools, memory, services)
+- Determines execution order based on dependencies
+- Schedules node activities (can run on ANY worker)
+- Collects results and routes outputs to dependent nodes
+
+NO business logic in workflow - all execution happens in activities.
+This enables massive horizontal scaling and multi-tenant distribution.
+"""
+
+from datetime import timedelta
+from typing import Any, Dict, List, Set
+
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+
+# Config handles - nodes connecting via these are config nodes (not executed)
+# AI Agent handles: input-memory, input-tools, input-model
+# Chat Agent handles: input-skill, input-tools
+CONFIG_HANDLES = {"input-tools", "input-memory", "input-model", "input-skill"}
+
+# Android service types (connect to androidTool, not executed directly)
+ANDROID_SERVICE_TYPES = {
+    "batteryMonitor", "locationService", "deviceState",
+    "systemInfo", "appList", "appLauncher",
+}
+
+# Skill node types (connect to Chat Agent's input-skill, not executed directly)
+SKILL_NODE_TYPES = {
+    "assistantPersonality", "whatsappSkill", "memorySkill", "mapsSkill",
+    "httpSkill", "schedulerSkill", "androidSkill", "codeSkill", "customSkill",
+}
+
+@workflow.defn(sandboxed=False)
+class MachinaWorkflow:
+    """Distributed workflow orchestrator.
+
+    This workflow ONLY orchestrates - all execution happens in activities
+    that can run on any worker in the cluster.
+
+    Features:
+    - Continuous scheduling (FIRST_COMPLETED pattern)
+    - Per-node retry policies
+    - Config node filtering (tools, memory, services)
+    - Multi-tenant support via tenant_id in context
+    """
+
+    @workflow.run
+    async def run(self, workflow_data: Dict[str, Any]) -> Dict[str, Any]:
+        print("[Workflow] ========== RUN METHOD CALLED ==========")
+        """Execute workflow by orchestrating node activities.
+
+        Args:
+            workflow_data: Dict containing:
+                - nodes: List of node definitions from React Flow
+                - edges: List of edge definitions from React Flow
+                - session_id: Session identifier
+                - workflow_id: Workflow ID for tracking
+                - tenant_id: Tenant identifier for multi-tenancy
+
+        Returns:
+            Dict with success, outputs, execution_trace, and errors
+        """
+        nodes = workflow_data.get("nodes", [])
+        edges = workflow_data.get("edges", [])
+        session_id = workflow_data.get("session_id", "default")
+        workflow_id = workflow_data.get("workflow_id")
+        tenant_id = workflow_data.get("tenant_id")
+
+        workflow.logger.info(
+            f"Starting workflow orchestration: {len(nodes)} nodes, {len(edges)} edges"
+        )
+        # Debug print for console visibility
+        print(f"[Workflow] Starting: {len(nodes)} nodes, {len(edges)} edges")
+
+        if not nodes:
+            return {
+                "success": False,
+                "error": "No nodes provided",
+                "outputs": {},
+                "execution_trace": [],
+            }
+
+        # 1. Filter out config nodes (tools, memory, services)
+        exec_nodes, exec_edges = self._filter_executable_graph(nodes, edges)
+
+        workflow.logger.info(
+            f"After filtering: {len(exec_nodes)} executable nodes "
+            f"(filtered {len(nodes) - len(exec_nodes)} config nodes)"
+        )
+        print(f"[Workflow] Executable: {len(exec_nodes)}, Config filtered: {len(nodes) - len(exec_nodes)}")
+
+        # 2. Build dependency maps
+        deps, node_map = self._build_dependency_maps(exec_nodes, exec_edges)
+
+        # 3. Initialize state
+        outputs: Dict[str, Any] = {}  # node_id -> result
+        completed: Set[str] = set()
+        running: Dict[str, Any] = {}  # node_id -> activity handle
+        errors: List[Dict] = []
+        execution_trace: List[str] = []
+
+        # 4. Handle pre-executed triggers (already have their output)
+        pre_executed_count = 0
+        for node in exec_nodes:
+            if node.get("_pre_executed"):
+                node_id = node["id"]
+                outputs[node_id] = {
+                    "success": True,
+                    "result": node.get("_trigger_output", {}),
+                    "pre_executed": True,
+                }
+                completed.add(node_id)
+                execution_trace.append(node_id)
+                pre_executed_count += 1
+                workflow.logger.info(f"Pre-executed trigger: {node_id}")
+
+        workflow.logger.info(f"Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")
+        print(f"[Workflow] Pre-executed: {pre_executed_count}, To execute: {len(node_map) - pre_executed_count}")
+
+        # 5. Retry policy for node activities
+        retry_policy = RetryPolicy(
+            initial_interval=timedelta(seconds=1),
+            maximum_interval=timedelta(seconds=30),
+            maximum_attempts=3,
+        )
+
+        # 6. Continuous scheduling loop
+        loop_count = 0
+        while True:
+            loop_count += 1
+            # Find ready nodes (all deps completed, not running/completed)
+            ready = self._find_ready_nodes(deps, completed, running, node_map)
+            workflow.logger.info(f"Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")
+            print(f"[Workflow] Loop {loop_count}: ready={len(ready)}, running={len(running)}, completed={len(completed)}")
+
+            # Start activities for ready nodes
+            for node_id in ready:
+                node = node_map[node_id]
+
+                # Build immutable context for this node
+                context = {
+                    "node_id": node_id,
+                    "node_type": node.get("type", "unknown"),
+                    "node_data": node.get("data", {}),
+                    "inputs": self._get_node_inputs(node_id, deps, outputs),
+                    "workflow_id": workflow_id,
+                    "tenant_id": tenant_id,
+                    "session_id": session_id,
+                    "nodes": nodes,  # Full list for tool/memory detection
+                    "edges": edges,  # Full list for tool/memory detection
+                    # Include pre-executed info if applicable
+                    "pre_executed": node.get("_pre_executed", False),
+                    "trigger_output": node.get("_trigger_output"),
+                }
+
+                # Schedule activity by name (for class-based activities)
+                # The activity is registered as NodeExecutionActivities.execute_node_activity
+                handle = workflow.start_activity(
+                    "execute_node_activity",
+                    args=[context],
+                    start_to_close_timeout=timedelta(minutes=10),
+                    heartbeat_timeout=timedelta(minutes=2),
+                    retry_policy=retry_policy,
+                )
+                running[node_id] = handle
+
+                workflow.logger.info(f"Scheduled activity for node: {node_id}")
+                print(f"[Workflow] Scheduled: {node_id}")
+
+            # Exit if nothing running and nothing ready
+            if not running:
+                break
+
+            # Wait for ANY activity to complete (FIRST_COMPLETED pattern)
+            done_id, result = await self._wait_any_complete(running)
+
+            if result.get("success"):
+                outputs[done_id] = result
+                completed.add(done_id)
+                execution_trace.append(done_id)
+                workflow.logger.info(f"Node completed: {done_id}")
+            else:
+                # Node failed after all retries
+                error_info = {
+                    "node_id": done_id,
+                    "error": result.get("error", "Unknown error"),
+                }
+                errors.append(error_info)
+                workflow.logger.error(f"Node failed: {done_id} - {error_info['error']}")
+
+                # Stop workflow on failure
+                # TODO: Could add option to continue with partial results
+                break
+
+        # Build final result
+        success = len(errors) == 0 and len(completed) == len(node_map)
+
+        workflow.logger.info(
+            f"Workflow complete: success={success}, "
+            f"executed={len(execution_trace)}/{len(node_map)}"
+        )
+        print(f"[Workflow] Complete: success={success}, executed={len(execution_trace)}/{len(node_map)}")
+
+        return {
+            "success": success,
+            "outputs": outputs,
+            "execution_trace": execution_trace,
+            "errors": errors if errors else None,
+        }
+
+    def _get_node_inputs(
+        self,
+        node_id: str,
+        deps: Dict[str, Set[str]],
+        outputs: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Get outputs from upstream nodes as inputs for this node."""
+        inputs = {}
+        for dep_id in deps.get(node_id, set()):
+            if dep_id in outputs:
+                inputs[dep_id] = outputs[dep_id].get("result", {})
+        return inputs
+
+    async def _wait_any_complete(self, running: Dict[str, Any]) -> tuple:
+        """Wait for any activity to complete, return (node_id, result).
+
+        Uses Temporal's native wait mechanism for efficient polling.
+        """
+        # Convert to list for iteration
+        items = list(running.items())
+
+        # Check if any already done
+        for node_id, handle in items:
+            if handle.done():
+                del running[node_id]
+                try:
+                    result = await handle
+                    return node_id, result
+                except Exception as e:
+                    return node_id, {"success": False, "error": str(e)}
+
+        # Wait for first completion using Temporal's wait
+        handles = [h for _, h in items]
+
+        # Use asyncio.wait pattern via workflow.wait
+        await workflow.wait_condition(
+            lambda: any(h.done() for _, h in items)
+        )
+
+        # Find the completed one
+        for node_id, handle in items:
+            if handle.done():
+                del running[node_id]
+                try:
+                    result = await handle
+                    return node_id, result
+                except Exception as e:
+                    return node_id, {"success": False, "error": str(e)}
+
+        # Should not reach here
+        raise RuntimeError("No activity completed after wait")
+
+    def _filter_executable_graph(
+        self,
+        nodes: List[Dict],
+        edges: List[Dict],
+    ) -> tuple:
+        """Filter out config nodes based on edge handles.
+
+        Config nodes (tools, memory, model configs) connect via special handles
+        and are consumed by their target nodes, not executed independently.
+
+        Returns:
+            Tuple of (executable_nodes, executable_edges)
+        """
+        node_map = {n["id"]: n for n in nodes}
+        config_ids = set()
+
+        for edge in edges:
+            handle = edge.get("targetHandle", "")
+            source_id = edge.get("source")
+
+            # Edges to config handles mean source is a config node
+            if handle in CONFIG_HANDLES:
+                config_ids.add(source_id)
+
+            # Android services connecting to androidTool
+            source_node = node_map.get(source_id, {})
+            if source_node.get("type") in ANDROID_SERVICE_TYPES:
+                config_ids.add(source_id)
+
+            # Skill nodes (always config, connect to Chat Agent)
+            if source_node.get("type") in SKILL_NODE_TYPES:
+                config_ids.add(source_id)
+
+        # Filter nodes and edges
+        exec_nodes = [n for n in nodes if n["id"] not in config_ids]
+        exec_edges = [
+            e for e in edges
+            if e.get("source") not in config_ids
+            and e.get("target") not in config_ids
+            and e.get("targetHandle", "") not in CONFIG_HANDLES
+        ]
+
+        return exec_nodes, exec_edges
+
+    def _build_dependency_maps(
+        self,
+        nodes: List[Dict],
+        edges: List[Dict],
+    ) -> tuple:
+        """Build dependency graph from nodes and edges.
+
+        Returns:
+            Tuple of (dependencies_map, node_map)
+            - dependencies_map: node_id -> set of node IDs it depends on
+            - node_map: node_id -> node definition
+        """
+        node_map = {n["id"]: n for n in nodes}
+        node_ids = set(node_map.keys())
+
+        deps = {nid: set() for nid in node_ids}
+
+        for edge in edges:
+            src, tgt = edge.get("source"), edge.get("target")
+            if src in node_ids and tgt in node_ids:
+                deps[tgt].add(src)
+
+        return deps, node_map
+
+    def _find_ready_nodes(
+        self,
+        deps: Dict[str, Set[str]],
+        completed: Set[str],
+        running: Dict[str, Any],
+        node_map: Dict[str, Dict],
+    ) -> List[str]:
+        """Find nodes ready to execute.
+
+        A node is ready when:
+        - All its dependencies have completed
+        - It's not already running
+        - It's not already completed
+        """
+        ready = []
+        for node_id in node_map:
+            if node_id in completed or node_id in running:
+                continue
+            if deps[node_id] <= completed:
+                ready.append(node_id)
+        return ready