machinaos 0.0.76 → 0.0.78

package/server/services/status_broadcaster.py

@@ -96,15 +96,22 @@ class StatusBroadcaster:
     async def connect(self, websocket: WebSocket):
         """Accept a new WebSocket connection.
 
-        Sends cached status immediately so the client is unblocked,
-        then refreshes service statuses in a background task.
+        Sends the cached status snapshot immediately. Status changes
+        flow in via the originating code path (WhatsApp Go-service
+        events, Telegram bot connect/disconnect, OAuth callbacks,
+        Android relay events) -- the cache is kept fresh by those
+        event-driven broadcasts, so the connecting client doesn't need
+        a per-connect refresh-all storm.
+
+        Initial cache population (and load-bearing auto-reconnects for
+        Telegram + Android relay) happens in a one-time lifespan-startup
+        invocation of :meth:`_refresh_all_services` from ``main.py``.
         """
         await websocket.accept()
         async with self._lock:
             self._connections.add(websocket)
         logger.info(f"[StatusBroadcaster] Client connected. Total: {len(self._connections)}")
 
-        # Send cached status immediately -- don't block on service refreshes
         try:
             await websocket.send_json({
                 "type": "initial_status",
@@ -113,8 +120,45 @@ class StatusBroadcaster:
         except Exception as e:
             logger.error(f"[StatusBroadcaster] Failed to send initial status: {e}")
 
-        # Refresh service statuses in background -- updates broadcast when ready
-        asyncio.create_task(self._refresh_all_services())
+        # Reconcile snapshot — every fresh client gets the
+        # currently-running-deployments truth so a stale FE
+        # `deploymentStatus.isRunning=true` (carried forward through a
+        # backend restart that wiped DeploymentManager._deployments)
+        # gets cleared. CloudEvents-shaped envelope; empty list is
+        # meaningful and triggers FE-side reset.
+        try:
+            await self._send_deployment_snapshot(websocket)
+        except Exception as e:
+            logger.error(f"[StatusBroadcaster] Failed to send deployment snapshot: {e}")
+
+    async def _send_deployment_snapshot(self, websocket: WebSocket) -> None:
+        """Build + send a CloudEvents deployment_snapshot to one client.
+
+        Lazy container resolution because the broadcaster is constructed
+        before WorkflowService and we do not want a circular dependency
+        at module-load time.
+        """
+        try:
+            from core.container import container
+            from services.events import WorkflowEvent
+
+            workflow_service = container.workflow_service()
+            dm = workflow_service._get_deployment_manager()
+            running_ids = dm.get_deployed_workflows()
+        except Exception as e:
+            # Backend startup race or DI not wired yet -- skip the snapshot
+            # rather than fail the connect entirely. The empty-list path
+            # would still be safer than crashing here.
+            logger.debug(
+                f"[StatusBroadcaster] DeploymentManager unavailable for snapshot: {e}",
+            )
+            return
+
+        event = WorkflowEvent.deployment_snapshot(running_ids)
+        await websocket.send_json({
+            "type": "deployment_snapshot",
+            "data": event.model_dump(mode="json"),
+        })
 
     async def disconnect(self, websocket: WebSocket):
         """Remove a WebSocket connection."""
@@ -123,20 +167,21 @@ class StatusBroadcaster:
         logger.info(f"[StatusBroadcaster] Client disconnected. Total: {len(self._connections)}")
 
     async def _refresh_all_services(self):
-        """Spawn per-service refresh tasks; each broadcasts as soon as
-        its own status is ready.
+        """Fan out plugin-registered refresh callbacks.
 
         Uses ``asyncio.TaskGroup`` (Python 3.11+) for structured
         concurrency: every refresh runs as an independent task and
         publishes its own ``<service>_status`` message the moment its
         cache slot is populated. The slowest service no longer gates
-        the others — the credentials/status UI hydrates incrementally.
+        the others -- the credentials/status UI hydrates incrementally.
 
-        Plugin packages register their own refresh callbacks via
-        :func:`register_service_refresh` (see
-        ``nodes/<group>/__init__.py``). The hardcoded core refreshes
-        below predate the registry; they'll move into their own plugin
-        folders in subsequent waves.
+        Wave 11.I, milestone J: every refresher now lives in its
+        plugin folder (``nodes/<plugin>/_refresh.py``) and registers
+        via :func:`register_service_refresh`. The broadcaster has zero
+        per-plugin knowledge in this dispatch loop. Wave 11.I, V: this
+        method runs ONCE at FastAPI lifespan startup (no longer per
+        WS-client connect). State changes after that flow through the
+        originating code path's event-driven broadcasts.
 
         Each callback swallows its own exceptions, so TaskGroup never
         sees one. Kept inside try/except* defensively in case a future
@@ -145,11 +190,6 @@ class StatusBroadcaster:
         with tracer.start_as_current_span("broadcaster.refresh_all_services"):
             try:
                 async with asyncio.TaskGroup() as tg:
-                    tg.create_task(self._refresh_whatsapp_status())
-                    tg.create_task(self._refresh_twitter_status())
-                    tg.create_task(self._refresh_google_status())
-                    tg.create_task(self._auto_reconnect_android_relay())
-                    # Plugin-registered refreshes (telegram, future ones)
                     for callback in list(_SERVICE_REFRESH_CALLBACKS):
                         tg.create_task(callback(self))
             except* Exception as eg:
@@ -230,6 +270,87 @@ class StatusBroadcaster:
         """Get API key validation status for a provider."""
         return self._status["api_keys"].get(provider)
 
+    # =========================================================================
+    # Credential Mutation Broadcasts (CloudEvents v1.0)
+    # =========================================================================
+    #
+    # Every backend handler that mutates credential state (store / remove API
+    # keys, OAuth login / logout) MUST emit a credential event via this helper
+    # so the frontend `useCatalogueQuery` cache stays coherent across all
+    # connected clients. Wire-format key stays `credential_catalogue_updated`
+    # for FE back-compat; the body is the CloudEvents-shaped `WorkflowEvent`
+    # so future EventBridge / Knative interop is a JSON-schema swap.
+    #
+    # Pytest invariant `test_credential_broadcasts.py` locks the contract.
+
+    async def broadcast_credential_event(
+        self,
+        event_type: str,
+        *,
+        provider: str,
+        customer_id: Optional[str] = None,
+    ) -> None:
+        """Emit a CloudEvents-typed credential-mutation broadcast.
+
+        Args:
+            event_type: CloudEvents `type` field. Convention:
+                ``"credential.api_key.saved"`` / ``".deleted"``,
+                ``"credential.oauth.disconnected"``.
+            provider: Provider id (e.g. ``"openai"``, ``"twitter"``).
+            customer_id: For multi-tenant OAuth flows. Default omitted.
+        """
+        # Local import keeps the broadcaster module independent of the
+        # event framework's load order during startup.
+        from services.events import WorkflowEvent
+
+        event = WorkflowEvent(
+            source="machinaos://services/credentials",
+            type=event_type,
+            subject=provider,
+            data={
+                "provider": provider,
+                **({"customer_id": customer_id} if customer_id else {}),
+            },
+        )
+
+        await self.broadcast({
+            "type": "credential_catalogue_updated",
+            "data": event.model_dump(mode="json"),
+        })
+
+    async def broadcast_agent_progress(
+        self,
+        node_id: str,
+        *,
+        workflow_id: Optional[str],
+        iteration: int,
+        max_iterations: int,
+        phase: Optional[str] = None,
+    ) -> None:
+        """Emit a CloudEvents-typed agent-progress event.
+
+        Wire key ``agent_progress`` is a parallel channel to
+        ``node_status``: same per-node scope, but the inner payload is a
+        full CloudEvents envelope instead of a raw dict. The FE routes
+        envelope.data into ``nodeStatusStore`` so the existing
+        ``useNodeStatus`` consumers see ``iteration`` /
+        ``max_iterations`` without a separate store.
+        """
+        from services.events import WorkflowEvent
+
+        event = WorkflowEvent.agent_progress(
+            node_id,
+            workflow_id=workflow_id,
+            iteration=iteration,
+            max_iterations=max_iterations,
+            phase=phase,
+        )
+
+        await self.broadcast({
+            "type": "agent_progress",
+            "data": event.model_dump(mode="json"),
+        })
+
     # =========================================================================
     # Android Status Updates
     # =========================================================================
@@ -263,218 +384,14 @@ class StatusBroadcaster:
         })
 
     # =========================================================================
-    # WhatsApp Status Updates
+    # Status updates -- per-service refresh bodies live in their plugin
+    # folders (Wave 11.I, milestone J): nodes/<plugin>/_refresh.py.
+    # The plugin's __init__.py self-registers via
+    # services.status_broadcaster.register_service_refresh; the central
+    # _refresh_all_services fans out to every registered callback with
+    # zero per-plugin knowledge.
     # =========================================================================
 
-    async def _refresh_whatsapp_status(self):
-        """Fetch fresh WhatsApp status from Go service and update cache.
-
-        Called on client connect to ensure initial_status has accurate data.
-        Silently fails if WhatsApp service is unavailable. Span emitted
-        via OTel for cold-start benchmarking.
-        """
-        with tracer.start_as_current_span("broadcaster.refresh_whatsapp") as span:
-            try:
-                from services.whatsapp_service import get_client
-                import time
-
-                client = await get_client()
-                status_data = await client.call("status")
-
-                self._status["whatsapp"] = {
-                    "connected": status_data.get("connected", False),
-                    "has_session": status_data.get("has_session", False),
-                    "running": status_data.get("running", False),
-                    "pairing": status_data.get("pairing", False),
-                    "device_id": status_data.get("device_id"),
-                    "qr": None,
-                    "timestamp": time.time()
-                }
-                logger.debug(f"[StatusBroadcaster] Refreshed WhatsApp status: connected={status_data.get('connected')}")
-                await self.broadcast({
-                    "type": "whatsapp_status",
-                    "data": self._status["whatsapp"],
-                })
-                span.set_attribute("connected", bool(status_data.get("connected", False)))
-            except Exception as e:
-                span.record_exception(e)
-                # Don't fail client connection if WhatsApp service is down
-                logger.debug(f"[StatusBroadcaster] Could not refresh WhatsApp status: {e}")
-
-    async def _refresh_twitter_status(self):
-        """Fetch Twitter status from stored OAuth tokens in database.
-
-        Called on client connect to check if user is authenticated with Twitter.
-        Uses OAuth token system (matches REST endpoint and node handlers).
-        Span emitted via OTel for cold-start benchmarking.
-        """
-        with tracer.start_as_current_span("broadcaster.refresh_twitter") as span:
-            try:
-                from core.container import container
-                auth_service = container.auth_service()
-
-                # Get tokens from OAuth system (NOT api_key system)
-                tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
-                if not tokens or not tokens.get("access_token"):
-                    self._status["twitter"] = {
-                        "connected": False,
-                        "username": None,
-                        "user_id": None,
-                        "name": None,
-                        "profile_image_url": None
-                    }
-                else:
-                    # User info is stored in the OAuth token record
-                    email = tokens.get("email", "")  # Stored as "@username"
-                    name = tokens.get("name", "")
-                    username = email.lstrip("@") if email.startswith("@") else email
-
-                    self._status["twitter"] = {
-                        "connected": True,
-                        "username": username or None,
-                        "user_id": None,
-                        "name": name or None,
-                        "profile_image_url": None
-                    }
-                    logger.debug(f"[StatusBroadcaster] Twitter status: connected as @{username}")
-
-                await self.broadcast({
-                    "type": "twitter_status",
-                    "data": self._status["twitter"],
-                })
-                span.set_attribute("connected", bool(self._status["twitter"]["connected"]))
-            except Exception as e:
-                span.record_exception(e)
-                logger.debug(f"[StatusBroadcaster] Could not refresh Twitter status: {e}")
-
-    async def _refresh_google_status(self):
-        """Fetch Google Workspace status from stored OAuth tokens.
-
-        Called on client connect to check if user is authenticated with Google.
-        Span emitted via OTel for cold-start benchmarking.
-        """
-        with tracer.start_as_current_span("broadcaster.refresh_google") as span:
-            try:
-                from core.container import container
-                auth_service = container.auth_service()
-
-                tokens = await auth_service.get_oauth_tokens("google", customer_id="owner")
-                if not tokens or not tokens.get("access_token"):
-                    self._status["google"] = {
-                        "connected": False,
-                        "email": None,
-                        "name": None,
-                    }
-                else:
-                    self._status["google"] = {
-                        "connected": True,
-                        "email": tokens.get("email"),
-                        "name": tokens.get("name"),
-                    }
-                    logger.debug(f"[StatusBroadcaster] Google status: connected as {tokens.get('email')}")
-
-                await self.broadcast({
-                    "type": "google_status",
-                    "data": self._status["google"],
-                })
-                span.set_attribute("connected", bool(self._status["google"]["connected"]))
-            except Exception as e:
-                span.record_exception(e)
-                logger.debug(f"[StatusBroadcaster] Could not refresh Google status: {e}")
-
-    async def _auto_reconnect_android_relay(self):
-        """Auto-reconnect to Android relay if there's a stored pairing session.
-
-        Called on client connect to re-establish relay connection after server restart.
-        The stored session contains relay URL, API key, and paired device info.
-        Span emitted via OTel for cold-start benchmarking.
-        """
-        with tracer.start_as_current_span("broadcaster.refresh_android") as span:
-            await self._auto_reconnect_android_relay_body(span)
-
-    async def _auto_reconnect_android_relay_body(self, span):
-        try:
-            # Check if already connected
-            from services.android.manager import get_current_relay_client
-            existing = get_current_relay_client()
-            if existing and existing.is_connected():
-                # Already connected, just refresh status
-                self._status["android"] = {
-                    "connected": True,
-                    "paired": existing.is_paired(),
-                    "device_id": existing.paired_device_id,
-                    "device_name": existing.paired_device_name,
-                    "connected_devices": list(existing.get_connected_devices()),
-                    "connection_type": "relay",
-                    "qr_data": existing.qr_data,
-                    "session_token": existing.session_token
-                }
-                logger.debug("[StatusBroadcaster] Android relay already connected")
-                await self.broadcast({
-                    "type": "android_status",
-                    "data": self._status["android"],
-                })
-                span.set_attribute("path", "already_connected")
-                return
-
-            # Check for stored session
-            from core.container import container
-            database = container.database()
-
-            session = await database.get_android_relay_session()
-            if not session:
-                span.set_attribute("path", "no_session")
-                logger.debug("[StatusBroadcaster] No stored Android relay session")
-                return
-
-            relay_url = session.get("relay_url")
-            api_key = session.get("api_key")
-            device_id = session.get("device_id")
-            session.get("device_name")
-
-            if not relay_url or not api_key:
-                span.set_attribute("path", "session_missing_creds")
-                logger.debug("[StatusBroadcaster] Stored session missing relay URL or API key")
-                return
-
-            span.set_attribute("path", "auto_reconnect")
-            logger.info("[StatusBroadcaster] Auto-reconnecting to Android relay...",
-                       relay_url=relay_url, device_id=device_id)
-
-            # Attempt to reconnect
-            from services.android.manager import get_relay_client
-            client, error = await get_relay_client(relay_url, api_key)
-
-            if client and client.is_connected():
-                logger.info("[StatusBroadcaster] Android relay reconnected successfully")
-                # Update status - connected to relay but need to check if still paired
-                # The relay server creates a new session on each connect, so pairing is lost
-                # Update the cached status to reflect the current state
-                self._status["android"] = {
-                    "connected": True,
-                    "paired": client.is_paired(),
-                    "device_id": client.paired_device_id,
-                    "device_name": client.paired_device_name,
-                    "connected_devices": list(client.get_connected_devices()),
-                    "connection_type": "relay",
-                    "qr_data": client.qr_data,
-                    "session_token": client.session_token
-                }
-                await self.broadcast({
-                    "type": "android_status",
-                    "data": self._status["android"],
-                })
-                span.set_attribute("reconnect_ok", True)
-            else:
-                span.set_attribute("reconnect_ok", False)
-                logger.warning(f"[StatusBroadcaster] Failed to reconnect Android relay: {error}")
-                # Clear the stored session since reconnect failed
-                await database.clear_android_relay_session()
-
-        except Exception as e:
-            span.record_exception(e)
-            logger.debug(f"[StatusBroadcaster] Could not auto-reconnect Android relay: {e}")
-
     async def update_whatsapp_status(
         self,
         connected: bool,
@@ -501,10 +418,6 @@ class StatusBroadcaster:
             "data": self._status["whatsapp"]
         })
 
-    def get_whatsapp_status(self) -> Dict[str, Any]:
-        """Get WhatsApp connection status."""
-        return self._status["whatsapp"].copy()
-
     # =========================================================================
     # Telegram Status Updates
     # =========================================================================
@@ -533,10 +446,6 @@ class StatusBroadcaster:
             "data": self._status["telegram"]
         })
 
-    def get_telegram_status(self) -> Dict[str, Any]:
-        """Get Telegram bot connection status."""
-        return self._status["telegram"].copy()
-
     # =========================================================================
     # Node Status Updates
     # =========================================================================
@@ -1221,6 +1130,15 @@ _ServiceRefreshCallback = _typing.Callable[
 ]
 _SERVICE_REFRESH_CALLBACKS: _typing.List[_ServiceRefreshCallback] = []
 
+from services.plugin.registry import IdempotentList as _IdempotentList  # noqa: E402
+
+# Backed by the module-level _SERVICE_REFRESH_CALLBACKS list so the
+# existing iterator at line 153 (``for callback in list(_SERVICE_REFRESH_CALLBACKS)``)
+# keeps working unchanged.
+_SERVICE_REFRESH_FANOUT: _IdempotentList[_ServiceRefreshCallback] = _IdempotentList(
+    "service_refresh", items=_SERVICE_REFRESH_CALLBACKS
+)
+
 
 def register_service_refresh(callback: _ServiceRefreshCallback) -> None:
     """Register a per-service refresh callback.
@@ -1229,8 +1147,7 @@ def register_service_refresh(callback: _ServiceRefreshCallback) -> None:
     callback runs once per ``_refresh_all_services()`` cycle (i.e. on
     every WebSocket client connect).
     """
-    if callback not in _SERVICE_REFRESH_CALLBACKS:
-        _SERVICE_REFRESH_CALLBACKS.append(callback)
+    _SERVICE_REFRESH_FANOUT.register(callback)
 
 
 # Global singleton instance

package/server/services/temporal/workflow.py

@@ -25,13 +25,13 @@ CONFIG_HANDLES = {"input-tools", "input-memory", "input-model", "input-skill", "
 # Trigger node types — event listeners that should never be scheduled
 # as blocking activities. Imported from constants to avoid drift (was
 # previously redefined here with a "keep in sync" comment — Wave 11.E.2).
-from constants import WORKFLOW_TRIGGER_TYPES as TRIGGER_NODE_TYPES
-
-# Android service types (connect to androidTool, not executed directly)
-ANDROID_SERVICE_TYPES = {
-    "batteryMonitor", "locationService", "deviceState",
-    "systemInfo", "appList", "appLauncher",
-}
+# Android service types follow the same pattern: imported from constants
+# so the canonical 16-entry list (Wave 11.I, milestone P -- the local
+# 6-entry copy that lived here was a stale subset).
+from constants import (
+    ANDROID_SERVICE_NODE_TYPES as ANDROID_SERVICE_TYPES,
+    WORKFLOW_TRIGGER_TYPES as TRIGGER_NODE_TYPES,
+)
 
 # Skill node types (connect to Zeenie's input-skill, not executed directly)
 SKILL_NODE_TYPES = {

package/server/services/workflow.py

@@ -27,9 +27,9 @@ if TYPE_CHECKING:
     from core.database import Database
     from core.cache import CacheService
     from services.ai import AIService
-    from services.maps import MapsService
+    from nodes.location._service import MapsService
     from services.text import TextService
-    from services.android_service import AndroidService
+    from nodes.android._dispatcher import AndroidService
     from services.temporal import TemporalExecutor
 
 logger = get_logger(__name__)
@@ -461,7 +461,25 @@ class WorkflowService:
             workflow_id: Specific workflow to cancel. If None, cancels first running deployment.
         """
         manager = self._get_deployment_manager()
-        return await manager.cancel(workflow_id)
+        result = await manager.cancel(workflow_id)
+
+        # Also cancel any in-flight CLI agent batches (claude_code_agent /
+        # codex_agent) for this workflow. Best-effort — failure to cancel
+        # CLI sessions must not block the deployment cancel.
+        if workflow_id:
+            try:
+                from services.cli_agent.service import get_ai_cli_service
+                cli_svc = get_ai_cli_service()
+                cancelled = await cli_svc.cancel_workflow(workflow_id)
+                if cancelled:
+                    logger.info(
+                        "[workflow] cancelled %d CLI agent session(s) for workflow %s",
+                        cancelled, workflow_id,
+                    )
+            except Exception as exc:
+                logger.debug("[workflow] CLI agent cancel: %s", exc)
+
+        return result
 
     def get_deployment_status(self, workflow_id: Optional[str] = None) -> Dict[str, Any]:
         """Get deployment status.