machinaos 0.0.76 → 0.0.78

package/server/services/events/__init__.py

@@ -0,0 +1,54 @@
+"""Generalized event-source framework (Wave 12).
+
+Three concrete EventSource subclasses cover all current MachinaOs
+trigger integrations: ``PushEventSource`` (HTTP/RPC pushes),
+``PollingEventSource`` (interval-based pull), ``DaemonEventSource``
+(long-lived subprocess driver). Webhook flow is a thin specialisation
+of PushEventSource via :class:`WebhookSource` + the path registry in
+``server/routers/webhook.py``.
+
+The unified payload type :class:`WorkflowEvent` mirrors CloudEvents
+v1.0 verbatim — see ``envelope.py``. A back-compat shim in
+``services.event_waiter`` auto-wraps legacy ``Dict`` dispatches so
+existing plugins keep working untouched until they migrate.
+"""
+
+from __future__ import annotations
+
+from .cli import run_cli_command
+from .envelope import WorkflowEvent
+from .lifecycle import make_lifecycle_handlers, make_status_refresh
+from .source import EventSource
+from .push import PushEventSource
+from .polling import PollingEventSource
+from .daemon import DaemonEventSource
+from .triggers import BaseTriggerParams, WebhookTriggerNode
+from .webhook import WebhookSource, WEBHOOK_SOURCES, register_webhook_source
+from .verifiers import (
+    WebhookVerifier,
+    HmacVerifier,
+    StripeVerifier,
+    StandardWebhooksVerifier,
+    GitHubVerifier,
+)
+
+__all__ = [
+    "BaseTriggerParams",
+    "DaemonEventSource",
+    "EventSource",
+    "GitHubVerifier",
+    "HmacVerifier",
+    "PollingEventSource",
+    "PushEventSource",
+    "StandardWebhooksVerifier",
+    "StripeVerifier",
+    "WEBHOOK_SOURCES",
+    "WebhookSource",
+    "WebhookTriggerNode",
+    "WebhookVerifier",
+    "WorkflowEvent",
+    "make_lifecycle_handlers",
+    "make_status_refresh",
+    "register_webhook_source",
+    "run_cli_command",
+]

package/server/services/events/cli.py

@@ -0,0 +1,78 @@
+"""Generic CLI invocation helper.
+
+``run_cli_command`` resolves the binary on PATH, injects the
+plugin's API key via the convention flag (``--api-key`` by default),
+runs the subprocess with a timeout, captures stdout/stderr, and parses
+stdout as JSON when possible. Used by any plugin that wraps a CLI
+tool (Stripe, future GitHub-CLI / Cloudflare-Wrangler / etc.).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import shutil
+from typing import Any, Dict, List, Optional, Type
+
+
+async def run_cli_command(
+    *,
+    binary: str,
+    argv: List[str],
+    credential: Optional[Type] = None,
+    api_key_arg: str = "--api-key",
+    timeout: float = 30.0,
+    env: Optional[Dict[str, str]] = None,
+) -> Dict[str, Any]:
+    """Run ``<binary> <argv> [api_key_arg <key>]`` once, return parsed JSON.
+
+    ``env``: optional process environment override. When None, the child
+    inherits the parent's environment (asyncio default).
+
+    Returns a uniform envelope:
+        {"success": bool, "result": parsed-JSON-or-None,
+         "stdout": str, "stderr": str, "error": str or None}
+    """
+    api_key: Optional[str] = None
+    if credential is not None:
+        try:
+            secrets = await credential.resolve()
+        except PermissionError:
+            return {"success": False, "error": f"{binary}: credential required"}
+        api_key = secrets.get("api_key")
+        if not api_key:
+            return {"success": False, "error": f"{binary}: credential required"}
+
+    resolved = shutil.which(binary)
+    if resolved is None:
+        return {"success": False, "error": f"{binary!r} not on PATH"}
+
+    full_argv = [resolved, *argv]
+    if api_key:
+        full_argv.extend([api_key_arg, api_key])
+
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *full_argv,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            env=env,
+        )
+        stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+    except asyncio.TimeoutError:
+        return {"success": False, "error": f"{binary} timed out ({timeout}s)"}
+
+    out = stdout.decode(errors="replace").strip()
+    err = stderr.decode(errors="replace").strip()
+    if proc.returncode != 0:
+        return {
+            "success": False,
+            "stdout": out,
+            "stderr": err,
+            "error": err or f"exit {proc.returncode}",
+        }
+    try:
+        parsed = json.loads(out) if out else None
+    except json.JSONDecodeError:
+        parsed = None
+    return {"success": True, "result": parsed, "stdout": out, "stderr": err, "error": None}

package/server/services/events/daemon.py

@@ -0,0 +1,163 @@
+"""DaemonEventSource — long-lived subprocess (or SDK loop) that emits
+events into the framework.
+
+Used by stripe-listen, future telegram/whatsapp daemon migrations, and
+any plugin that wraps a CLI tool. The class delegates lifecycle to
+:class:`services.process_service.ProcessService` (already battle-tested:
+PATHEXT-aware, ``kill_tree`` cleanup, log capture, terminal broadcast)
+and subscribes to its standard per-line callback hook for typed event
+emission. We do NOT re-tail the on-disk log files — that's the same
+data ProcessService already streamed to us via ``line_handler``.
+
+Subclasses contribute:
+
+- :meth:`build_command(secrets) -> str` — full command string
+- :meth:`parse_line(stream, line) -> Optional[WorkflowEvent]` — turn one
+  line into one event (or ``None`` for log-only lines)
+- :attr:`binary_name` (optional) — pre-flight ``shutil.which`` check
+  with a clear error if the binary isn't on PATH
+"""
+
+from __future__ import annotations
+
+import asyncio
+import shutil
+from pathlib import Path
+from typing import Any, AsyncIterator, ClassVar, Dict, Optional
+
+from core.logging import get_logger
+from services.process_service import get_process_service
+
+from .envelope import WorkflowEvent
+from .source import EventSource
+
+logger = get_logger(__name__)
+
+
+class DaemonEventSource(EventSource):
+    """Long-lived subprocess driver. Subclasses provide:
+
+    * :attr:`process_name` — unique name for ProcessService key
+    * :attr:`binary_name` — the executable to look up via PATH
+    * :meth:`build_command(secrets)` — full command string
+    * :meth:`parse_line(stream, line)` — turn output lines into events
+      (return ``None`` to suppress; e.g. log-only lines)
+    * :meth:`install_hint` (optional) — install URL surfaced in the
+      "binary not on PATH" error
+    """
+
+    process_name: ClassVar[str] = ""
+    binary_name: ClassVar[str] = ""
+    workflow_namespace: ClassVar[str] = "_daemon"
+    install_hint: ClassVar[str] = ""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._pid: Optional[int] = None
+        self._lock = asyncio.Lock()
+        self._queue: asyncio.Queue[WorkflowEvent] = asyncio.Queue()
+
+    @property
+    def pid(self) -> Optional[int]:
+        return self._pid
+
+    def build_command(self, secrets: Dict) -> str:
+        raise NotImplementedError
+
+    def parse_line(self, stream: str, line: str) -> Optional[WorkflowEvent]:
+        """Override to emit events from output lines. ``stream`` is
+        ``"stdout"`` or ``"stderr"``. Returning ``None`` swallows the line."""
+        return None
+
+    def workdir(self) -> Path:
+        from core.config import Settings
+        cwd = Path(Settings().workspace_base_resolved).resolve() / self.workflow_namespace
+        cwd.mkdir(parents=True, exist_ok=True)
+        return cwd
+
+    async def _resolve_secrets(self) -> Dict[str, Any]:
+        if self.credential is None:
+            return {}
+        try:
+            return await self.credential.resolve()
+        except PermissionError:
+            return {}
+
+    async def has_credential(self) -> bool:
+        secrets = await self._resolve_secrets()
+        return bool(secrets.get("api_key"))
+
+    async def status(self) -> Dict[str, Any]:
+        return {
+            "type": self.type,
+            "running": self._started,
+            "pid": self._pid,
+        }
+
+    async def start(self) -> Dict[str, Any]:
+        async with self._lock:
+            if self._started:
+                logger.info("[%s] start() noop — already running (pid=%s)", self.type, self._pid)
+                return {"success": True, "message": "already running", "status": await self.status()}
+
+            logger.info("[%s] start() entered: resolving secrets + checking credential gate", self.type)
+            secrets = await self._resolve_secrets()
+            # Gate via :meth:`has_credential` so subclasses that authenticate
+            # without an ``api_key`` field (Stripe — auth lives in the CLI's
+            # config file, signalled by ``is_logged_in()``) can override the
+            # check. The default ``has_credential`` still tests
+            # ``secrets["api_key"]`` so api-key plugins keep working.
+            if self.credential is not None and not await self.has_credential():
+                logger.warning(
+                    "[%s] start() blocked: has_credential() returned False — refusing to spawn daemon",
+                    self.type,
+                )
+                return {"success": False, "error": f"{self.type}: credential required"}
+
+            if self.binary_name and shutil.which(self.binary_name) is None:
+                hint = f" Install: {self.install_hint}" if self.install_hint else ""
+                return {"success": False, "error": f"{self.binary_name!r} not on PATH.{hint}"}
+
+            cmd = self.build_command(secrets)
+            cwd = self.workdir()
+            result = await get_process_service().start(
+                name=self.process_name,
+                command=cmd,
+                workflow_id=self.workflow_namespace,
+                working_directory=str(cwd),
+                line_handler=self._on_line,
+            )
+            if not result.get("success"):
+                return result
+
+            self._pid = (result.get("result") or {}).get("pid")
+            self._started = True
+            self._stopped = False
+            logger.info("[%s] daemon started pid=%s", self.type, self._pid)
+            return {"success": True, "message": f"started (pid {self._pid})", "status": await self.status()}
+
+    async def stop(self) -> Dict[str, Any]:
+        async with self._lock:
+            self._stopped = True
+            await get_process_service().stop(
+                name=self.process_name, workflow_id=self.workflow_namespace,
+            )
+            self._started = False
+            self._pid = None
+            return {"success": True, "message": "disconnected"}
+
+    async def restart(self) -> Dict[str, Any]:
+        await self.stop()
+        return await self.start()
+
+    async def _on_line(self, stream: str, line: str) -> None:
+        """ProcessService callback: one decoded stdout/stderr line.
+        Subclasses' :meth:`parse_line` decides whether to emit an event."""
+        event = self.parse_line(stream, line)
+        if event is not None:
+            await self._queue.put(event)
+
+    async def emit(self) -> AsyncIterator[WorkflowEvent]:
+        while not self._stopped:
+            event = await self._queue.get()
+            yield event

package/server/services/events/envelope.py

@@ -0,0 +1,281 @@
+"""WorkflowEvent — CloudEvents v1.0 envelope (in-house, no external dep).
+
+Field set mirrors https://github.com/cloudevents/spec/blob/v1.0.2/cloudevents/spec.md
+verbatim so future interop with EventBridge / Knative is a JSON-schema swap.
+The MachinaOs extensions (workflow_id, trigger_node_id, correlation_id)
+ride as CloudEvents extension attributes — fully compliant with the spec.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any, Literal, Mapping, Optional
+from uuid import uuid4
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class WorkflowEvent(BaseModel):
+    """Unified event envelope used by every EventSource."""
+
+    specversion: str = "1.0"
+    id: str = Field(default_factory=lambda: uuid4().hex)
+    source: str
+    type: str
+    time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    subject: Optional[str] = None
+    datacontenttype: str = "application/json"
+    dataschema: Optional[str] = None
+    data: Any = None
+
+    workflow_id: Optional[str] = None
+    trigger_node_id: Optional[str] = None
+    correlation_id: Optional[str] = None
+
+    model_config = ConfigDict(extra="allow")
+
+    @classmethod
+    def from_legacy(cls, event_type: str, payload: dict) -> "WorkflowEvent":
+        """Wrap a legacy Dict payload from the pre-framework dispatch path.
+
+        Used by the back-compat shim in event_waiter.dispatch so existing
+        plugins (telegram, whatsapp, gmail, etc.) keep working until they
+        migrate to native WorkflowEvent emission.
+        """
+        return cls(
+            source=f"legacy://{event_type}",
+            type=event_type,
+            data=payload,
+        )
+
+    # ---- Typed factory classmethods (Wave 11.I, milestone Q) -----------
+    #
+    # Mirrors the official `cloudevents` Python SDK's `from_http` /
+    # `to_http` convenience pattern. Each factory enforces the same
+    # source/type/subject conventions across the codebase so callers
+    # don't hand-construct envelopes with drifting URI shapes.
+
+    @classmethod
+    def credential(
+        cls,
+        provider: str,
+        action: Literal[
+            "api_key.saved",
+            "api_key.deleted",
+            "api_key.validated",
+            "oauth.connected",
+            "oauth.disconnected",
+            "oauth.validated",
+        ],
+        **extra: Any,
+    ) -> "WorkflowEvent":
+        """Credential mutation event (matches the existing
+        ``broadcast_credential_event`` contract locked by
+        ``test_credential_broadcasts.py``)."""
+        return cls(
+            source="machinaos://services/credentials",
+            type=f"credential.{action}",
+            subject=provider,
+            data={"provider": provider, **extra} if extra else {"provider": provider},
+        )
+
+    @classmethod
+    def connection_status(
+        cls,
+        plugin: str,
+        *,
+        connected: bool,
+        subject: Optional[str] = None,
+        data: Optional[Mapping[str, Any]] = None,
+    ) -> "WorkflowEvent":
+        """Plugin connection-state event (whatsapp / telegram / android-relay
+        / twitter / google connect-disconnect)."""
+        return cls(
+            source=f"machinaos://nodes/{plugin}",
+            type=f"{plugin}.connection.{'opened' if connected else 'closed'}",
+            subject=subject,
+            data=dict(data) if data else {},
+        )
+
+    @classmethod
+    def oauth_completed(
+        cls,
+        provider: str,
+        *,
+        identifier: str,
+        data: Optional[Mapping[str, Any]] = None,
+    ) -> "WorkflowEvent":
+        """OAuth callback completion. ``identifier`` is the user-facing
+        handle (email / username) used as ``subject``."""
+        return cls(
+            source=f"machinaos://nodes/{provider}",
+            type=f"{provider}.oauth.completed",
+            subject=identifier,
+            data=dict(data) if data else {"identifier": identifier},
+        )
+
+    @classmethod
+    def message(
+        cls,
+        plugin: str,
+        direction: Literal["sent", "received"],
+        data: Mapping[str, Any],
+    ) -> "WorkflowEvent":
+        """Plugin-emitted message event (whatsapp / telegram / email
+        send + receive). ``subject`` defaults to the chat / sender id
+        in the payload (cast to str — Telegram uses numeric chat ids),
+        falling back to None."""
+        payload = dict(data)
+        raw_subject = (
+            payload.get("chat_id") or payload.get("from_id") or payload.get("sender")
+        )
+        return cls(
+            source=f"machinaos://nodes/{plugin}",
+            type=f"{plugin}.message.{direction}",
+            subject=str(raw_subject) if raw_subject is not None else None,
+            data=payload,
+        )
+
+    @classmethod
+    def team_event(
+        cls,
+        team_id: str,
+        kind: str,
+        data: Mapping[str, Any],
+    ) -> "WorkflowEvent":
+        """Agent-team lifecycle / task event (created / dissolved /
+        task.added / task.claimed / task.completed / task.failed /
+        message.sent)."""
+        return cls(
+            source="machinaos://services/agent_team",
+            type=f"team.{kind}",
+            subject=team_id,
+            data=dict(data),
+        )
+
+    @classmethod
+    def workflow_lifecycle(
+        cls,
+        stage: Literal[
+            "deployment.started",
+            "deployment.stopped",
+            "lock.acquired",
+            "lock.released",
+            "execution.started",
+            "execution.stopped",
+        ],
+        *,
+        workflow_id: str,
+        data: Optional[Mapping[str, Any]] = None,
+    ) -> "WorkflowEvent":
+        """Workflow lifecycle event. Carries ``workflow_id`` both as
+        ``subject`` (CloudEvents convention) and as the
+        ``workflow_id`` extension attribute (existing reader
+        contract)."""
+        return cls(
+            source="machinaos://services/workflow",
+            type=f"workflow.{stage}",
+            subject=workflow_id,
+            workflow_id=workflow_id,
+            data=dict(data) if data else {},
+        )
+
+    @classmethod
+    def agent_progress(
+        cls,
+        node_id: str,
+        *,
+        workflow_id: Optional[str],
+        iteration: int,
+        max_iterations: int,
+        phase: Optional[str] = None,
+        data: Optional[Mapping[str, Any]] = None,
+    ) -> "WorkflowEvent":
+        """Live LangGraph supervised-loop progress for one agent node.
+
+        Emitted from inside the ``astream`` loop in
+        ``services/ai.py:execute_agent`` and ``execute_chat_agent`` after
+        each super-step. ``iteration`` advances on every ``agent_node``
+        invocation; ``max_iterations`` mirrors the LangGraph
+        ``recursion_limit`` (sourced from
+        ``llm_defaults.json:agent.recursion_limit``).
+
+        ``subject`` carries the executing node id so the FE routes the
+        update straight to ``nodeStatusStore`` for that node. The
+        ``workflow_id`` extension attribute scopes per-workflow displays
+        the same way ``node_status`` broadcasts do.
+        """
+        return cls(
+            source="machinaos://services/agent",
+            type="agent.progress",
+            subject=node_id,
+            workflow_id=workflow_id,
+            data={
+                "node_id": node_id,
+                "iteration": iteration,
+                "max_iterations": max_iterations,
+                **({"phase": phase} if phase else {}),
+                **(dict(data) if data else {}),
+            },
+        )
+
+    @classmethod
+    def deployment_snapshot(
+        cls,
+        running_workflow_ids: list[str],
+    ) -> "WorkflowEvent":
+        """Push-on-connect snapshot of every currently-running deployment.
+
+        Emitted by ``broadcaster.broadcast_deployment_snapshot()`` to a
+        single WebSocket target right after ``initial_status``. Lets the
+        FE reconcile its local ``deploymentStatus`` / ``runningWorkflows``
+        cache against the backend's source of truth on every (re)connect,
+        instead of carrying stale "isRunning=true" forward through a
+        backend restart that wiped the in-memory deployment dict.
+
+        Distinct from ``workflow_lifecycle("deployment.started")`` —
+        that fires when a deployment STARTS (one-shot edge event).
+        ``deployment.snapshot`` is an idempotent state dump tied to
+        client connect, not to a state transition. Empty list is
+        meaningful: "no deployments are running, drop your stale
+        local state."
+        """
+        return cls(
+            source="machinaos://services/workflow",
+            type="workflow.deployment.snapshot",
+            data={"running_workflow_ids": list(running_workflow_ids)},
+        )
+
+    @classmethod
+    def task_completed(
+        cls,
+        task_id: str,
+        *,
+        status: Literal["completed", "error"],
+        agent: str,
+        data: Optional[Mapping[str, Any]] = None,
+    ) -> "WorkflowEvent":
+        """Delegated child-agent completion. Type discriminates on
+        succeeded vs failed; ``taskTrigger`` filters by both."""
+        return cls(
+            source="machinaos://services/agent",
+            type=f"agent.task.{'succeeded' if status == 'completed' else 'failed'}",
+            subject=task_id,
+            data=dict(data) if data else {"task_id": task_id, "agent": agent, "status": status},
+        )
+
+    def matches_type(self, pattern: str) -> bool:
+        """Glob-style match on event type. ``"all"``/empty matches any.
+
+        Examples:
+            "stripe.charge.succeeded" matches itself
+            "stripe.charge.*"        matches "stripe.charge.succeeded"
+            "stripe.*"               matches "stripe.charge.succeeded"
+            "all" or ""              matches everything
+        """
+        if not pattern or pattern == "all":
+            return True
+        if pattern.endswith(".*"):
+            prefix = pattern[:-2]
+            return self.type.startswith(prefix + ".") or self.type == prefix
+        return self.type == pattern

package/server/services/events/lifecycle.py

@@ -0,0 +1,99 @@
+"""Generic lifecycle wiring helpers.
+
+:func:`make_lifecycle_handlers` returns the standard 4 WebSocket
+handlers (``connect`` / ``disconnect`` / ``reconnect`` / ``status``) for
+any :class:`EventSource`. Plugins call it once and register the dict —
+no per-handler boilerplate.
+
+:func:`make_status_refresh` returns a ``register_service_refresh``
+callback that auto-reconnects the source when a credential is stored
+and mirrors its status into the broadcaster cache.
+
+Together these collapse ~50 LOC of identical boilerplate per plugin
+into two function calls.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Awaitable, Callable, Dict, TYPE_CHECKING
+
+from fastapi import WebSocket
+
+from core.logging import get_logger
+
+from .source import EventSource
+
+if TYPE_CHECKING:
+    from services.status_broadcaster import StatusBroadcaster
+
+logger = get_logger(__name__)
+
+
+def make_lifecycle_handlers(
+    prefix: str,
+    source: EventSource,
+    *,
+    extra: Dict[str, Callable[[Dict[str, Any], WebSocket], Awaitable[Dict[str, Any]]]] | None = None,
+) -> Dict[str, Callable[[Dict[str, Any], WebSocket], Awaitable[Dict[str, Any]]]]:
+    """Return ``{prefix}_connect / _disconnect / _reconnect / _status`` handlers.
+
+    Pass ``extra`` to merge additional plugin-specific handlers (e.g.
+    ``{"stripe_trigger": handle_stripe_trigger}``).
+    """
+
+    async def _connect(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+        return await source.start()
+
+    async def _disconnect(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+        return await source.stop()
+
+    async def _reconnect(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+        restart = getattr(source, "restart", None)
+        if restart is not None:
+            return await restart()
+        await source.stop()
+        return await source.start()
+
+    async def _status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+        status = await source.status()
+        if hasattr(source, "has_credential"):
+            status["has_stored_key"] = await source.has_credential()  # type: ignore[attr-defined]
+        return {"success": True, "status": status}
+
+    handlers: Dict[str, Callable[[Dict[str, Any], WebSocket], Awaitable[Dict[str, Any]]]] = {
+        f"{prefix}_connect": _connect,
+        f"{prefix}_disconnect": _disconnect,
+        f"{prefix}_reconnect": _reconnect,
+        f"{prefix}_status": _status,
+    }
+    if extra:
+        handlers.update(extra)
+    return handlers
+
+
+def make_status_refresh(
+    source: EventSource,
+    *,
+    status_key: str,
+    broadcast_type: str,
+) -> Callable[["StatusBroadcaster"], Awaitable[None]]:
+    """Return a ``register_service_refresh`` callback.
+
+    The callback auto-starts the source if it has stored credentials,
+    mirrors its status into ``broadcaster._status[status_key]``, and
+    emits a ``broadcast_type`` broadcast.
+    """
+
+    async def refresh(broadcaster: "StatusBroadcaster") -> None:
+        try:
+            if not getattr(source, "_started", False) and hasattr(source, "has_credential"):
+                if await source.has_credential():  # type: ignore[attr-defined]
+                    logger.info("[StatusBroadcaster] auto-reconnecting %s", status_key)
+                    await source.start()
+            status = await source.status()
+            broadcaster._status[status_key] = status
+            await broadcaster.broadcast({"type": broadcast_type, "data": status})
+        except Exception as e:
+            logger.debug("[StatusBroadcaster] %s refresh failed: %s", status_key, e)
+
+    return refresh