machinaos 0.0.78 → 0.0.80

package/server/services/temporal/client.py

@@ -65,25 +65,29 @@ class TemporalClientWrapper:
                 )
                 # Verify namespace is ready (gRPC port may accept connections
                 # before the server finishes registering namespaces)
-                await client.service_client.workflow_service.describe_namespace(
-                    DescribeNamespaceRequest(namespace=self.namespace)
-                )
+                await client.service_client.workflow_service.describe_namespace(DescribeNamespaceRequest(namespace=self.namespace))
                 self._client = client
-                print(f"[Temporal] Connected to {self.server_address}", flush=True)
-                logger.info("Connected to Temporal server")
+                logger.info(f"Connected to Temporal server at {self.server_address}")
+                # Wave 12 A4: idempotently register the event-framework
+                # Search Attributes. Failure here is non-fatal — the
+                # framework still works without them, dispatch.emit just
+                # falls back to broadcast-only routing instead of
+                # signalling consumers.
+                try:
+                    from services.temporal.search_attributes import (
+                        register_search_attributes,
+                    )
+
+                    await register_search_attributes(self._client, self.namespace)
+                except Exception as sa_exc:  # noqa: BLE001 — non-fatal
+                    logger.warning(f"Search-attribute registration failed (non-fatal): {sa_exc}")
                 return self._client
             except Exception as e:
-                print(f"[Temporal] Connection attempt {attempt}/{retries} failed: {e}", flush=True)
-                logger.warning(
-                    f"Temporal connection attempt {attempt}/{retries} failed: {e}"
-                )
+                logger.warning(f"Temporal connection attempt {attempt}/{retries} failed: {e}")
                 if attempt < retries:
                     await asyncio.sleep(delay)
 
-        print(f"[Temporal] Failed to connect after {retries} attempts", flush=True)
-        logger.error(
-            f"Failed to connect to Temporal server at {self.server_address} after {retries} attempts"
-        )
+        logger.error(f"Failed to connect to Temporal server at {self.server_address} after {retries} attempts")
         return None
 
     async def disconnect(self) -> None:
@@ -91,3 +95,44 @@ class TemporalClientWrapper:
         if self._client is not None:
             self._client = None
             logger.info("Disconnected from Temporal server")
+
+    async def terminate_running_workflows(
+        self,
+        *,
+        reason: str = "MachinaOS startup: auto-resumption disabled",
+    ) -> int:
+        """Terminate every workflow in ``Running`` state in our namespace.
+
+        Preserves history — terminated workflows remain visible in the
+        Temporal UI as ``Terminated``, only the active execution stops.
+        Run from ``main.py`` lifespan between client connect and worker
+        start so the worker doesn't accept activities from workflows
+        that are about to be terminated. Gated by
+        ``Settings.temporal_terminate_running_on_startup`` at the call
+        site; this method always terminates when invoked.
+
+        Returns the count of workflows terminated. Failures on
+        individual workflows are logged but don't abort the sweep —
+        terminating a workflow that completed mid-query is a benign
+        race that produces ``WorkflowNotFoundError``.
+        """
+        if self._client is None:
+            logger.warning("terminate_running_workflows called before connect; no-op")
+            return 0
+
+        count = 0
+        async for wf in self._client.list_workflows(
+            "ExecutionStatus = 'Running'",
+        ):
+            try:
+                handle = self._client.get_workflow_handle(
+                    wf.id,
+                    run_id=wf.run_id,
+                )
+                await handle.terminate(reason=reason)
+                count += 1
+            except Exception as exc:  # noqa: BLE001 — best-effort sweep
+                logger.debug(f"Failed to terminate workflow id={wf.id} " f"run_id={wf.run_id}: {exc}")
+        if count:
+            logger.info(f"Terminated {count} running workflow(s) at startup " "(history preserved; resumption disabled)")
+        return count

package/server/services/temporal/executor.py

@@ -98,9 +98,7 @@ class TemporalExecutor:
                             result.get("outputs", {}).get(node_id, {}),
                         )
                     except Exception as e:
-                        logger.warning(
-                            f"Status callback error for node {node_id}: {e}"
-                        )
+                        logger.warning(f"Status callback error for node {node_id}: {e}")
 
             logger.info(
                 "Temporal workflow completed",
@@ -124,6 +122,7 @@ class TemporalExecutor:
 
         except Exception as e:
             import traceback
+
             execution_time = time.time() - start_time
             error_details = f"{type(e).__name__}: {str(e)}"
             tb = traceback.format_exc()

package/server/services/temporal/plugin_activities.py

@@ -27,7 +27,7 @@ a specialised worker pool.
 
 from __future__ import annotations
 
-from typing import Any, Callable, Dict, Iterable, List, Optional
+from typing import Callable, Dict, Iterable, List, Optional
 
 from core.logging import get_logger
 
@@ -69,7 +69,9 @@ def collect_plugin_activities(
             activity_fn = cls.as_activity()
         except Exception as e:
             logger.warning(
-                "Failed to build activity for %s: %s", node_type, e,
+                "Failed to build activity for %s: %s",
+                node_type,
+                e,
             )
             continue
         activities.append(activity_fn)
@@ -87,6 +89,39 @@ def collect_plugin_activities_for_queue(task_queue: str) -> List[Callable]:
     return collect_plugin_activities(task_queue=task_queue)
 
 
+def collect_polling_activities() -> List[Callable]:
+    """Return ``@activity.defn``-decorated per-cycle activities for every
+    :class:`services.plugin.PollingTriggerNode` subclass.
+
+    Wave 12 C2: each polling plugin's
+    :meth:`PollingTriggerNode.as_poll_activity` produces an activity that
+    does ONE poll cycle (vs ``cls.as_activity()`` which does ONE workflow
+    execution). The new :class:`PollingTriggerWorkflow` calls these per
+    ``workflow.sleep`` tick.
+
+    Stable activity name: ``poll.{type}.v{version}``. Registered alongside
+    the regular per-type activities in the Temporal worker.
+    """
+    from services.node_registry import registered_node_classes
+    from services.plugin import PollingTriggerNode
+
+    activities: List[Callable] = []
+    for node_type, cls in registered_node_classes().items():
+        if not isinstance(cls, type) or not issubclass(cls, PollingTriggerNode):
+            continue
+        try:
+            activities.append(cls.as_poll_activity())
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Failed to build poll activity for %s: %s",
+                node_type,
+                exc,
+            )
+
+    logger.info("Collected %d polling activities", len(activities))
+    return activities
+
+
 def distinct_task_queues() -> List[str]:
     """Return every task_queue string declared by any registered plugin.
 

package/server/services/temporal/plugin_registry.py

@@ -0,0 +1,82 @@
+"""Plugin registry for Temporal SDK ``Worker(plugins=[...])``.
+
+Plugins that ship workflow classes / activities / interceptors build a
+:class:`temporalio.plugin.SimplePlugin` instance and register it from
+their ``__init__.py``. The Temporal worker passes the snapshot to
+``Worker(plugins=[...])`` once at startup; the Temporal SDK's
+chain-of-responsibility ``configure_worker()`` hook merges each plugin's
+contributions into the worker's effective workflow / activity list
+internally.
+
+Same mechanism as the Temporal SDK's own contrib plugins
+(``temporalio.contrib.openai_agents`` ships ``OpenAIAgentsPlugin`` for
+exactly this purpose). Documented at
+https://python.temporal.io/temporalio.plugin.SimplePlugin.html and
+https://docs.temporal.io/develop/plugins-guide .
+
+Why a registry rather than direct framework imports
+---------------------------------------------------
+
+The framework worker stays plugin-agnostic — no
+``from nodes.scheduler... import CronTriggerWorkflow`` at the framework
+level. Plugins push their ``SimplePlugin`` instances in; the worker
+reads the list and hands it to Temporal. Same direction-of-dependency
+contract the eight other Wave-11.I self-registration registries enforce
+(canary_trigger_type / poll_coroutine_factory / filter_builder /
+ws_handler / router / oauth_callback_path / service_refresh /
+output_schema / shutdown_hook / service_factory / social_send_handler).
+"""
+
+from __future__ import annotations
+
+from typing import List
+
+from temporalio.plugin import SimplePlugin
+
+from services.plugin.registry import IdempotentRegistry
+
+
+# Keyed by plugin name so duplicates raise at import time (e.g. two
+# plugins both declaring ``name="cron-scheduler"`` would surface a
+# namespace clash). SimplePlugin instances are content-comparable via
+# the IdempotentRegistry's qualname check (reload safe).
+_REGISTRY: IdempotentRegistry[str, SimplePlugin] = IdempotentRegistry("temporal_plugin")
+
+
+def register_temporal_plugin(plugin: SimplePlugin) -> None:
+    """Publish a Temporal ``SimplePlugin`` for the worker.
+
+    Idempotent on re-import. Raises ``ValueError`` on a namespace
+    conflict (same ``plugin.name`` registered by a different caller).
+
+    Args:
+        plugin: A :class:`temporalio.plugin.SimplePlugin` instance.
+            The ``name`` field is used as the registry key; ``workflows``,
+            ``activities``, ``interceptors`` etc. surface in the
+            worker's effective configuration via Temporal's plugin chain.
+    """
+    # ``SimplePlugin.name`` is a method (per temporalio.plugin API) — call
+    # it to get the actual identifier string the registry keys on.
+    _REGISTRY.register(plugin.name(), plugin)
+
+
+def temporal_plugins() -> List[SimplePlugin]:
+    """Return every registered plugin (snapshot, in registration order).
+
+    The Temporal worker reads this once at startup::
+
+        worker = Worker(
+            client,
+            task_queue=...,
+            plugins=temporal_plugins(),
+            workflows=[MachinaWorkflow, ...],
+            activities=[...],
+        )
+    """
+    return list(_REGISTRY.values())
+
+
+__all__ = [
+    "register_temporal_plugin",
+    "temporal_plugins",
+]

package/server/services/temporal/polling_trigger_workflow.py

@@ -0,0 +1,267 @@
+"""Wave 12 C2: long-lived Temporal workflow for polling triggers.
+
+Closes the durability gap on polling triggers (gmailReceive,
+twitterReceive, …) the same way :class:`TriggerListenerWorkflow`
+closed it for push triggers. Today's polling lives in
+``services/deployment/triggers.py::setup_polling_trigger`` —
+a collector/processor ``asyncio.Task`` pair that dies on FastAPI
+restart, losing the seen-ID baseline and any in-flight cycle.
+
+This workflow owns the poll loop INSIDE Temporal:
+
+- ``workflow.sleep(interval)`` between cycles → replayable, no
+  per-cycle heartbeat overhead, survives worker restarts
+  (per temporal.io/blog/very-long-running-workflows).
+- Per-cycle Temporal activity (``poll.{type}.v{version}``, emitted
+  by :meth:`PollingTriggerNode.as_poll_activity`) does ONE cycle and
+  returns new events + the updated seen-id set.
+- For each new event, spawn a child :class:`MachinaWorkflow` with
+  the trigger pre-executed (same ``_build_run_graph`` helper as
+  the push-trigger listener — single source of truth for run-filter
+  semantics).
+- ``continueAsNew`` every ~16K processed events to keep Event
+  History bounded.
+
+Cross-confirmed pattern with Temporal docs + samples-python:
+- Long-lived workflow with sleep+continueAsNew: temporal.io/blog/
+  very-long-running-workflows
+- Activities for external I/O (Gmail / Twitter API calls): docs.
+  temporal.io/develop/python/core-application#activities
+- ParentClosePolicy.ABANDON for child run workflows so listener
+  cancel never strands in-flight executions
+"""
+
+from __future__ import annotations
+
+from datetime import timedelta
+from typing import Any, Dict, Set
+
+from temporalio import workflow
+from temporalio.common import WorkflowIDReusePolicy
+from temporalio.workflow import ParentClosePolicy
+
+from ._retry_policies import DEFAULT_ACTIVITY_RETRY
+
+
+# Same continueAsNew threshold as TriggerListenerWorkflow — keeps Event
+# History under Temporal's soft ceiling. The actual count depends on
+# both cycles AND events spawned per cycle; 16K events ≈ 16K spawn
+# entries + N cycles, well below the 50K guidance.
+_MAX_EVENTS_BEFORE_CONTINUE_AS_NEW = 16_000
+
+# Default poll interval (seconds) if listener_data doesn't supply one.
+# Mirrors PollingTriggerNode.default_poll_interval defaults.
+_DEFAULT_POLL_INTERVAL_S = 60
+
+# Activity timeout: 4× the poll interval gives the activity plenty of
+# headroom for slow Gmail / Twitter responses without hanging forever.
+# Workflow ``RetryPolicy`` (default) handles transient failures.
+_ACTIVITY_TIMEOUT_MULT = 4
+
+
+@workflow.defn(name="PollingTriggerWorkflow", sandboxed=False)
+class PollingTriggerWorkflow:
+    """Long-lived polling-trigger workflow.
+
+    Determinism note: all state lives on ``self`` and is reconstructed
+    from Event History on replay. Provider-side ``seen_ids`` (e.g.
+    Gmail message IDs) is the activity's output → workflow state →
+    next activity's input — never mutated mid-workflow. Event-id dedup
+    set drains on ``continueAsNew`` (intentional; events arriving in
+    the new run are by definition not duplicates of the prior run).
+    """
+
+    def __init__(self) -> None:
+        self._seen_event_ids: Set[str] = set()
+        self._processed_count: int = 0
+
+    @workflow.run
+    async def run(self, listener_data: Dict[str, Any]) -> Dict[str, Any]:
+        """Poll loop body.
+
+        ``listener_data`` shape (deployment-supplied)::
+
+            {
+                "workflow_id": str,        # MachinaOs deployment workflow_id
+                "trigger_node_id": str,    # node id that fires on each event
+                "node_type": str,          # e.g. "googleGmailReceive"
+                "version": int,            # plugin class version (for activity name)
+                "filter_params": Dict,     # plugin params (poll_interval etc.)
+                "nodes": List[Dict],       # full deployment graph snapshot
+                "edges": List[Dict],
+                "session_id": str,
+                "tenant_id": Optional[str],
+                "seen_ids": List[str],     # carried across continueAsNew; empty on first start
+            }
+
+        Returns when ``continueAsNew`` fires. Deployment cancel uses
+        a graceful ``workflow.cancel()`` — the loop's
+        ``CancelledError`` propagates and the workflow ends.
+        """
+        node_type = listener_data["node_type"]
+        version = listener_data.get("version", 1)
+        activity_name = f"poll.{node_type}.v{version}"
+
+        params = listener_data.get("filter_params", {}) or {}
+        poll_interval = int(params.get("poll_interval") or _DEFAULT_POLL_INTERVAL_S)
+        activity_timeout_s = max(30, poll_interval * _ACTIVITY_TIMEOUT_MULT)
+
+        # Carry seen_ids across continueAsNew. First-start payload has
+        # ``seen_ids=[]`` and the first activity call is baseline-only.
+        seen_ids: Set[str] = set(listener_data.get("seen_ids") or [])
+        is_baseline = not seen_ids
+
+        workflow.logger.info(
+            f"PollingTriggerWorkflow started: workflow_id={listener_data.get('workflow_id')} "
+            f"node={listener_data.get('trigger_node_id')} type={node_type} "
+            f"interval={poll_interval}s baseline={is_baseline}"
+        )
+
+        while True:
+            if is_baseline:
+                # Establish seen baseline immediately on first run so we
+                # don't re-emit items the user has had since before deploy.
+                is_baseline = False
+                cycle_payload = {
+                    "node_id": listener_data["trigger_node_id"],
+                    "params": params,
+                    "seen_ids": [],
+                    "baseline_only": True,
+                }
+            else:
+                await workflow.sleep(timedelta(seconds=poll_interval))
+                cycle_payload = {
+                    "node_id": listener_data["trigger_node_id"],
+                    "params": params,
+                    "seen_ids": list(seen_ids),
+                    "baseline_only": False,
+                }
+
+            try:
+                # Wave 12 D1: explicit RetryPolicy with
+                # non_retryable_error_types=("NodeUserError", ...) —
+                # poll-cycle failures from user-correctable causes
+                # (bad filter expression, missing credential) fail fast
+                # instead of burning 3 retries per cycle.
+                result = await workflow.execute_activity(
+                    activity_name,
+                    cycle_payload,
+                    start_to_close_timeout=timedelta(seconds=activity_timeout_s),
+                    retry_policy=DEFAULT_ACTIVITY_RETRY,
+                )
+            except Exception as exc:  # noqa: BLE001
+                # Activity exhausted its RetryPolicy. Log + continue —
+                # don't terminate the listener over one bad cycle.
+                # Workflow Event History records the failure for ops.
+                workflow.logger.error(f"PollingTriggerWorkflow cycle failed (will retry next interval): {exc}")
+                continue
+
+            seen_ids = set(result.get("seen_ids") or [])
+            events = result.get("events") or []
+
+            for event in events:
+                event_id = event.get("id")
+                if not event_id or event_id in self._seen_event_ids:
+                    continue
+                self._seen_event_ids.add(event_id)
+                try:
+                    await self._spawn_child_run(event, listener_data)
+                except Exception as spawn_exc:  # noqa: BLE001
+                    # Per-event spawn failure logged; subsequent events
+                    # still try. Same isolation contract as the push
+                    # listener.
+                    workflow.logger.error(f"PollingTriggerWorkflow spawn failed for event.id={event_id}: {spawn_exc}")
+                self._processed_count += 1
+
+            if self._processed_count >= _MAX_EVENTS_BEFORE_CONTINUE_AS_NEW:
+                workflow.logger.info(f"PollingTriggerWorkflow continue_as_new: processed={self._processed_count}")
+                # Carry seen_ids forward so the new run doesn't re-emit
+                # what's already been seen by the provider.
+                listener_data["seen_ids"] = list(seen_ids)
+                workflow.continue_as_new(listener_data)
+
+    async def _spawn_child_run(
+        self,
+        event: Dict[str, Any],
+        listener_data: Dict[str, Any],
+    ) -> None:
+        """Start a child :class:`MachinaWorkflow` with the trigger
+        pre-executed against this event payload.
+
+        Reuses ``_build_run_graph`` from
+        :mod:`services.temporal.trigger_listener_workflow` so the
+        filtered-graph semantics (n8n stop-at-trigger downstream walk,
+        config nodes via input handles, toolkit sub-nodes, agent tool
+        nodes) stay single-source. Mirrors
+        :meth:`TriggerListenerWorkflow._spawn_child_run` exactly.
+        """
+        from services.temporal.trigger_listener_workflow import (
+            _broadcast_trigger_idle,
+            _broadcast_trigger_waiting,
+            _build_run_graph,
+        )
+
+        trigger_node_id = listener_data["trigger_node_id"]
+        nodes = listener_data["nodes"]
+        edges = listener_data["edges"]
+        session_id = listener_data.get("session_id", "default")
+        workflow_id = listener_data.get("workflow_id")
+        tenant_id = listener_data.get("tenant_id")
+
+        # Polling activity returns plugin-native payload dicts (Gmail
+        # email envelope, Twitter tweet payload). For Temporal-side
+        # introspection we pass the dict as both the trigger output
+        # AND nest it under ``_event_envelope`` so downstream nodes
+        # can route off the original shape — matches the push-listener
+        # contract.
+        trigger_output = {**event, "_event_envelope": event}
+
+        filtered_nodes, filtered_edges = _build_run_graph(
+            trigger_node_id=trigger_node_id,
+            trigger_output=trigger_output,
+            nodes=nodes,
+            edges=edges,
+        )
+
+        # Lazy fallback to workflow.uuid4() only when event.id is missing —
+        # eager-eval default-arg form would trip _NotInWorkflowEventLoopError
+        # in unit tests + waste entropy on the hot path.
+        event_id = event.get("id") or workflow.uuid4().hex
+        child_id = f"run-{workflow_id}-{event_id}"
+
+        await _broadcast_trigger_idle(
+            node_id=trigger_node_id,
+            workflow_id=workflow_id,
+            event_id=event_id,
+            event_type=listener_data.get("event_type", ""),
+        )
+
+        await workflow.start_child_workflow(
+            "MachinaWorkflow",
+            args=[
+                {
+                    "nodes": filtered_nodes,
+                    "edges": filtered_edges,
+                    "session_id": session_id,
+                    "workflow_id": workflow_id,
+                    "tenant_id": tenant_id,
+                }
+            ],
+            id=child_id,
+            parent_close_policy=ParentClosePolicy.ABANDON,
+            id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY,
+            execution_timeout=timedelta(hours=1),
+            run_timeout=timedelta(hours=1),
+        )
+
+        workflow.logger.info(f"PollingTriggerWorkflow spawned child run: child_id={child_id} " f"event.id={event.get('id')}")
+
+        await _broadcast_trigger_waiting(
+            node_id=trigger_node_id,
+            workflow_id=workflow_id,
+            event_type=listener_data.get("event_type", ""),
+            processed_count=self._processed_count + 1,
+        )
+
+
+__all__ = ["PollingTriggerWorkflow"]