machinaos 0.0.78 → 0.0.80

package/server/services/temporal/schedules.py

@@ -0,0 +1,220 @@
+"""Wave 12 C3: Temporal Schedule helpers for cron triggers.
+
+Thin framework-level wrappers around the Temporal Schedule API. Plugin
+ownership stays in the plugin folder — this module's only job is the
+**mechanics** of creating + cancelling Schedules. The plugin's workflow
+class (``CronTriggerWorkflow``) and its specific action args are
+plugin-owned.
+
+Same shape as the Visibility-based cancel sweep in
+:meth:`services.deployment.manager.DeploymentManager._cancel_canary_listeners`
+— deterministic id, no in-memory tracking, server-side state is the
+registry. ``Schedule`` resources live in their OWN Visibility list
+(``client.list_schedules``), distinct from workflow Visibility.
+
+Refs:
+  - https://docs.temporal.io/develop/python/schedules
+  - https://python.temporal.io/temporalio.client.Schedule.html
+  - https://docs.temporal.io/encyclopedia/scheduled-execution
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from temporalio.client import (
+    Client,
+    Schedule,
+    ScheduleActionStartWorkflow,
+    ScheduleAlreadyRunningError,
+    ScheduleIntervalSpec,  # noqa: F401 — re-exported for any caller that needs it
+    ScheduleOverlapPolicy,
+    SchedulePolicy,
+    ScheduleSpec,
+)
+from temporalio.common import (
+    SearchAttributeKey,
+    SearchAttributePair,
+    TypedSearchAttributes,
+)
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+# Schedule ID prefix used by both create + cancel-sweep. Mirrors the
+# ``trigger-listener-{wf}-{node}`` deterministic-id shape that the C1
+# canary uses for its listener workflows — same locality + uniqueness
+# properties.
+def cron_schedule_id(deployment_workflow_id: str, node_id: str) -> str:
+    """Deterministic Schedule ID for a (deployment, cron-node) pair.
+
+    Mapped to the business entity so re-deploying the same MachinaOs
+    workflow targets the same Schedule. Pairs with the create call's
+    ``"already exists" → no-op`` semantics (per
+    :exc:`temporalio.client.ScheduleAlreadyRunningError`).
+    """
+    return f"cron-schedule-{deployment_workflow_id}-{node_id}"
+
+
+# Child workflow ID prefix Temporal stamps onto each firing. The
+# ``{{.ScheduledStartTime}}`` placeholder is substituted by Temporal
+# server-side, so each tick produces a unique workflow_id without us
+# needing per-tick code (per
+# https://docs.temporal.io/develop/python/schedules#use-a-pre-generated-workflow-id).
+def cron_action_workflow_id(deployment_workflow_id: str, node_id: str) -> str:
+    return f"cron-fire-{deployment_workflow_id}-{node_id}-{{{{.ScheduledStartTime}}}}"
+
+
+async def create_cron_schedule(
+    client: Client,
+    *,
+    deployment_workflow_id: str,
+    node_id: str,
+    cron_expression: str,
+    timezone: str,
+    listener_data: Dict[str, Any],
+    task_queue: str = "machina-tasks",
+    overlap_policy: ScheduleOverlapPolicy = ScheduleOverlapPolicy.SKIP,
+) -> str:
+    """Create-or-reuse a Temporal Schedule for a cron trigger.
+
+    Returns the Schedule's id. Idempotent: a re-deploy with the same
+    ``(deployment_workflow_id, node_id)`` pair reuses the existing
+    Schedule (Temporal raises
+    :exc:`ScheduleAlreadyRunningError` which we swallow as a no-op).
+
+    Args:
+        client: Connected Temporal client.
+        deployment_workflow_id: MachinaOs deployment workflow_id.
+        node_id: Cron-trigger node id.
+        cron_expression: Crontab string (5 or 6 field).
+        timezone: IANA tz name (e.g. ``"America/New_York"``).
+        listener_data: Frozen action args for the workflow run
+            (deployment graph snapshot + cron metadata).
+        task_queue: Worker task queue that hosts ``CronTriggerWorkflow``.
+        overlap_policy: How concurrent firings interact. ``SKIP`` (default)
+            drops a firing if the prior run is still going; mirrors the
+            pre-Wave-12 APScheduler behaviour for slow workflows.
+    """
+    schedule_id = cron_schedule_id(deployment_workflow_id, node_id)
+    action_workflow_id = cron_action_workflow_id(deployment_workflow_id, node_id)
+
+    schedule = Schedule(
+        action=ScheduleActionStartWorkflow(
+            "CronTriggerWorkflow",
+            args=[listener_data],
+            id=action_workflow_id,
+            task_queue=task_queue,
+        ),
+        spec=ScheduleSpec(
+            cron_expressions=[cron_expression],
+            time_zone_name=timezone or "UTC",
+        ),
+        policy=SchedulePolicy(overlap=overlap_policy),
+    )
+
+    # Search Attributes mirror the listener-canary contract so the
+    # cancel sweep can find the Schedule via the same EventWorkflowId
+    # filter. EventTriggerKind="cron" lets ops dashboards filter by
+    # canary kind independently of the underlying primitive. Note:
+    # ``search_attributes`` is a ``client.create_schedule`` kwarg, not
+    # a field on the ``Schedule`` dataclass itself (per the Temporal
+    # SDK API).
+    schedule_search_attributes = TypedSearchAttributes(
+        [
+            SearchAttributePair(
+                SearchAttributeKey.for_keyword("EventWorkflowId"),
+                deployment_workflow_id,
+            ),
+            SearchAttributePair(
+                SearchAttributeKey.for_keyword("TriggerNodeId"),
+                node_id,
+            ),
+            SearchAttributePair(
+                SearchAttributeKey.for_keyword("EventTriggerKind"),
+                "cron",
+            ),
+        ]
+    )
+
+    try:
+        await client.create_schedule(
+            schedule_id,
+            schedule,
+            search_attributes=schedule_search_attributes,
+        )
+        logger.info(
+            "Created Temporal cron Schedule",
+            schedule_id=schedule_id,
+            cron_expression=cron_expression,
+            timezone=timezone,
+        )
+    except ScheduleAlreadyRunningError:
+        # Idempotent re-deploy. The pre-existing Schedule keeps its
+        # state; the caller's create-or-reuse contract is satisfied.
+        logger.info(
+            "Temporal cron Schedule already exists (re-deploy idempotency)",
+            schedule_id=schedule_id,
+        )
+
+    return schedule_id
+
+
+async def delete_cron_schedules_for_deployment(
+    client: Client,
+    deployment_workflow_id: str,
+) -> int:
+    """Delete every cron Schedule for ``deployment_workflow_id``.
+
+    Visibility-equivalent sweep: ``client.list_schedules(query=...)``
+    is the registry, no local handle dict. Per-Schedule failures don't
+    block the sweep (mirror of
+    :meth:`DeploymentManager._cancel_canary_listeners` semantics).
+
+    Returns count of Schedules deleted.
+    """
+    query = f"EventWorkflowId='{deployment_workflow_id}' " f"AND EventTriggerKind='cron'"
+
+    deleted = 0
+    try:
+        # ``Client.list_schedules`` is ``async def`` (returns a coroutine
+        # that resolves to ``ScheduleAsyncIterator``), unlike
+        # ``Client.list_workflows`` which is a plain function returning
+        # the iterator directly. Calling ``async for`` on the coroutine
+        # raises "'async for' requires an object with __aiter__".
+        # Reference:
+        #   https://python.temporal.io/temporalio.client.Client.html#list_schedules
+        iterator = await client.list_schedules(query=query)
+        async for desc in iterator:
+            sched_id = desc.id
+            try:
+                handle = client.get_schedule_handle(sched_id)
+                await handle.delete()
+                deleted += 1
+            except Exception as exc:  # noqa: BLE001
+                logger.warning(
+                    f"Failed to delete cron Schedule {sched_id!r}: {exc}",
+                    deployment_workflow_id=deployment_workflow_id,
+                )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            f"Visibility query for cron Schedules failed: {exc} " f"(query={query!r})",
+            deployment_workflow_id=deployment_workflow_id,
+        )
+
+    if deleted:
+        logger.info(
+            "Cron Schedules deleted",
+            deployment_workflow_id=deployment_workflow_id,
+            count=deleted,
+        )
+    return deleted
+
+
+__all__ = [
+    "cron_schedule_id",
+    "create_cron_schedule",
+    "delete_cron_schedules_for_deployment",
+]

package/server/services/temporal/search_attributes.py

@@ -0,0 +1,177 @@
+"""Wave 12 A4: Temporal custom Search Attributes registration.
+
+Defines the 6 Search Attributes the event framework needs for the
+Visibility API to route inbound events to running consumer workflows
+(see ``services/events/dispatch.py:emit``). Registration is idempotent —
+Temporal Server's ``add_search_attributes`` rejects duplicates with an
+``AlreadyExists`` error which we treat as success.
+
+Attribute declarations live in one place (:data:`EVENT_SEARCH_ATTRIBUTES`)
+as a structured list so callers iterate rather than restate. Adding a
+new attribute = add an entry to that list; everything else (registration,
+docs, error handling) flows from it.
+
+References:
+- https://docs.temporal.io/search-attribute
+- https://python.temporal.io/temporalio.client.html#Client.operator_service
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Sequence
+
+from temporalio.api.enums.v1 import IndexedValueType
+from temporalio.api.operatorservice.v1 import (
+    AddSearchAttributesRequest,
+    ListSearchAttributesRequest,
+)
+from temporalio.client import Client
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+@dataclass(frozen=True)
+class SearchAttributeSpec:
+    """Declarative spec for one Temporal Search Attribute."""
+
+    name: str
+    indexed_type: IndexedValueType
+    description: str
+
+
+# Single source of truth for the 6 event-framework Search Attributes.
+# Anything that names or filters by these attributes (the dispatch
+# helper, the Visibility queries in admin handlers, the test suite)
+# imports from this list rather than restating the strings.
+EVENT_SEARCH_ATTRIBUTES: Sequence[SearchAttributeSpec] = (
+    SearchAttributeSpec(
+        name="EventType",
+        indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+        description=(
+            "CloudEvents type the workflow is consuming "
+            "(e.g. com.machinaos.whatsapp.message.received). Used by "
+            "dispatch.emit to find running consumers via Visibility "
+            "queries: ListWorkflows(query=f\"EventType='{event.type}' "
+            "AND ExecutionStatus='Running'\")."
+        ),
+    ),
+    SearchAttributeSpec(
+        name="EventSource",
+        indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+        description=(
+            "CloudEvents source URI fragment "
+            "(e.g. machinaos://nodes/whatsapp). Used for routing when "
+            "the same type can arrive from multiple sources."
+        ),
+    ),
+    SearchAttributeSpec(
+        name="EventWorkflowId",
+        indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+        description=(
+            "MachinaOs workflow_id the event belongs to (distinct from "
+            "the Temporal Workflow ID which addresses the workflow run). "
+            "Used to scope event delivery to a specific deployment."
+        ),
+    ),
+    SearchAttributeSpec(
+        name="TriggerNodeId",
+        indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+        description=(
+            "Trigger node id that emitted or will consume the event. "
+            "Used for per-trigger event-history queries and the "
+            "list_event_waiters admin handler."
+        ),
+    ),
+    SearchAttributeSpec(
+        name="EventTriggerKind",
+        indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+        description=(
+            "High-level event source classification (webhook / polling / "
+            "whatsapp / telegram / ...). Coarser-grained than EventType; "
+            "useful for ops dashboards."
+        ),
+    ),
+    SearchAttributeSpec(
+        name="EventReceivedAt",
+        indexed_type=IndexedValueType.INDEXED_VALUE_TYPE_DATETIME,
+        description=(
+            "When the event entered the framework. Supports time-range "
+            "visibility queries (e.g. 'all events received in the last "
+            "hour')."
+        ),
+    ),
+)
+
+
+def attribute_names() -> Sequence[str]:
+    """Return the configured attribute names. Helper for tests + admin code."""
+    return tuple(spec.name for spec in EVENT_SEARCH_ATTRIBUTES)
+
+
+async def register_search_attributes(client: Client, namespace: str) -> dict:
+    """Idempotently register every :data:`EVENT_SEARCH_ATTRIBUTES` entry
+    on the given Temporal namespace.
+
+    Returns a status dict keyed by attribute name:
+        {name: "registered" | "already_exists" | "error: ..."}
+
+    Safe to call multiple times — already-present attributes are
+    detected via a pre-flight ``ListSearchAttributes`` call and skipped.
+    Attributes that fail to register surface in the result dict with
+    the error string so the caller can decide whether to fail-fast.
+    """
+    operator = client.service_client.operator_service
+
+    # Pre-flight: which attributes already exist on this namespace?
+    try:
+        existing_resp = await operator.list_search_attributes(
+            ListSearchAttributesRequest(namespace=namespace),
+        )
+        existing_names = set(existing_resp.custom_attributes.keys())
+    except Exception as exc:  # noqa: BLE001 — surface every failure
+        logger.warning(
+            f"Could not list existing Search Attributes on namespace={namespace!r}: {exc}. "
+            "Attempting registration anyway; duplicates will be reported as errors."
+        )
+        existing_names = set()
+
+    status: dict[str, str] = {}
+    to_register: dict[str, IndexedValueType] = {
+        spec.name: spec.indexed_type for spec in EVENT_SEARCH_ATTRIBUTES if spec.name not in existing_names
+    }
+    for name in existing_names & {s.name for s in EVENT_SEARCH_ATTRIBUTES}:
+        status[name] = "already_exists"
+
+    if not to_register:
+        logger.info(
+            f"All {len(EVENT_SEARCH_ATTRIBUTES)} event-framework Search Attributes " f"already registered on namespace={namespace!r}",
+        )
+        return status
+
+    try:
+        await operator.add_search_attributes(
+            AddSearchAttributesRequest(
+                namespace=namespace,
+                search_attributes=to_register,
+            ),
+        )
+    except Exception as exc:  # noqa: BLE001
+        # Whole batch failed — mark each requested attribute with the error.
+        err = f"error: {type(exc).__name__}: {exc}"
+        for name in to_register:
+            status[name] = err
+        logger.error(
+            f"Failed to register Search Attributes on namespace={namespace!r}: {exc}",
+        )
+        return status
+
+    for name in to_register:
+        status[name] = "registered"
+
+    logger.info(
+        f"Registered {len(to_register)} event-framework Search Attributes on " f"namespace={namespace!r}: {list(to_register.keys())}",
+    )
+    return status