ltcai 4.0.0 → 4.0.1

package/latticeai/services/agent_runtime.py

@@ -21,6 +21,7 @@ it, the frontend) now depends on this boundary instead of internal paths.
 
 from __future__ import annotations
 
+from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
 from latticeai.core.multi_agent import (
@@ -41,8 +42,12 @@ ROLE_DESCRIPTIONS = {
 # Run statuses the orchestrator can emit that mean "still working". The default
 # orchestrator runs synchronously, so persisted runs are always terminal; this
 # set lets the runtime report live work if a future async runner lands.
-_ACTIVE_STATUSES = {"running", "in_progress", "queued", "retrying"}
-_TERMINAL_STATUSES = {"ok", "retried_ok", "failed", "rejected", "cancelled"}
+_ACTIVE_STATUSES = {"running", "in_progress", "queued", "retrying", "cancelling"}
+_TERMINAL_STATUSES = {"ok", "retried_ok", "failed", "rejected", "cancelled", "interrupted", "partial"}
+
+
+def _now() -> str:
+    return datetime.now().isoformat(timespec="seconds")
 
 
 class AgentRuntime:
@@ -64,6 +69,13 @@ class AgentRuntime:
         # Lifecycle hooks registry (optional). When present, ``start`` fires the
         # ``pre_run`` / ``post_run`` hooks; a blocking ``pre_run`` aborts the run.
         self._hooks = hooks
+        self._run_executor: Any = None
+
+    def attach_executor(self, executor: Any) -> None:
+        self._run_executor = executor
+
+    def _execution_mode(self) -> str:
+        return "async" if self._run_executor is not None else "synchronous"
 
     # ── configuration ─────────────────────────────────────────────────────
     def config(self) -> Dict[str, Any]:
@@ -72,7 +84,12 @@ class AgentRuntime:
             "roles": list(AGENT_ROLES),
             "default_pipeline": list(CORE_PIPELINE),
             "max_retries_cap": self._max_retries_cap,
-            "execution_mode": "synchronous",
+            "execution_mode": self._execution_mode(),
+            "cancellation": (
+                "cooperative; running synchronous model/tool calls finish their current step before a cancelled status is persisted"
+                if self._run_executor is not None else
+                "not supported for the synchronous runtime"
+            ),
         }
 
     def roles(self) -> List[Dict[str, Any]]:
@@ -150,7 +167,7 @@ class AgentRuntime:
             "runtime": {
                 "ready": True,
                 "version": MULTI_AGENT_VERSION,
-                "execution_mode": "synchronous",
+                "execution_mode": self._execution_mode(),
                 "default_pipeline": list(CORE_PIPELINE),
                 "total_runs": len(runs),
                 "active_runs": active,
@@ -184,6 +201,225 @@ class AgentRuntime:
         }
 
     # ── execution ─────────────────────────────────────────────────────────
+    def _fire_pre_run(
+        self,
+        *,
+        goal: str,
+        roles: Optional[List[str]],
+        max_retries: int,
+        user_email: Optional[str],
+        scope: Optional[str],
+    ) -> Optional[Dict[str, Any]]:
+        pre_dispatch: Optional[Dict[str, Any]] = None
+        if self._hooks is not None:
+            pre_dispatch = self._hooks.fire_hook(
+                "pre_run", "agent.run",
+                payload={"goal": goal, "roles": roles or None, "max_retries": max_retries},
+                user_email=user_email, workspace_id=scope,
+            )
+            if pre_dispatch.get("blocked"):
+                self._append_audit_event(
+                    "multi_agent_run_blocked",
+                    user_email=user_email,
+                    reason=pre_dispatch.get("block_reason"),
+                )
+                raise PermissionError(pre_dispatch.get("block_reason") or "Agent run blocked by a pre_run hook.")
+        return pre_dispatch
+
+    @staticmethod
+    def _result_patch(result: Any, goal: str) -> Dict[str, Any]:
+        return {
+            "agent_id": result.agent_id,
+            "status": result.status,
+            "input": goal,
+            "output_text": result.output,
+            "timeline": result.timeline,
+            "relationships": [ROLE_AGENT_IDS.get(r, f"agent:{r}") for r in result.roles_run],
+            "handoffs": result.handoffs,
+            "context_packets": result.context_packets,
+            "plan": result.plan,
+            "plan_review": result.plan_review,
+            "review_history": result.review_history,
+            "retry_history": result.retry_history,
+            "memory_snapshots": result.memory_snapshots,
+            "mode": getattr(result, "mode", "simulation"),
+            "current_role": None,
+        }
+
+    def _post_run_hooks(
+        self,
+        *,
+        run_id: Optional[str],
+        result: Any,
+        user_email: Optional[str],
+        scope: Optional[str],
+        status: Optional[str] = None,
+    ) -> Optional[Dict[str, Any]]:
+        if self._hooks is None:
+            return None
+        return self._hooks.fire_hook(
+            "post_run", "agent.run",
+            payload={
+                "run_id": run_id,
+                "agent_id": result.agent_id,
+                "status": status or result.status,
+                "retries": result.retries,
+            },
+            user_email=user_email, workspace_id=scope,
+        )
+
+    def reserve_run(
+        self,
+        goal: str,
+        *,
+        user_email: Optional[str],
+        scope: Optional[str],
+        roles: Optional[List[str]] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        max_retries: int = 2,
+    ) -> Dict[str, Any]:
+        """Create the durable queued row used by the async executor."""
+        if not str(goal or "").strip():
+            raise ValueError("goal is required")
+        pre_dispatch = self._fire_pre_run(
+            goal=goal,
+            roles=roles,
+            max_retries=max_retries,
+            user_email=user_email,
+            scope=scope,
+        )
+        try:
+            orchestrator = self._orchestrator_factory(user_email or None, scope)
+            mode = getattr(orchestrator, "mode", "simulation")
+        except Exception:
+            mode = "simulation"
+        run = self._store.record_agent_run(
+            agent_id=ROLE_AGENT_IDS.get("executor", "agent:executor"),
+            status="queued",
+            input_text=goal,
+            output_text="",
+            timeline=[{"event": "agent_started", "status": "queued", "timestamp": _now()}],
+            relationships=[],
+            handoffs=[],
+            context_packets=[],
+            plan=[],
+            plan_review={},
+            review_history=[],
+            retry_history=[],
+            memory_snapshots=[],
+            user_email=user_email or None,
+            graph=None,
+            workspace_id=scope,
+            mode=mode,
+        )
+        run = self._store.update_agent_run(
+            run.get("id"),
+            workspace_id=scope,
+            execution_mode="async",
+            requested_roles=roles or None,
+            inputs=inputs or {},
+            max_retries=max_retries,
+        )
+        payload: Dict[str, Any] = {"run": run}
+        if pre_dispatch is not None:
+            payload["pre_run_hooks"] = pre_dispatch
+        return payload
+
+    def complete_reserved_run(
+        self,
+        run_id: str,
+        goal: str,
+        *,
+        user_email: Optional[str],
+        scope: Optional[str],
+        roles: Optional[List[str]] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        max_retries: int = 2,
+        pre_dispatch: Optional[Dict[str, Any]] = None,
+        cancel_requested: Optional[Callable[[], bool]] = None,
+    ) -> Dict[str, Any]:
+        """Execute orchestration and update an existing durable async row."""
+        run = self._store.get_agent_run(run_id, workspace_id=scope)
+        base_timeline = list(run.get("timeline") or [])
+        self._store.update_agent_run(
+            run_id,
+            workspace_id=scope,
+            status="running",
+            current_role=(roles or list(CORE_PIPELINE))[0] if (roles or CORE_PIPELINE) else None,
+            started_at=run.get("started_at") or _now(),
+        )
+        try:
+            orchestrator = self._orchestrator_factory(user_email or None, scope)
+            result = orchestrator.run(
+                goal,
+                user_email=user_email or None,
+                workspace_id=scope,
+                inputs=inputs or {},
+                roles=roles or None,
+                max_retries=max(0, min(int(max_retries or 0), self._max_retries_cap)),
+            )
+        except Exception as exc:
+            failed = self._store.update_agent_run(
+                run_id,
+                workspace_id=scope,
+                graph=self._workspace_graph(),
+                status="failed",
+                current_role=None,
+                error=str(exc),
+                output_text=str(exc),
+                timeline=base_timeline + [{
+                    "event": "execution_failed",
+                    "status": "failed",
+                    "detail": str(exc),
+                    "timestamp": _now(),
+                }],
+            )
+            self._append_audit_event("multi_agent_run", user_email=user_email, agent_id=failed.get("agent_id"), status="failed", retries=0)
+            return {"run": failed, "result": {"status": "failed", "error": str(exc)}}
+
+        patch = self._result_patch(result, goal)
+        patch["timeline"] = base_timeline + list(result.timeline or [])
+        if cancel_requested is not None and cancel_requested():
+            patch["status"] = "cancelled"
+            patch["current_role"] = None
+            patch["cancel_reason"] = "cancelled after the current synchronous step completed"
+            patch["cancelled_at"] = _now()
+            patch["timeline"] = patch["timeline"] + [{
+                "event": "execution_cancelled",
+                "status": "cancelled",
+                "reason": patch["cancel_reason"],
+                "timestamp": _now(),
+            }]
+        updated = self._store.update_agent_run(
+            run_id,
+            workspace_id=scope,
+            graph=self._workspace_graph(),
+            patch=patch,
+        )
+        self._append_audit_event(
+            "multi_agent_run",
+            user_email=user_email,
+            agent_id=result.agent_id,
+            status=updated.get("status") or result.status,
+            retries=result.retries,
+        )
+        post_dispatch = self._post_run_hooks(
+            run_id=run_id,
+            result=result,
+            user_email=user_email,
+            scope=scope,
+            status=updated.get("status") or result.status,
+        )
+        result_payload = result.as_dict()
+        if updated.get("status") == "cancelled":
+            result_payload = {"status": "cancelled", "reason": updated.get("cancel_reason"), "completed_result": result_payload}
+        payload = {"run": updated, "result": result_payload}
+        if pre_dispatch is not None:
+            payload["pre_run_hooks"] = pre_dispatch
+        if post_dispatch is not None:
+            payload["post_run_hooks"] = post_dispatch
+        return payload
+
     def start(
         self,
         goal: str,
@@ -281,6 +517,8 @@ class AgentRuntime:
         exists the run has already completed. Report that honestly rather than
         pretending a cancellation occurred.
         """
+        if self._run_executor is not None:
+            return self._run_executor.cancel(run_id, kind="agent", scope=scope)
         try:
             run = self._store.get_agent_run(run_id, workspace_id=scope)
         except FileNotFoundError:

package/latticeai/services/run_executor.py

@@ -0,0 +1,328 @@
+"""Durable asyncio run executor for v4 Act runtimes.
+
+The executor owns server-loop tasks for agent and workflow runs while the
+workspace store remains the durable source of truth. Work is persisted before it
+starts, updated as it moves through queued/running/cancelling/final states, and
+reconciled on startup so orphaned active rows never masquerade as live work.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any, Callable, Dict, Optional
+
+from latticeai.core.workflow_engine import WorkflowEngine
+
+
+ACTIVE_STATUSES = {"queued", "running", "in_progress", "retrying", "cancelling"}
+TERMINAL_STATUSES = {"ok", "retried_ok", "failed", "rejected", "cancelled", "interrupted", "partial"}
+
+
+def _now() -> str:
+    return datetime.now().isoformat(timespec="seconds")
+
+
+@dataclass
+class _RunHandle:
+    run_id: str
+    kind: str
+    scope: Optional[str]
+    task: Optional[asyncio.Task] = None
+    cancel_requested: bool = False
+    started: bool = False
+
+
+class RunExecutor:
+    """Async task manager for persisted agent/workflow executions."""
+
+    def __init__(
+        self,
+        *,
+        store: Any,
+        agent_runtime: Any,
+        build_workflow_runners: Callable[[Optional[str], Optional[str]], Dict[str, Callable[..., Any]]],
+        workspace_graph: Callable[[], Any],
+        append_audit_event: Callable[..., None],
+        hooks: Any = None,
+    ) -> None:
+        self.store = store
+        self.agent_runtime = agent_runtime
+        self.build_workflow_runners = build_workflow_runners
+        self.workspace_graph = workspace_graph
+        self.append_audit_event = append_audit_event
+        self.hooks = hooks
+        self._handles: Dict[str, _RunHandle] = {}
+        self._results: Dict[str, Dict[str, Any]] = {}
+
+    # ── startup reconciliation ───────────────────────────────────────────
+
+    def reconcile_startup(self) -> Dict[str, Any]:
+        return self.store.reconcile_interrupted_runs(reason="server_startup")
+
+    # ── agent runs ───────────────────────────────────────────────────────
+
+    async def start_agent(
+        self,
+        goal: str,
+        *,
+        user_email: Optional[str],
+        scope: Optional[str],
+        roles: Optional[list[str]] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        max_retries: int = 2,
+    ) -> Dict[str, Any]:
+        reserved = self.agent_runtime.reserve_run(
+            goal,
+            user_email=user_email,
+            scope=scope,
+            roles=roles,
+            inputs=inputs or {},
+            max_retries=max_retries,
+        )
+        run_id = reserved["run"]["id"]
+        handle = _RunHandle(run_id=run_id, kind="agent", scope=scope)
+        handle.task = asyncio.create_task(
+            self._run_agent(handle, goal, user_email=user_email, roles=roles, inputs=inputs or {}, max_retries=max_retries)
+        )
+        self._handles[run_id] = handle
+        return {
+            **reserved,
+            "execution_mode": "async",
+            "accepted": True,
+            "events_url": f"/agents/api/runs/{run_id}/events",
+            "stop_url": f"/agents/api/runs/{run_id}/stop",
+        }
+
+    async def _run_agent(
+        self,
+        handle: _RunHandle,
+        goal: str,
+        *,
+        user_email: Optional[str],
+        roles: Optional[list[str]],
+        inputs: Dict[str, Any],
+        max_retries: int,
+    ) -> None:
+        run_id = handle.run_id
+        try:
+            if handle.cancel_requested:
+                self._cancel_agent_record(run_id, handle.scope, "cancelled before execution started")
+                return
+            handle.started = True
+            payload = await asyncio.to_thread(
+                self.agent_runtime.complete_reserved_run,
+                run_id,
+                goal,
+                user_email=user_email,
+                scope=handle.scope,
+                roles=roles,
+                inputs=inputs,
+                max_retries=max_retries,
+                cancel_requested=lambda: handle.cancel_requested,
+            )
+            if handle.cancel_requested and (payload.get("run") or {}).get("status") != "cancelled":
+                self._cancel_agent_record(run_id, handle.scope, "cancelled after the final result was persisted")
+            else:
+                self._results[run_id] = payload
+        finally:
+            self._handles.pop(run_id, None)
+
+    def _cancel_agent_record(self, run_id: str, scope: Optional[str], reason: str) -> Dict[str, Any]:
+        run = self.store.get_agent_run(run_id, workspace_id=scope)
+        timeline = list(run.get("timeline") or [])
+        timeline.append({"event": "execution_cancelled", "status": "cancelled", "reason": reason, "timestamp": _now()})
+        cancelled = self.store.update_agent_run(
+            run_id,
+            workspace_id=scope,
+            status="cancelled",
+            current_role=None,
+            cancel_reason=reason,
+            cancelled_at=_now(),
+            output_text=run.get("output_preview") or reason,
+            timeline=timeline,
+            graph=self.workspace_graph(),
+        )
+        payload = {"run": cancelled, "result": {"status": "cancelled", "reason": reason}}
+        self._results[run_id] = payload
+        return cancelled
+
+    # ── workflow runs ────────────────────────────────────────────────────
+
+    async def start_workflow(
+        self,
+        workflow: Dict[str, Any],
+        *,
+        workflow_id: str,
+        user_email: Optional[str],
+        scope: Optional[str],
+        inputs: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        run = self.store.record_workflow_run(
+            workflow_id=workflow_id,
+            name=workflow.get("name") or "workflow",
+            status="queued",
+            timeline=[{"event": "workflow_started", "status": "queued", "timestamp": _now()}],
+            outputs={},
+            user_email=user_email,
+            graph=None,
+            workspace_id=scope,
+            mode="live",
+        )
+        run = self.store.update_workflow_run(
+            run["id"],
+            workspace_id=scope,
+            execution_mode="async",
+            inputs=inputs or {},
+        )
+        handle = _RunHandle(run_id=run["id"], kind="workflow", scope=scope)
+        handle.task = asyncio.create_task(
+            self._run_workflow(handle, workflow, user_email=user_email, inputs=inputs or {})
+        )
+        self._handles[run["id"]] = handle
+        return {
+            "run": run,
+            "execution_mode": "async",
+            "accepted": True,
+            "events_url": f"/workflows/api/runs/{run['id']}/replay",
+            "stop_url": f"/workflows/api/runs/{run['id']}/stop",
+        }
+
+    async def _run_workflow(
+        self,
+        handle: _RunHandle,
+        workflow: Dict[str, Any],
+        *,
+        user_email: Optional[str],
+        inputs: Dict[str, Any],
+    ) -> None:
+        run_id = handle.run_id
+        try:
+            if handle.cancel_requested:
+                self._cancel_workflow_record(run_id, handle.scope, "cancelled before execution started")
+                return
+            handle.started = True
+            run = self.store.get_workflow_run(run_id, workspace_id=handle.scope)
+            base_timeline = list(run.get("timeline") or [])
+            self.store.update_workflow_run(
+                run_id,
+                workspace_id=handle.scope,
+                status="running",
+                started_at=run.get("started_at") or _now(),
+            )
+            result = await asyncio.to_thread(self._execute_workflow_sync, workflow, user_email, handle.scope, inputs)
+            if handle.cancel_requested:
+                self._cancel_workflow_record(run_id, handle.scope, "cancelled after the current synchronous step completed")
+                return
+            pause = (
+                {"node": result.paused_node, "pending": result.pending_approval, "context": result.paused_context}
+                if result.status == "awaiting_approval" else None
+            )
+            updated = self.store.update_workflow_run(
+                run_id,
+                workspace_id=handle.scope,
+                graph=self.workspace_graph(),
+                status=result.status,
+                timeline=base_timeline + list(result.timeline or []),
+                outputs=result.outputs,
+                pause=pause,
+            )
+            self.append_audit_event(
+                "workflow_run",
+                user_email=user_email,
+                workflow_id=workflow.get("id"),
+                status=result.status,
+            )
+            self._results[run_id] = {"run": updated, "result": result.as_dict()}
+        except Exception as exc:
+            run = self.store.get_workflow_run(run_id, workspace_id=handle.scope)
+            timeline = list(run.get("timeline") or [])
+            timeline.append({"event": "execution_failed", "status": "failed", "detail": str(exc), "timestamp": _now()})
+            failed = self.store.update_workflow_run(
+                run_id,
+                workspace_id=handle.scope,
+                graph=self.workspace_graph(),
+                status="failed",
+                timeline=timeline,
+                outputs={"error": str(exc)},
+                pause=None,
+            )
+            self._results[run_id] = {"run": failed, "result": {"status": "failed", "error": str(exc)}}
+        finally:
+            self._handles.pop(run_id, None)
+
+    def _execute_workflow_sync(
+        self,
+        workflow: Dict[str, Any],
+        user_email: Optional[str],
+        scope: Optional[str],
+        inputs: Dict[str, Any],
+    ) -> Any:
+        runners = self.build_workflow_runners(user_email, scope)
+        return WorkflowEngine(runners, hooks=self.hooks).run(workflow, inputs=inputs)
+
+    def _cancel_workflow_record(self, run_id: str, scope: Optional[str], reason: str) -> Dict[str, Any]:
+        run = self.store.get_workflow_run(run_id, workspace_id=scope)
+        timeline = list(run.get("timeline") or [])
+        timeline.append({"event": "execution_cancelled", "status": "cancelled", "reason": reason, "timestamp": _now()})
+        cancelled = self.store.update_workflow_run(
+            run_id,
+            workspace_id=scope,
+            status="cancelled",
+            cancel_reason=reason,
+            cancelled_at=_now(),
+            timeline=timeline,
+            pause=None,
+            graph=self.workspace_graph(),
+        )
+        payload = {"run": cancelled, "result": {"status": "cancelled", "reason": reason}}
+        self._results[run_id] = payload
+        return cancelled
+
+    # ── cancellation/status ──────────────────────────────────────────────
+
+    def cancel(self, run_id: str, *, kind: Optional[str] = None, scope: Optional[str] = None) -> Dict[str, Any]:
+        handle = self._handles.get(run_id)
+        try:
+            run = (
+                self.store.get_workflow_run(run_id, workspace_id=scope)
+                if kind == "workflow" or (handle and handle.kind == "workflow")
+                else self.store.get_agent_run(run_id, workspace_id=scope)
+            )
+        except FileNotFoundError:
+            return {"stopped": False, "reason": "run not found", "run_id": run_id}
+
+        status = str(run.get("status") or "")
+        if status not in ACTIVE_STATUSES:
+            return {"stopped": False, "reason": "run already finished", "run_id": run_id, "status": status}
+
+        if handle is not None:
+            handle.cancel_requested = True
+        target_kind = (kind or (handle.kind if handle else "agent"))
+        updater = self.store.update_workflow_run if target_kind == "workflow" else self.store.update_agent_run
+        updater(
+            run_id,
+            workspace_id=scope,
+            status="cancelling",
+            cancel_requested=True,
+            cancel_requested_at=_now(),
+        )
+        if handle is None:
+            if target_kind == "workflow":
+                self._cancel_workflow_record(run_id, scope, "cancelled; no active worker owned this run")
+            else:
+                self._cancel_agent_record(run_id, scope, "cancelled; no active worker owned this run")
+        return {
+            "stopped": True,
+            "run_id": run_id,
+            "status": "cancelling" if handle is not None else "cancelled",
+            "cancellation": "cooperative",
+            "reason": "cancellation requested; synchronous work finishes its current step before the final cancelled status is stored",
+        }
+
+    async def wait(self, run_id: str, *, timeout: Optional[float] = None) -> Optional[Dict[str, Any]]:
+        handle = self._handles.get(run_id)
+        if handle and handle.task:
+            await asyncio.wait_for(handle.task, timeout=timeout)
+        return self._results.get(run_id)