ltcai 3.5.0 → 4.0.0

package/latticeai/core/model_compat.py

@@ -16,7 +16,7 @@ import logging
 import re
 import threading
 import time
-from dataclasses import dataclass, field, asdict
+from dataclasses import dataclass, asdict
 from typing import Any, Dict, List, Optional, Tuple
 
 logger = logging.getLogger(__name__)

package/latticeai/core/model_resolution.py

@@ -19,7 +19,6 @@
 from __future__ import annotations
 
 import logging
-import re
 from dataclasses import dataclass, field, asdict
 from enum import Enum
 from typing import Any, Dict, List, Optional

package/latticeai/core/multi_agent.py

@@ -14,7 +14,7 @@ from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
 
-MULTI_AGENT_VERSION = "3.5.0"
+MULTI_AGENT_VERSION = "4.0.0"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")
@@ -279,10 +279,13 @@ class AgentRunResult:
     retry_history: List[Dict[str, Any]] = field(default_factory=list)
     plan_review: Dict[str, Any] = field(default_factory=dict)
     memory_snapshots: List[Dict[str, Any]] = field(default_factory=list)
+    # "simulation" = deterministic LLM-free runner; "llm" = model-driven (v4 runtime).
+    mode: str = "simulation"
 
     def as_dict(self) -> Dict[str, Any]:
         return {
             "agent_id": self.agent_id,
+            "mode": self.mode,
             "status": self.status,
             "output": self.output,
             "timeline": self.timeline,
@@ -403,7 +406,221 @@ def default_role_runner(
             ctx.output = ctx.output or f"Released outcome for: {ctx.goal}"
             return {"role": role, "released": True, "summary": ctx.output}
 
-        return {"role": role, "status": "noop"}
+        return {
+            "role": role,
+            "status": "skipped",
+            "reason": "this role has no deterministic behaviour (custom agents require a loaded model)",
+        }
+
+    return runner
+
+
+def _extract_json_object(raw: str) -> Dict[str, Any]:
+    """Parse one JSON object out of an LLM response (fences/prose tolerated)."""
+    import json as _json
+    import re as _re
+
+    text = str(raw or "").strip()
+    fenced = _re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, flags=_re.DOTALL)
+    if fenced:
+        text = fenced.group(1)
+    elif not text.startswith("{"):
+        start, end = text.find("{"), text.rfind("}")
+        if start >= 0 and end > start:
+            text = text[start : end + 1]
+    parsed = _json.loads(text)
+    if not isinstance(parsed, dict):
+        raise ValueError("model returned JSON that is not an object")
+    return parsed
+
+
+def llm_role_runner(
+    *,
+    generate: Callable[..., str],
+    planner_prompt: str,
+    critic_prompt: str,
+    context_provider: Optional[Callable[[str], List[str]]] = None,
+    workflow_runner: Optional[Callable[..., Any]] = None,
+    plugin_runner: Optional[Callable[..., Any]] = None,
+    custom_agents: Optional[Dict[str, Dict[str, Any]]] = None,
+) -> Callable[[str, OrchestrationContext], Dict[str, Any]]:
+    """Model-driven role runner — the real Multi-Agent Runtime (T7b).
+
+    ``generate(message, context, max_tokens, temperature) -> str`` is a
+    synchronous bridge to the loaded model. Honesty contract (design-review
+    amendment): when the model responds but its plan/critique cannot be
+    parsed, the RUN FAILS with the raw output preserved in the records —
+    it never silently falls back to fabricated deterministic artifacts.
+    """
+    base = default_role_runner(
+        workflow_runner=workflow_runner,
+        plugin_runner=plugin_runner,
+        context_provider=context_provider,
+    )
+
+    def _fail(ctx: OrchestrationContext, role: str, reason: str, raw: str) -> Dict[str, Any]:
+        ctx.inputs["__llm_failure__"] = {"role": role, "reason": reason, "raw": raw[:2000]}
+        ctx.review = {
+            "outcome": "reject",
+            "verdict": "fail",
+            "reason": f"{role}: {reason}",
+            "raw_output": raw[:2000],
+            "reviewed_at": _now(),
+        }
+        return {"role": role, "status": "error", "reason": reason, "raw": raw[:2000]}
+
+    def runner(role: str, ctx: OrchestrationContext) -> Dict[str, Any]:
+        failure = ctx.inputs.get("__llm_failure__")
+
+        custom = (custom_agents or {}).get(role)
+        if custom is not None:
+            # Executable registry entry (T7e): the agent's persisted config
+            # (system_prompt, max_tokens, temperature) is actually loaded.
+            cfg = custom.get("config") or {}
+            system = str(
+                cfg.get("system_prompt")
+                or custom.get("description")
+                or f"You are {custom.get('name') or role}."
+            )
+            try:
+                out = str(generate(
+                    ctx.output or ctx.goal,
+                    context=system,
+                    max_tokens=int(cfg.get("max_tokens") or 1024),
+                    temperature=float(cfg.get("temperature") or 0.2),
+                ))
+            except Exception as exc:
+                return _fail(ctx, role, f"custom agent generation failed ({exc})", "")
+            ctx.output = out
+            return {"role": role, "agent": custom.get("name"), "status": "ok",
+                    "output": out[:2000]}
+
+        if role == "planner":
+            research = "\n".join(f"- {item}" for item in (ctx.research or [])[:8])
+            raw = generate(
+                "Produce a JSON execution plan for this goal. Respond with one JSON "
+                'object: {"goal": str, "steps": [{"description": str}, ...]} and nothing else.',
+                context=f"{planner_prompt}\n\nGoal: {ctx.goal}\n\nKnown context:\n{research}",
+                max_tokens=1024,
+                temperature=0.1,
+            )
+            try:
+                parsed = _extract_json_object(str(raw))
+            except Exception as exc:
+                return _fail(ctx, role, f"plan output unparseable ({exc})", str(raw))
+            steps = []
+            for i, step in enumerate(parsed.get("steps") or []):
+                description = step.get("description") if isinstance(step, dict) else str(step)
+                steps.append({"index": i, "description": str(description or f"Step {i + 1}"), "status": "planned"})
+            if not steps:
+                return _fail(ctx, role, "model returned a plan with no steps", str(raw))
+            if ctx.inputs.get("workflow"):
+                steps[0]["workflow"] = ctx.inputs.get("workflow")
+            if ctx.inputs.get("plugin"):
+                steps[0]["plugin"] = ctx.inputs.get("plugin")
+            ctx.plan = steps
+            ctx.plan_id = f"plan-{abs(hash((ctx.goal, len(steps)))) % 10_000_000}"
+            ctx.plan_review = {
+                "plan_id": ctx.plan_id,
+                "outcome": "approve",
+                "reason": "model-generated plan parsed and bounded",
+                "reviewed_at": _now(),
+            }
+            return {"role": role, "plan_id": ctx.plan_id, "steps": len(steps), "plan": steps, "plan_review": ctx.plan_review}
+
+        if role == "executor":
+            if failure:
+                return {"role": role, "status": "error", "reason": f"skipped — {failure['role']} failed"}
+            results = []
+            for step in ctx.plan:
+                outcome: Dict[str, Any] = {"index": step["index"], "description": step["description"]}
+                wf = step.get("workflow")
+                pl = step.get("plugin")
+                if wf and workflow_runner is not None:
+                    try:
+                        outcome["workflow_result"] = workflow_runner(wf, ctx)
+                        ctx.workflow_outputs.append(outcome["workflow_result"])
+                    except Exception as exc:
+                        outcome["workflow_error"] = str(exc)
+                if pl and plugin_runner is not None:
+                    try:
+                        outcome["plugin_result"] = plugin_runner(pl, ctx)
+                        ctx.plugin_outputs.append(outcome["plugin_result"])
+                    except Exception as exc:
+                        outcome["plugin_error"] = str(exc)
+                try:
+                    outcome["result"] = str(generate(
+                        f"Execute this step and return the concrete result only.\n"
+                        f"Goal: {ctx.goal}\nStep: {step['description']}",
+                        context="",
+                        max_tokens=1024,
+                        temperature=0.2,
+                    ))[:4000]
+                except Exception as exc:
+                    outcome["error"] = str(exc)
+                if outcome.get("workflow_error") or outcome.get("plugin_error") or outcome.get("error"):
+                    step["status"] = "failed"
+                    outcome["status"] = "error"
+                else:
+                    step["status"] = "done"
+                    outcome["status"] = "done"
+                results.append(outcome)
+            ctx.executed = results
+            done = [r for r in results if r.get("status") == "done"]
+            ctx.output = "\n\n".join(str(r.get("result") or "") for r in done).strip() or (
+                f"Completed {len(done)}/{len(results)} step(s) for: {ctx.goal}"
+            )
+            return {"role": role, "executed": len(results), "results": results}
+
+        if role == "reviewer":
+            if failure:
+                # Fail-closed: an upstream unparseable model output means this
+                # run is failed, with the raw output preserved — never rescued
+                # by a rubber-stamp review.
+                ctx.review = {
+                    "outcome": "reject",
+                    "verdict": "fail",
+                    "reason": f"{failure['role']} output unparseable",
+                    "raw_output": failure.get("raw"),
+                    "reviewed_at": _now(),
+                }
+                return {"role": role, **ctx.review}
+            raw = generate(
+                "Review this execution. Respond with one JSON object: "
+                '{"approve": bool, "reason": str} and nothing else.',
+                context=(
+                    f"{critic_prompt}\n\nGoal: {ctx.goal}\n\n"
+                    f"Steps: {[s.get('status') for s in ctx.plan]}\n\nOutput:\n{(ctx.output or '')[:3000]}"
+                ),
+                max_tokens=512,
+                temperature=0.1,
+            )
+            try:
+                parsed = _extract_json_object(str(raw))
+                approve = bool(parsed.get("approve"))
+                reason = str(parsed.get("reason") or "")
+            except Exception as exc:
+                ctx.review = {
+                    "outcome": "reject",
+                    "verdict": "fail",
+                    "reason": f"critic output unparseable ({exc})",
+                    "raw_output": str(raw)[:2000],
+                    "reviewed_at": _now(),
+                }
+                return {"role": role, **ctx.review}
+            ctx.review = {
+                "outcome": "approve" if approve else "retry",
+                "verdict": "pass" if approve else "retry",
+                "reason": reason or ("model approved the result" if approve else "model requested a retry"),
+                "confidence": 0.9 if approve else 0.4,
+                "notes": [],
+                "reviewed_at": _now(),
+            }
+            return {"role": role, **ctx.review}
+
+        # researcher / release / anything else: the deterministic behaviour is
+        # real work (memory recall, bookkeeping) — reuse it.
+        return base(role, ctx)
 
     return runner
 
@@ -411,8 +628,21 @@ def default_role_runner(
 class MultiAgentOrchestrator:
     """Drives a role pipeline with handoff, planning, review, and retry."""
 
-    def __init__(self, role_runner: Optional[Callable[[str, OrchestrationContext], Dict[str, Any]]] = None):
+    def __init__(
+        self,
+        role_runner: Optional[Callable[[str, OrchestrationContext], Dict[str, Any]]] = None,
+        mode: str = "simulation",
+        custom_agents: Optional[Dict[str, Dict[str, Any]]] = None,
+    ):
         self.role_runner = role_runner or default_role_runner()
+        # Executable registry entries (T7e): a requested role may be a
+        # registered custom agent id; its config (system_prompt, …) is
+        # actually loaded at run time — registration is no longer a UI illusion.
+        self.custom_agents = dict(custom_agents or {})
+        # Honest execution-mode label persisted on every run record. The
+        # built-in runner never calls a model, so the default is "simulation";
+        # an LLM-backed runner must declare mode="llm" explicitly.
+        self.mode = mode
 
     def _run_role(self, role: str, ctx: OrchestrationContext) -> Dict[str, Any]:
         started = _now()
@@ -473,7 +703,10 @@ class MultiAgentOrchestrator:
             workspace_id=workspace_id,
             inputs=inputs or {},
         )
-        pipeline = [r for r in (roles or list(CORE_PIPELINE)) if r in AGENT_ROLES]
+        pipeline = [
+            r for r in (roles or list(CORE_PIPELINE))
+            if r in AGENT_ROLES or r in self.custom_agents
+        ]
         if not pipeline:
             pipeline = list(CORE_PIPELINE)
         max_retries = max(0, int(max_retries or 0))
@@ -558,4 +791,5 @@ class MultiAgentOrchestrator:
             retry_history=ctx.retry_history,
             plan_review=ctx.plan_review,
             memory_snapshots=ctx.memory_snapshots,
+            mode=self.mode,
         )

package/latticeai/core/security.py

@@ -6,7 +6,7 @@ import re
 import secrets
 import threading
 import time
-from typing import Dict, List, Optional
+from typing import Dict, List
 
 from fastapi import HTTPException
 

package/latticeai/core/sessions.py

@@ -1,5 +1,12 @@
-"""File-backed session store with sliding-window TTL."""
+"""File-backed session store with sliding-window TTL.
 
+v4: bearer tokens are stored **hashed** (sha256) at rest — a process that can
+read ``sessions.json`` must not be able to hijack every session. Pre-v4 files
+holding raw tokens are migrated transparently on load (sessions survive the
+upgrade; the raw token never touches disk again).
+"""
+
+import hashlib
 import json
 import logging
 import os
@@ -13,6 +20,16 @@ SESSION_TTL = 60 * 60 * 24  # 24 hours
 SESSION_REFRESH_THRESHOLD = 60 * 15  # only persist if >15 min since last bump
 _lock = threading.Lock()
 
+_HEX64 = frozenset("0123456789abcdef")
+
+
+def _hash_token(token: str) -> str:
+    return hashlib.sha256((token or "").encode("utf-8")).hexdigest()
+
+
+def _looks_hashed(key: str) -> bool:
+    return len(key) == 64 and set(key) <= _HEX64
+
 
 def _sessions_file(data_dir: Optional[Path] = None) -> Path:
     d = data_dir or Path(os.getenv("LATTICEAI_DATA_DIR") or (Path.home() / ".ltcai"))
@@ -25,7 +42,19 @@ def load_sessions(data_dir: Optional[Path] = None) -> Dict[str, tuple]:
         f = _sessions_file(data_dir)
         if f.exists():
             raw = json.loads(f.read_text())
-            return {k: tuple(v) for k, v in raw.items()}
+            sessions: Dict[str, tuple] = {}
+            migrated = False
+            for key, value in raw.items():
+                if _looks_hashed(key):
+                    sessions[key] = tuple(value)
+                else:
+                    # Pre-v4 entry: the key IS the raw bearer token. Re-key it
+                    # under its hash so the plaintext stops living on disk.
+                    sessions[_hash_token(key)] = tuple(value)
+                    migrated = True
+            if migrated:
+                persist_sessions(sessions, data_dir)
+            return sessions
     except Exception as e:
         logging.warning("load_sessions failed (starting empty): %s", e)
     return {}
@@ -46,27 +75,28 @@ class SessionStore:
     def create(self, email: str) -> str:
         token = secrets.token_urlsafe(32)
         with _lock:
-            self._sessions[token] = (email, time.time())
+            self._sessions[_hash_token(token)] = (email, time.time())
             persist_sessions(self._sessions, self._data_dir)
         return token
 
     def get_email(self, token: str) -> Optional[str]:
         now = time.time()
+        key = _hash_token(token)
         with _lock:
-            entry = self._sessions.get(token)
+            entry = self._sessions.get(key)
             if entry is None:
                 return None
             email, created_at = entry
             if now - created_at > SESSION_TTL:
-                self._sessions.pop(token, None)
+                self._sessions.pop(key, None)
                 persist_sessions(self._sessions, self._data_dir)
                 return None
             if now - created_at > SESSION_REFRESH_THRESHOLD:
-                self._sessions[token] = (email, now)
+                self._sessions[key] = (email, now)
                 persist_sessions(self._sessions, self._data_dir)
             return email
 
     def invalidate(self, token: str) -> None:
         with _lock:
-            self._sessions.pop(token, None)
+            self._sessions.pop(_hash_token(token), None)
             persist_sessions(self._sessions, self._data_dir)

package/latticeai/core/workflow_engine.py

@@ -185,11 +185,17 @@ def _evaluate_condition(config: Dict[str, Any], context: Dict[str, Any]) -> bool
 class WorkflowRun:
     workflow_id: Optional[str]
     name: str
-    status: str = "ok"  # ok | failed | partial
+    status: str = "ok"  # ok | failed | partial | awaiting_approval
     timeline: List[Dict[str, Any]] = field(default_factory=list)
     outputs: Dict[str, Any] = field(default_factory=dict)
     started_at: str = field(default_factory=_now)
     finished_at: Optional[str] = None
+    # Suspension cursor (status == awaiting_approval): the paused node, what
+    # it is waiting for, and a JSON-serializable context snapshot resume()
+    # re-enters with — completed nodes are never re-executed.
+    paused_node: Optional[str] = None
+    pending_approval: Optional[Dict[str, Any]] = None
+    paused_context: Optional[Dict[str, Any]] = None
 
     def as_dict(self) -> Dict[str, Any]:
         return {
@@ -201,9 +207,36 @@ class WorkflowRun:
             "started_at": self.started_at,
             "finished_at": self.finished_at,
             "step_count": len(self.timeline),
+            "paused_node": self.paused_node,
+            "pending_approval": self.pending_approval,
+            "paused_context": self.paused_context,
         }
 
 
+class ApprovalRequired(Exception):
+    """A node needs an explicit human decision before it may execute.
+
+    Raised by governed runners (e.g. a non-auto-approve tool). The engine
+    pauses the run into ``awaiting_approval`` with a serializable cursor —
+    it never records a fake success and never silently skips the node.
+    """
+
+    def __init__(self, message: str, *, tool: Optional[str] = None,
+                 args: Optional[Dict[str, Any]] = None,
+                 permission: Optional[Dict[str, Any]] = None):
+        super().__init__(message)
+        self.tool = tool
+        self.args = args or {}
+        self.permission = permission or {}
+
+
+def _json_safe(value: Any) -> Any:
+    """Round-trip through JSON so paused context is durably serializable."""
+    import json as _json
+
+    return _json.loads(_json.dumps(value, ensure_ascii=False, default=str))
+
+
 class WorkflowEngine:
     """Interprets a validated workflow definition over injected runners.
 
@@ -212,6 +245,11 @@ class WorkflowEngine:
     node as ``skipped`` with a reason rather than failing the whole run, so a
     workflow that references a capability the host has not wired degrades
     gracefully (and the gap is visible in the timeline).
+
+    Suspension model (v4): a runner raising :class:`ApprovalRequired` pauses
+    the run (status ``awaiting_approval``) with the node cursor and a
+    JSON-serializable context snapshot. :meth:`resume` re-enters at the
+    paused node — completed nodes are NEVER re-executed.
     """
 
     def __init__(self, runners: Optional[Dict[str, Callable[..., Any]]] = None, *, hooks: Any = None):
@@ -243,8 +281,60 @@ class WorkflowEngine:
 
         nodes = {node["id"]: node for node in definition["nodes"]}
         context: Dict[str, Any] = {"inputs": inputs or {}, **(inputs or {})}
-
         current = _entry_node(definition["nodes"])
+        return self._execute(definition, run, nodes, context, current)
+
+    def resume(
+        self,
+        workflow: Dict[str, Any],
+        *,
+        paused_node: str,
+        paused_context: Dict[str, Any],
+        approved: bool,
+        prior_timeline: Optional[List[Dict[str, Any]]] = None,
+    ) -> WorkflowRun:
+        """Re-enter a paused run at its cursor; completed nodes never re-run.
+
+        ``approved=True`` marks the paused node as human-approved (its runner
+        sees the node id in ``context['__approved_nodes__']``); ``False``
+        records an explicit denial and fails the run honestly.
+        """
+        definition = normalize_definition(workflow)
+        nodes = {node["id"]: node for node in definition["nodes"]}
+        node = nodes.get(paused_node)
+        run = WorkflowRun(workflow_id=definition.get("id"), name=definition.get("name") or "workflow")
+        if prior_timeline:
+            run.timeline.extend(prior_timeline)
+        if node is None:
+            run.status = "failed"
+            run.timeline.append({"node": paused_node, "type": "resume", "status": "failed",
+                                 "reason": "paused node no longer exists in the definition",
+                                 "timestamp": _now()})
+            run.finished_at = _now()
+            return run
+        context: Dict[str, Any] = dict(paused_context or {})
+        if not approved:
+            run.status = "failed"
+            run.timeline.append({"node": paused_node, "type": node.get("type"),
+                                 "name": node.get("name") or paused_node,
+                                 "status": "denied",
+                                 "reason": "approval denied by the user",
+                                 "timestamp": _now()})
+            run.finished_at = _now()
+            return run
+        approvals = set(context.get("__approved_nodes__") or [])
+        approvals.add(paused_node)
+        context["__approved_nodes__"] = sorted(approvals)
+        return self._execute(definition, run, nodes, context, node)
+
+    def _execute(
+        self,
+        definition: Dict[str, Any],
+        run: WorkflowRun,
+        nodes: Dict[str, Dict[str, Any]],
+        context: Dict[str, Any],
+        current: Optional[Dict[str, Any]],
+    ) -> WorkflowRun:
         steps = 0
         had_error = False
         had_skip = False
@@ -301,6 +391,28 @@ class WorkflowEngine:
                 entry["result"] = result
                 context["last_output"] = result
                 context[nid] = result
+            except ApprovalRequired as pause:
+                # Suspend — never a fake success, never a silent skip.
+                entry["status"] = "awaiting_approval"
+                entry["pending"] = {
+                    "tool": pause.tool, "args": pause.args,
+                    "permission": pause.permission, "reason": str(pause),
+                }
+                run.timeline.append(entry)
+                run.status = "awaiting_approval"
+                run.paused_node = nid
+                run.pending_approval = entry["pending"]
+                try:
+                    run.paused_context = _json_safe(context)
+                except Exception:
+                    run.paused_context = {"inputs": context.get("inputs") or {}}
+                if self.hooks is not None:
+                    self.hooks.fire_hook(
+                        "post_workflow", "workflow.paused",
+                        payload={"workflow_id": definition.get("id"),
+                                 "status": run.status, "node": nid},
+                    )
+                return run
             except Exception as exc:
                 entry["status"] = "error"
                 entry["reason"] = str(exc)