ltcai 9.6.0 → 9.8.0

package/latticeai/core/agent_eval.py

@@ -34,6 +34,7 @@ from latticeai.core.agent import (
     AgentState,
     SingleAgentRuntime,
 )
+from latticeai.tools import ToolError
 
 _AUTO_POLICY = {
     "auto_approve": True, "risk": "low", "shell": False, "network": False,
@@ -43,6 +44,61 @@ _DESTRUCTIVE_POLICY = {
     "auto_approve": False, "risk": "destructive", "shell": True, "network": False,
     "destructive": True, "sandbox": False, "rollback": "none",
 }
+# Mirrors production write-tool governance when the change governor is wired:
+# not auto-approved, so only the governor's additive/proposed verdicts (never
+# the model) decide whether a write runs or is staged for review.
+_GOVERNED_WRITE_POLICY = {
+    "auto_approve": False, "risk": "write", "shell": False, "network": False,
+    "destructive": False, "sandbox": "workspace", "rollback": "git",
+}
+
+# Deterministic Brain-port fixtures. Grounding scenarios assert their final
+# message against these exact values (node ids, concepts, snippets), so a
+# response that was not derived from the tool result — a hallucination in
+# loop terms — cannot pass.
+_EVAL_INGEST_RESULT = {
+    "ok": True,
+    "node_id": "node-ing-1",
+    "concepts": ["mlx", "quantization"],
+    "relations": [{"from": "mlx", "to": "quantization", "type": "related_to"}],
+}
+_EVAL_SEARCH_RESULT = {
+    "ok": True,
+    "matches": [
+        {"id": "node-42", "title": "Q3 roadmap", "snippet": "ship v9.8 by August"}
+    ],
+    "count": 1,
+}
+
+
+class _EvalChangeGovernor:
+    """Deterministic stand-in for ChangeProposalService's governor port.
+
+    Mirrors the wire contract of
+    :meth:`latticeai.services.change_proposals.ChangeProposalService.review`:
+    ``None`` falls through, additive writes get ``allow_additive``, and
+    mutations of "existing" paths come back ``proposed`` with a proposal id —
+    exactly what the agent loop routes into the review queue.
+    """
+
+    governed_tools = frozenset({"write_file", "edit_file"})
+
+    def __init__(self) -> None:
+        self.proposals: List[Dict[str, Any]] = []
+
+    def review(self, name, args, *, policy=None, user_email=None, workspace_id=None, conversation_id=None):
+        if name not in self.governed_tools:
+            return None
+        path = str(args.get("path") or "")
+        if path.startswith("existing"):
+            proposal = {"id": f"eval-proposal-{len(self.proposals) + 1}", "path": path}
+            self.proposals.append(proposal)
+            return {
+                "decision": "proposed",
+                "classification": {"change_class": "mutation"},
+                "proposal": proposal,
+            }
+        return {"decision": "allow_additive", "classification": {"change_class": "additive"}}
 
 
 @dataclass
@@ -56,6 +112,16 @@ class Scenario:
     expect_min: Dict[str, int] = field(default_factory=dict)
     expect_exact: Dict[str, Any] = field(default_factory=dict)
     expect_tool_outcomes: Dict[str, int] = field(default_factory=dict)
+    # Exact ordered sequence of tool names that actually executed (successful
+    # execute_tool calls only — proposed/blocked/failed calls never run).
+    expect_tool_calls: List[str] = field(default_factory=list)
+    # Wire a change governor so governed tools follow the proposal path.
+    use_governor: bool = False
+    # Substrings the final user-facing message must contain. Grounding
+    # scenarios point these at tokens that only exist in the fake tool port's
+    # canned results (node ids, snippets, proposal ids), so an answer that is
+    # not grounded in the retrieved/executed evidence fails the scenario.
+    expect_final_contains: List[str] = field(default_factory=list)
     max_steps: int = 8
 
 
@@ -69,7 +135,12 @@ class _Req:
         self.message = message
 
 
-def _build_deps(replies: List[str], tool_log: List[Dict[str, Any]]) -> AgentDeps:
+def _build_deps(
+    replies: List[str],
+    tool_log: List[Dict[str, Any]],
+    *,
+    governor: Any = None,
+) -> AgentDeps:
     queue = list(replies)
 
     async def generate_as(model_id, message, context, max_tokens, temperature):
@@ -83,14 +154,27 @@ def _build_deps(replies: List[str], tool_log: List[Dict[str, Any]]) -> AgentDeps
         return '{"action": "noop"}'
 
     def execute_tool(name: str, args: dict) -> dict:
+        # File-producing tools fail like the real dispatcher when the model
+        # forgets the target path — scenarios use this to exercise recovery.
+        if name in ("write_file", "generate_file") and not args.get("path"):
+            raise ToolError(f"{name} requires args.path")
         tool_log.append({"name": name, "args": args})
+        # Brain ports return canned, stable payloads so grounding scenarios
+        # can assert the final answer cites what was actually retrieved.
+        if name == "knowledge_graph_ingest":
+            return dict(_EVAL_INGEST_RESULT)
+        if name == "knowledge_graph_search":
+            return dict(_EVAL_SEARCH_RESULT)
         return {"ok": True, "path": args.get("path", "")}
 
     def policy_for(name: str, args: dict) -> dict:
         if name == "delete_everything":
             return dict(_DESTRUCTIVE_POLICY)
+        if governor is not None and name in ("write_file", "edit_file"):
+            return dict(_GOVERNED_WRITE_POLICY)
         return dict(_AUTO_POLICY)
 
+    write_policy = dict(_GOVERNED_WRITE_POLICY) if governor is not None else dict(_AUTO_POLICY)
     return AgentDeps(
         generate_as=generate_as,
         generate=generate,
@@ -99,17 +183,21 @@ def _build_deps(replies: List[str], tool_log: List[Dict[str, Any]]) -> AgentDeps
         risk_level=lambda p: p["risk"],
         check_role=lambda name, user: None,
         tool_governance={
-            "write_file": dict(_AUTO_POLICY),
+            "write_file": write_policy,
             "read_file": dict(_AUTO_POLICY),
+            "generate_file": dict(_AUTO_POLICY),
+            "knowledge_graph_ingest": dict(_AUTO_POLICY),
+            "knowledge_graph_search": dict(_AUTO_POLICY),
             "delete_everything": dict(_DESTRUCTIVE_POLICY),
         },
-        file_create_actions=frozenset({"write_file"}),
+        file_create_actions=frozenset({"write_file", "generate_file"}),
         recent_chat_context=lambda **kw: "",
         clear_history=lambda keep: {"ok": True},
         knowledge_save=lambda *a, **kw: None,
         audit=lambda *a, **kw: None,
         planner_prompt="plan", executor_prompt="exec", critic_prompt="critic",
         memory_updater_prompt="mem", agent_root=Path("/tmp/agent-eval"),
+        change_governor=governor,
     )
 
 
@@ -193,12 +281,185 @@ def default_scenarios() -> List[Scenario]:
             expect_min={"parse_errors": 3},
             expect_exact={"tool_outcomes": {}},
         ),
+        # ── file generation ─────────────────────────────────────────────
+        Scenario(
+            name="file-generation-happy-path",
+            replies=[
+                '{"action": "plan", "goal": "generate landing page", '
+                '"steps": [{"action": "generate_file"}]}',
+                # Small local models wrap the payload in reasoning + fences;
+                # the loop must still extract one clean tool call.
+                "<think>layout first, then hero copy</think>\n"
+                '```json\n{"thoughts": "produce the full document", '
+                '"action": "generate_file", "args": {"path": "site/index.html", '
+                '"content": "<!DOCTYPE html><html><body>hello</body></html>"}}\n```',
+                _FINAL,
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_tool_outcomes={"ok": 1},
+            expect_tool_calls=["generate_file"],
+        ),
+        Scenario(
+            name="file-generation-bad-args-recovers",
+            replies=[
+                '{"action": "plan", "goal": "generate report", '
+                '"steps": [{"action": "generate_file"}]}',
+                # Malformed tool call: the model forgot the target path, the
+                # tool port fails like the real dispatcher would…
+                '{"thoughts": "write it", "action": "generate_file", '
+                '"args": {"content": "<html></html>"}}',
+                # …and the next turn recovers with a complete call.
+                '{"thoughts": "add the missing path", "action": "generate_file", '
+                '"args": {"path": "reports/q3.html", "content": "<html></html>"}}',
+                _FINAL,
+                _PASS,
+            ],
+            expect_tool_outcomes={"error": 1, "ok": 1},
+            expect_tool_calls=["generate_file"],
+        ),
+        # ── multi-step workflow ─────────────────────────────────────────
+        Scenario(
+            name="multi-step-workflow-chain",
+            replies=[
+                '{"action": "plan", "goal": "read spec, generate report, save summary", '
+                '"steps": [{"action": "read_file"}, {"action": "generate_file"}, '
+                '{"action": "write_file"}]}',
+                '{"thoughts": "step 1: read the source spec", '
+                '"action": "read_file", "args": {"path": "spec.md"}}',
+                '{"thoughts": "step 2: the spec asks for an HTML report", '
+                '"action": "generate_file", "args": {"path": "report.html", '
+                '"content": "<html><body>report</body></html>"}}',
+                '{"thoughts": "step 3: persist a short summary next to it", '
+                '"action": "write_file", "args": {"path": "summary.txt", '
+                '"content": "report generated"}}',
+                _FINAL,
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_min={"llm_calls": 6},
+            expect_tool_outcomes={"ok": 3},
+            expect_tool_calls=["read_file", "generate_file", "write_file"],
+        ),
+        # ── governed-tool proposal path ─────────────────────────────────
+        Scenario(
+            name="governed-write-proposal-path",
+            use_governor=True,
+            replies=[
+                # write_file is NOT auto-approved here, but it is governed —
+                # approve() must not hard-block the plan (core invariant:
+                # governed tools are excluded from the non-auto set).
+                '{"action": "plan", "goal": "update existing page, add new note", '
+                '"steps": [{"action": "write_file"}]}',
+                # Mutation of existing content → staged as proposal, not written.
+                '{"thoughts": "rewrite the existing page", "action": "write_file", '
+                '"args": {"path": "existing/site.html", "content": "<new>"}}',
+                # Additive create → governor allows it to run immediately.
+                '{"thoughts": "add a fresh note", "action": "write_file", '
+                '"args": {"path": "fresh/new-note.md", "content": "hello"}}',
+                _FINAL,
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_tool_outcomes={"proposed": 1, "ok": 1},
+            expect_tool_calls=["write_file"],
+        ),
+        # ── brain ingestion ─────────────────────────────────────────────
+        Scenario(
+            name="ingestion-chain-confirms-save",
+            replies=[
+                '{"action": "plan", "goal": "ingest web article into the Brain", '
+                '"steps": [{"action": "knowledge_graph_ingest"}]}',
+                # Weak-model dressing (think block) around a clean ingest call.
+                "<think>save the text first, then confirm with the node id</think>\n"
+                '{"thoughts": "ingest the pasted article", '
+                '"action": "knowledge_graph_ingest", "args": {"source": "web", '
+                '"url": "https://example.com/mlx-notes", '
+                '"text": "MLX quantization notes"}}',
+                # Confirmation cites the node id the ingest port returned —
+                # the loop must carry the tool result into the final turn.
+                '{"action": "final", "message": "Saved to the Brain as node-ing-1."}',
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_tool_outcomes={"ok": 1},
+            expect_tool_calls=["knowledge_graph_ingest"],
+            expect_final_contains=["node-ing-1"],
+        ),
+        # ── concept extraction ──────────────────────────────────────────
+        Scenario(
+            name="concept-extraction-reflected-in-answer",
+            replies=[
+                '{"action": "plan", "goal": "ingest the note and report extracted concepts", '
+                '"steps": [{"action": "knowledge_graph_ingest"}]}',
+                '{"thoughts": "ingest the file so the pipeline extracts concepts", '
+                '"action": "knowledge_graph_ingest", "args": {"source": "file", '
+                '"path": "notes/mlx.md", "text": "quantization on MLX"}}',
+                # The confirmation must surface what the pipeline extracted
+                # (concepts + relation), not merely say "done".
+                '{"action": "final", "message": "Ingested notes/mlx.md - extracted '
+                'concepts mlx and quantization (mlx -> quantization, related_to)."}',
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_tool_outcomes={"ok": 1},
+            expect_tool_calls=["knowledge_graph_ingest"],
+            expect_final_contains=["mlx", "quantization", "related_to"],
+        ),
+        # ── RAG-grounded answer ─────────────────────────────────────────
+        Scenario(
+            name="rag-grounded-answer-cites-retrieval",
+            replies=[
+                '{"action": "plan", "goal": "answer from the Brain, not from priors", '
+                '"steps": [{"action": "knowledge_graph_search"}]}',
+                '{"thoughts": "retrieve evidence before answering", '
+                '"action": "knowledge_graph_search", "args": {"query": "Q3 roadmap"}}',
+                # Grounded: cites the retrieved node id and snippet — both exist
+                # only in the fake port's canned search result, so an ungrounded
+                # (hallucinated) final message cannot satisfy the expectation.
+                '{"action": "final", "message": "Per Q3 roadmap (node-42): ship v9.8 by August."}',
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_tool_outcomes={"ok": 1},
+            expect_tool_calls=["knowledge_graph_search"],
+            expect_final_contains=["node-42", "ship v9.8 by August"],
+        ),
+        # ── automation suggestion under proposal-first governance ───────
+        Scenario(
+            name="automation-suggestion-proposal-first",
+            use_governor=True,
+            replies=[
+                '{"action": "plan", "goal": "recurring daily question detected - stage a '
+                'digest automation for review", '
+                '"steps": [{"action": "knowledge_graph_search"}, {"action": "write_file"}]}',
+                # Evidence first: confirm the repeated pattern from the Brain.
+                '{"thoughts": "confirm the pattern is real before suggesting automation", '
+                '"action": "knowledge_graph_search", '
+                '"args": {"query": "recurring question daily digest"}}',
+                # Proposal-first: changing existing automation state is a
+                # mutation, so the governor stages it as a review proposal —
+                # the model can never silently install/enable an automation.
+                '{"thoughts": "stage the automation as a reviewable proposal", '
+                '"action": "write_file", '
+                '"args": {"path": "existing/automations/daily-digest.json", '
+                '"content": "trigger interval, disabled draft, review_queue"}}',
+                '{"action": "final", "message": "Automation staged for review as '
+                'eval-proposal-1 - enable it from the review queue."}',
+                _PASS,
+            ],
+            expect_exact={"parse_errors": 0},
+            expect_tool_outcomes={"ok": 1, "proposed": 1},
+            expect_tool_calls=["knowledge_graph_search"],
+            expect_final_contains=["eval-proposal-1", "review"],
+        ),
     ]
 
 
 async def _run_scenario(scenario: Scenario) -> Dict[str, Any]:
     tool_log: List[Dict[str, Any]] = []
-    deps = _build_deps(scenario.replies, tool_log)
+    governor = _EvalChangeGovernor() if scenario.use_governor else None
+    deps = _build_deps(scenario.replies, tool_log, governor=governor)
     runtime = SingleAgentRuntime(deps)
     ctx = AgentRunContext()
     ctx.state = AgentState.PLANNING
@@ -226,6 +487,17 @@ async def _run_scenario(scenario: Scenario) -> Dict[str, Any]:
             failures.append(
                 f"tool_outcomes[{outcome}]={summary['tool_outcomes'].get(outcome, 0)} != {count}"
             )
+    executed = [call["name"] for call in tool_log]
+    if scenario.expect_tool_calls and executed != scenario.expect_tool_calls:
+        failures.append(f"tool_calls={executed} != {scenario.expect_tool_calls}")
+    for needle in scenario.expect_final_contains:
+        if needle not in ctx.final_message:
+            failures.append(f"final_message missing {needle!r}")
+    if governor is not None and summary["tool_outcomes"].get("proposed", 0) != len(governor.proposals):
+        failures.append(
+            f"governor proposals={len(governor.proposals)} != "
+            f"traced proposed={summary['tool_outcomes'].get('proposed', 0)}"
+        )
     return {
         "name": scenario.name,
         "ok": not failures,
@@ -233,6 +505,8 @@ async def _run_scenario(scenario: Scenario) -> Dict[str, Any]:
         "final_state": ctx.state.value,
         "summary": summary,
         "tool_calls": len(tool_log),
+        "executed_tools": executed,
+        "proposals": len(governor.proposals) if governor is not None else 0,
     }
 
 

package/latticeai/core/legacy_compatibility.py

@@ -14,7 +14,7 @@ from pathlib import Path
 from typing import Any, Dict, List
 
 
-LEGACY_COMPATIBILITY_VERSION = "9.6.0"
+LEGACY_COMPATIBILITY_VERSION = "9.8.0"
 
 
 @dataclass(frozen=True)
@@ -82,6 +82,20 @@ LEGACY_SHIMS: List[LegacyShim] = [
         reason="Old gardener scripts referenced the root reinforcement helper.",
         removal_phase="major-release-after-8.x",
     ),
+    LegacyShim(
+        path="mcp_registry.py",
+        owner="latticeai.core.mcp_registry",
+        replacement="import latticeai.core.mcp_registry",
+        reason="Older integrations imported the MCP registry from the repo root.",
+        removal_phase="major-release-after-9.x",
+    ),
+    LegacyShim(
+        path="llm_router.py",
+        owner="latticeai.models.router",
+        replacement="import latticeai.models.router",
+        reason="Historical scripts and tests imported the LLM router from the repo root.",
+        removal_phase="major-release-after-9.x",
+    ),
     LegacyShim(
         path="server.py",
         owner="latticeai.server_app",

package/latticeai/core/marketplace.py

@@ -11,7 +11,7 @@ from copy import deepcopy
 from typing import Any, Dict, List, Optional
 
 
-MARKETPLACE_VERSION = "9.6.0"
+MARKETPLACE_VERSION = "9.8.0"
 TEMPLATE_KINDS = ("plugin", "workflow", "agent", "ingestion_bridge")
 
 

package/latticeai/core/workspace_os.py

@@ -49,7 +49,7 @@ __all__ = [
     "remove_skill_directory",
 ]
 
-WORKSPACE_OS_VERSION = "9.6.0"
+WORKSPACE_OS_VERSION = "9.8.0"
 
 # Workspace types separate single-user Personal workspaces from shared
 # Organization workspaces. Both keep the same local-first JSON store; the type

package/latticeai/runtime/router_registration.py

@@ -604,6 +604,7 @@ def register_review_and_brain_tail_routers(
     gardener: Any,
     create_setup_router: Any,
     model_router: Any,
+    change_proposals: Any = None,
 ) -> Any:
     """Register the final review/browser/brain tail routes in legacy order."""
 
@@ -616,6 +617,7 @@ def register_review_and_brain_tail_routers(
             gate_write=gate_write,
             run_review_item=run_review_item,
             append_audit_event=append_audit_event,
+            change_proposals=change_proposals,
         ),
         create_browser_router(
             pipeline=ingestion_pipeline,

package/latticeai/services/architecture_readiness.py

@@ -17,7 +17,7 @@ from typing import Any, Dict, List
 from latticeai.core.legacy_compatibility import legacy_shim_report
 
 
-ARCHITECTURE_VERSION_TARGET = "9.6.0"
+ARCHITECTURE_VERSION_TARGET = "9.8.0"
 
 PREFERRED_REFACTORING_ORDER = [
     "agent-runtime",