ltcai 9.7.0 → 9.9.0

package/latticeai/core/tool_governor.py

@@ -44,6 +44,63 @@ _ADDITIVE_TOOLS = frozenset({
     "knowledge_graph_ingest",
 })
 
+# ── Mutating-tool inventory (single source of truth) ─────────────────────
+# Every tool with a side effect is classified into exactly one category so a
+# CI check can prove nothing slips through ungoverned. Categories:
+#   new_artifact         — only ever creates new content (additive)
+#   existing_content_update — can rewrite content that already exists
+#   delete               — removes existing content
+#   external_side_effect — acts outside the Brain (shell, deploy, desktop, net)
+#   internal_state       — agent-internal bookkeeping, not user content
+NEW_ARTIFACT = "new_artifact"
+EXISTING_CONTENT_UPDATE = "existing_content_update"
+DELETE = "delete"
+EXTERNAL_SIDE_EFFECT = "external_side_effect"
+INTERNAL_STATE = "internal_state"
+
+MUTATING_TOOL_INVENTORY: Dict[str, str] = {
+    # text files — fully proposal-capable (staged + applied as reviewed)
+    "write_file": EXISTING_CONTENT_UPDATE,
+    "edit_file": EXISTING_CONTENT_UPDATE,
+    # binary/document creators — can overwrite an existing file, but their
+    # output cannot be staged as a text diff, so an overwrite is fail-closed.
+    "create_docx": EXISTING_CONTENT_UPDATE,
+    "create_xlsx": EXISTING_CONTENT_UPDATE,
+    "create_pptx": EXISTING_CONTENT_UPDATE,
+    "create_pdf": EXISTING_CONTENT_UPDATE,
+    # home-sandbox write — same story: overwrite is fail-closed until the
+    # proposal service learns to stage home-sandbox paths.
+    "local_write": EXISTING_CONTENT_UPDATE,
+    # additive-only writes
+    "create_web_project": NEW_ARTIFACT,
+    "knowledge_save": NEW_ARTIFACT,
+    "obsidian_save": NEW_ARTIFACT,
+    "knowledge_graph_ingest": NEW_ARTIFACT,
+    # agent-internal state
+    "todo_write": INTERNAL_STATE,
+    # deletions
+    "delete_file": DELETE,
+    "remove_file": DELETE,
+    "clear_history": DELETE,
+    # external side effects (approval-gated, never proposal-based)
+    "run_command": EXTERNAL_SIDE_EFFECT,
+    "build_project": EXTERNAL_SIDE_EFFECT,
+    "deploy_project": EXTERNAL_SIDE_EFFECT,
+    "computer_click": EXTERNAL_SIDE_EFFECT,
+    "computer_type": EXTERNAL_SIDE_EFFECT,
+    "computer_key": EXTERNAL_SIDE_EFFECT,
+    "computer_scroll": EXTERNAL_SIDE_EFFECT,
+    "computer_drag": EXTERNAL_SIDE_EFFECT,
+    "computer_move": EXTERNAL_SIDE_EFFECT,
+    "computer_open_app": EXTERNAL_SIDE_EFFECT,
+    "computer_open_url": EXTERNAL_SIDE_EFFECT,
+}
+
+# Tools whose existing-content update the ChangeProposalService can actually
+# stage AND apply as a reviewed proposal. A tool that is proposal_required but
+# NOT here is fail-closed (blocked) rather than silently applied.
+PROPOSAL_CAPABLE_TOOLS = frozenset({"write_file", "edit_file"})
+
 
 def classify_tool_call(
     name: str,
@@ -83,19 +140,53 @@ def classify_tool_call(
         change = CHANGE_ADDITIVE
         reason = "write tool without an existing target"
 
+    proposal_required = change in {CHANGE_MUTATION, CHANGE_DESTRUCTIVE}
+    proposal_supported = name in PROPOSAL_CAPABLE_TOOLS
+    # A change that must be reviewed as a proposal but that we cannot stage is
+    # fail-closed: callers must block it instead of applying it silently.
+    fail_closed = proposal_required and not proposal_supported
+
     return {
         "tool": name,
         "change_class": change,
-        "proposal_required": change in {CHANGE_MUTATION, CHANGE_DESTRUCTIVE},
+        "proposal_required": proposal_required,
+        "proposal_supported": proposal_supported,
+        "fail_closed": fail_closed,
         "reason": reason,
     }
 
 
+def assert_governance_coverage(tool_names: "Any") -> None:
+    """Raise if any side-effecting registry tool is not classified.
+
+    A new mutating tool added to the registry without an inventory entry makes
+    this raise, so CI fails closed instead of shipping an ungoverned mutator.
+    Read-only tools (risk ``read``) are intentionally exempt.
+    """
+    missing = [
+        name for name in tool_names
+        if name not in MUTATING_TOOL_INVENTORY
+    ]
+    if missing:
+        raise ValueError(
+            "ungoverned mutating tools (add to MUTATING_TOOL_INVENTORY): "
+            + ", ".join(sorted(missing))
+        )
+
+
 __all__ = [
     "CHANGE_READ",
     "CHANGE_ADDITIVE",
     "CHANGE_MUTATION",
     "CHANGE_DESTRUCTIVE",
     "CHANGE_EXEC",
+    "NEW_ARTIFACT",
+    "EXISTING_CONTENT_UPDATE",
+    "DELETE",
+    "EXTERNAL_SIDE_EFFECT",
+    "INTERNAL_STATE",
+    "MUTATING_TOOL_INVENTORY",
+    "PROPOSAL_CAPABLE_TOOLS",
     "classify_tool_call",
+    "assert_governance_coverage",
 ]

package/latticeai/core/workspace_os.py

@@ -49,7 +49,7 @@ __all__ = [
     "remove_skill_directory",
 ]
 
-WORKSPACE_OS_VERSION = "9.7.0"
+WORKSPACE_OS_VERSION = "9.9.0"
 
 # Workspace types separate single-user Personal workspaces from shared
 # Organization workspaces. Both keep the same local-first JSON store; the type

package/latticeai/services/architecture_readiness.py

@@ -17,7 +17,7 @@ from typing import Any, Dict, List
 from latticeai.core.legacy_compatibility import legacy_shim_report
 
 
-ARCHITECTURE_VERSION_TARGET = "9.7.0"
+ARCHITECTURE_VERSION_TARGET = "9.9.0"
 
 PREFERRED_REFACTORING_ORDER = [
     "agent-runtime",

package/latticeai/services/automation_intelligence.py

@@ -22,6 +22,14 @@ automation suggestions:
 
 Every suggestion id is deterministic, so install calls are idempotent and the
 UI can safely re-request suggestions without duplicates.
+
+v9.8.0 quality layer (additive): every suggestion carries a deterministic
+``confidence`` score plus its ``confidence_factors`` evidence (repeat count,
+distinct phrasings, intent match, related Brain nodes / indexed files);
+suggestions below a minimum confidence are suppressed, duplicate suggestions
+targeting an already-suggested or already-installed starter recipe are
+deduplicated, and the response reports the suppression counters under
+``quality``.
 """
 
 from __future__ import annotations
@@ -40,6 +48,50 @@ _MIN_PATTERN_COUNT = 2
 _MAX_HISTORY = 4000
 _SIGNATURE_SIMILARITY = 0.6
 
+# Suggestion quality gates (v9.8.0, additive). Suggestions below the minimum
+# confidence are suppressed (insufficient evidence); suggestions between the
+# minimum and the low-confidence threshold are shown but flagged so the UI
+# can render them less prominently.
+_MIN_SUGGESTION_CONFIDENCE = 0.35
+_LOW_CONFIDENCE_THRESHOLD = 0.5
+_KG_GROUNDING_LIMIT = 5
+
+
+def _question_confidence(
+    count: int,
+    examples: List[str],
+    recipe_id: Optional[str],
+    kg_related: Optional[int],
+) -> tuple:
+    """Deterministic confidence for a recurring-question suggestion.
+
+    Evidence factors: how often the question repeats, how many distinct
+    phrasings exist, whether it maps onto a known starter-recipe intent, and
+    (when the graph is available) how many Brain nodes relate to it.
+    """
+    score = 0.3 + 0.5 * min(1.0, (int(count) - 1) / 4)
+    score += min(0.15, 0.05 * len(examples or []))
+    if recipe_id:
+        score += 0.15
+    if kg_related:
+        score += min(0.2, 0.05 * int(kg_related))
+    factors = {
+        "repeat_count": int(count),
+        "distinct_examples": len(examples or []),
+        "intent_match": bool(recipe_id),
+        "kg_related_nodes": kg_related,
+    }
+    return round(min(1.0, score), 2), factors
+
+
+def _source_confidence(indexed: int, watch_enabled: bool) -> tuple:
+    """Deterministic confidence for a knowledge-source digest suggestion."""
+    score = 0.25 + 0.6 * min(1.0, int(indexed) / 25)
+    if watch_enabled:
+        score += 0.1
+    factors = {"indexed_files": int(indexed), "watch_enabled": bool(watch_enabled)}
+    return round(min(1.0, score), 2), factors
+
 _QUESTION_HINT_RE = re.compile(
     r"(\?|어때|뭐야|뭐가|뭘까|알려줘|보여줘|정리해|요약해|정리 좀|요약 좀|해줘"
     r"|what|how|why|when|where|status|summar|remind|list|show me|tell me)",
@@ -223,38 +275,101 @@ class AutomationIntelligenceService:
             LOGGER.exception("automation intelligence source read failed")
             return []
 
+    def _kg_related_count(
+        self, text: str, *, workspace_id: Optional[str]
+    ) -> Optional[int]:
+        """Count Brain nodes related to a recurring question (grounding evidence).
+
+        Returns ``None`` when the graph is unavailable so confidence scoring can
+        distinguish "grounding impossible" from "grounded with zero hits".
+        Scoping mirrors the conversation-history read: an explicit workspace is
+        strict, no workspace falls back to the unscoped/legacy view.
+        """
+        if not self._enable_graph or not hasattr(self._kg, "search"):
+            return None
+        try:
+            report = self._kg.search(
+                str(text or "")[:200],
+                limit=_KG_GROUNDING_LIMIT,
+                allowed_workspaces={workspace_id} if workspace_id is not None else None,
+                include_legacy_global=workspace_id is None,
+            )
+        except Exception:
+            LOGGER.exception("automation intelligence KG grounding failed")
+            return None
+        if not isinstance(report, dict):
+            return None
+        return len(report.get("matches") or [])
+
     # ── suggestions ──────────────────────────────────────────────────────
 
-    def _installed_suggestion_ids(self, *, workspace_id: Optional[str]) -> Dict[str, Dict[str, Any]]:
-        installed: Dict[str, Dict[str, Any]] = {}
+    def _installed_workflows(
+        self, *, workspace_id: Optional[str]
+    ) -> tuple:
+        """Map installed automation workflows by suggestion id and recipe id.
+
+        The recipe map lets a recurring-question suggestion that targets an
+        already-installed starter recipe surface as *installed* instead of
+        re-suggesting the same automation — the install API is idempotent on
+        exactly this provenance, so the read side must agree.
+        """
+        by_suggestion: Dict[str, Dict[str, Any]] = {}
+        by_recipe: Dict[str, Dict[str, Any]] = {}
         if self._store is None:
-            return installed
+            return by_suggestion, by_recipe
         try:
             workflows = self._store.list_workflows(workspace_id=workspace_id).get("workflows") or []
         except Exception:
             LOGGER.exception("automation intelligence workflow read failed")
-            return installed
+            return by_suggestion, by_recipe
         for workflow in workflows:
             metadata = (workflow or {}).get("metadata") or {}
-            suggestion_id = metadata.get("suggestion_id")
-            if metadata.get("created_from") == "automation_suggestion" and suggestion_id:
-                installed[str(suggestion_id)] = workflow
-        return installed
+            created_from = metadata.get("created_from")
+            if created_from == "automation_suggestion" and metadata.get("suggestion_id"):
+                by_suggestion[str(metadata["suggestion_id"])] = workflow
+            elif created_from == "brain_automation_recipe" and metadata.get("recipe_id"):
+                by_recipe[str(metadata["recipe_id"])] = workflow
+        return by_suggestion, by_recipe
 
     def suggestions(
         self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
     ) -> Dict[str, Any]:
         pattern_report = self.question_patterns(user_email=user_email, workspace_id=workspace_id)
-        installed = self._installed_suggestion_ids(workspace_id=workspace_id)
+        by_suggestion, by_recipe = self._installed_workflows(workspace_id=workspace_id)
 
         items: List[Dict[str, Any]] = []
+        suppressed_low_confidence = 0
+        suppressed_duplicates = 0
+        seen_recipe_ids: set = set()
+
+        # Patterns arrive sorted by (count, last_asked) desc, so the first
+        # suggestion per recipe is the strongest — later ones targeting the
+        # same recipe would install the identical workflow and are suppressed.
         for pattern in pattern_report["patterns"]:
+            recipe_id = pattern["recipe_id"]
+            if recipe_id and recipe_id in seen_recipe_ids:
+                suppressed_duplicates += 1
+                continue
+            kg_related = self._kg_related_count(
+                pattern["representative"], workspace_id=workspace_id
+            )
+            confidence, factors = _question_confidence(
+                pattern["count"], pattern["examples"], recipe_id, kg_related
+            )
+            if confidence < _MIN_SUGGESTION_CONFIDENCE:
+                suppressed_low_confidence += 1
+                continue
+            if recipe_id:
+                seen_recipe_ids.add(recipe_id)
             suggestion_id = _stable_id("sug-q", pattern["id"])
+            workflow = by_suggestion.get(suggestion_id) or (
+                by_recipe.get(str(recipe_id)) if recipe_id else None
+            )
             items.append({
                 "id": suggestion_id,
                 "kind": "recurring_question",
                 "intent": pattern["intent"],
-                "recipe_id": pattern["recipe_id"],
+                "recipe_id": recipe_id,
                 "title": pattern["representative"],
                 "reason": {
                     "type": "repeated_question",
@@ -263,8 +378,11 @@ class AutomationIntelligenceService:
                     "examples": pattern["examples"],
                 },
                 "cadence": "daily",
-                "installed": suggestion_id in installed,
-                "workflow_id": (installed.get(suggestion_id) or {}).get("id"),
+                "confidence": confidence,
+                "confidence_factors": factors,
+                "low_confidence": confidence < _LOW_CONFIDENCE_THRESHOLD,
+                "installed": workflow is not None,
+                "workflow_id": (workflow or {}).get("id"),
             })
 
         for source in self._knowledge_sources():
@@ -272,6 +390,14 @@ class AutomationIntelligenceService:
             indexed = sum(int(v or 0) for v in file_status.values())
             if indexed <= 0:
                 continue
+            confidence, factors = _source_confidence(
+                indexed, bool(source.get("watch_enabled"))
+            )
+            if confidence < _MIN_SUGGESTION_CONFIDENCE:
+                # A barely-indexed folder is not enough evidence for a digest
+                # automation yet — do not suggest.
+                suppressed_low_confidence += 1
+                continue
             suggestion_id = _stable_id("sug-src", str(source.get("id") or source.get("root_path") or ""))
             items.append({
                 "id": suggestion_id,
@@ -286,13 +412,22 @@ class AutomationIntelligenceService:
                     "watch_enabled": bool(source.get("watch_enabled")),
                 },
                 "cadence": "when new knowledge arrives",
-                "installed": suggestion_id in installed,
-                "workflow_id": (installed.get(suggestion_id) or {}).get("id"),
+                "confidence": confidence,
+                "confidence_factors": factors,
+                "low_confidence": confidence < _LOW_CONFIDENCE_THRESHOLD,
+                "installed": suggestion_id in by_suggestion,
+                "workflow_id": (by_suggestion.get(suggestion_id) or {}).get("id"),
             })
 
         return {
             "suggestions": items,
             "questions_scanned": pattern_report["questions_scanned"],
+            "quality": {
+                "min_confidence": _MIN_SUGGESTION_CONFIDENCE,
+                "low_confidence_threshold": _LOW_CONFIDENCE_THRESHOLD,
+                "suppressed_low_confidence": suppressed_low_confidence,
+                "suppressed_duplicates": suppressed_duplicates,
+            },
             "consent": {
                 "default_state": "draft_disabled",
                 "local_only": True,
@@ -403,6 +538,7 @@ class AutomationIntelligenceService:
                 "suggestion_kind": kind,
                 "suggestion_title": title,
                 "suggestion_reason": suggestion.get("reason"),
+                "suggestion_confidence": suggestion.get("confidence"),
                 "automation_state": "enabled" if enabled else "draft_disabled",
                 "local_only": True,
                 "external_actions": False,
@@ -442,6 +578,7 @@ class AutomationIntelligenceService:
             "suggestions": suggestion_report["suggestions"],
             "questions_scanned": suggestion_report["questions_scanned"],
             "installed": installed,
+            "quality": suggestion_report["quality"],
             "consent": suggestion_report["consent"],
             "generated_at": _now(),
         }

package/latticeai/services/brain_intelligence.py

@@ -257,6 +257,78 @@ class BrainIntelligenceService:
             "generated_at": _now(),
         }
 
+    # ── vector freshness (v9.8.0) ────────────────────────────────────────
+
+    def vector_freshness(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Fixed-contract vector index freshness for ``/api/brain/vector-freshness``.
+
+        Always returns ``{"status": "ready"|"pending"|"unavailable",
+        "pending_items": int, "total_items": int, "detail": str}`` and never
+        raises. The vector index is store-global (not workspace-partitioned);
+        scope arguments are accepted for router symmetry but do not narrow
+        the report.
+        """
+
+        def _unavailable(detail: str) -> Dict[str, Any]:
+            return {
+                "status": "unavailable",
+                "pending_items": 0,
+                "total_items": 0,
+                "detail": detail,
+            }
+
+        if not self._enable_graph or self._kg is None:
+            return _unavailable("knowledge graph is disabled; no vector index is configured")
+
+        freshness_fn = getattr(self._kg, "vector_freshness", None)
+        if callable(freshness_fn):
+            try:
+                raw = freshness_fn() or {}
+            except Exception as exc:
+                LOGGER.exception("vector freshness read failed")
+                return _unavailable(f"vector freshness read failed: {exc}")
+            status = str(raw.get("status") or "unavailable")
+            if status == "needs_reindex":
+                status = "pending"
+            if status not in {"ready", "pending", "unavailable"}:
+                status = "unavailable"
+            return {
+                "status": status,
+                "pending_items": int(raw.get("pending_items") or 0),
+                "total_items": int(raw.get("total_items") or 0),
+                "detail": str(raw.get("detail") or ""),
+            }
+
+        # Older/lighter stores: summarize index_status directly.
+        status_fn = getattr(self._kg, "index_status", None)
+        if callable(status_fn):
+            try:
+                raw = status_fn() or {}
+            except Exception as exc:
+                LOGGER.exception("vector index status read failed")
+                return _unavailable(f"vector index status unavailable: {exc}")
+            pending = int(raw.get("pending_items") or 0)
+            total = int(raw.get("source_items") or 0)
+            if pending > 0:
+                return {
+                    "status": "pending",
+                    "pending_items": pending,
+                    "total_items": total,
+                    "detail": (
+                        f"{pending} of {total} items are missing or stale in the vector index"
+                    ),
+                }
+            return {
+                "status": "ready",
+                "pending_items": 0,
+                "total_items": total,
+                "detail": "vector index is up to date",
+            }
+
+        return _unavailable("this knowledge store does not expose a vector index")
+
     # ── insights digest ──────────────────────────────────────────────────
 
     def insights(