ltcai 10.0.0 → 10.3.0

package/lattice_brain/conversations.py

@@ -19,8 +19,9 @@ import json
 import logging
 import sqlite3
 import threading
+from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterator, List, Optional
 
 
 def _message_hash(item: Dict[str, Any]) -> str:
@@ -40,11 +41,21 @@ class ConversationStore:
         self._lock = threading.RLock()
         self._init_db()
 
-    def _connect(self) -> sqlite3.Connection:
+    @contextmanager
+    def _connect(self) -> Iterator[sqlite3.Connection]:
+        """Transactional connection that is closed when the block exits.
+
+        ``with sqlite3.connect(...)`` commits but never closes; see
+        :meth:`lattice_brain.storage.base.StorageEngine.session`.
+        """
         conn = sqlite3.connect(str(self.db_path))
         conn.row_factory = sqlite3.Row
         conn.execute("PRAGMA journal_mode=WAL")
-        return conn
+        try:
+            with conn:
+                yield conn
+        finally:
+            conn.close()
 
     def _init_db(self) -> None:
         with self._lock, self._connect() as conn:

package/lattice_brain/embeddings.py

@@ -10,7 +10,6 @@ import struct
 from dataclasses import dataclass
 from typing import Any, Dict, Iterable, List, Optional
 
-
 DEFAULT_EMBEDDING_DIM = int(os.getenv("LATTICEAI_VECTOR_DIM", "384"))
 EMBEDDING_MODEL_ID = f"lattice-local-hash-v1:{DEFAULT_EMBEDDING_DIM}"
 
@@ -71,7 +70,18 @@ class LocalEmbeddingModel:
         return [value / norm for value in vector]
 
     def similarity(self, left: Iterable[float], right: Iterable[float]) -> float:
-        return float(sum(a * b for a, b in zip(left, right)))
+        # strict=True: a dimension mismatch means the two vectors came from
+        # different embedding models. Truncating to the shorter one produces a
+        # plausible-looking similarity that is meaningless — exactly the silent
+        # wrongness this codebase keeps finding. Callers that can hit a model
+        # swap already handle failure and fall back to lexical search.
+        left_v, right_v = list(left), list(right)
+        if len(left_v) != len(right_v):
+            raise ValueError(
+                f"embedding dimension mismatch: {len(left_v)} vs {len(right_v)}; "
+                "the vector index was built with a different model"
+            )
+        return float(sum(a * b for a, b in zip(left_v, right_v, strict=True)))
 
     def encode(self, vector: Iterable[float]) -> bytes:
         values = list(vector)

package/lattice_brain/graph/_kg_common.py

@@ -21,12 +21,13 @@ import sqlite3
 import time
 import zipfile
 from collections import Counter
+from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Tuple
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
 
 try:
-    from .schema import KGStoreV2, NodeType, EdgeType, _exec_script
+    from .schema import EdgeType, KGStoreV2, NodeType, _exec_script
 except Exception:  # pragma: no cover - v2 schema is optional at import time
     KGStoreV2 = None  # type: ignore[assignment]
     NodeType = None  # type: ignore[assignment]
@@ -44,6 +45,7 @@ _READ_FROM_V2_DEFAULT = os.getenv("LATTICEAI_KG_READ_V2", "1") != "0"
 # Static constants (projection/format versions, local-ingestion classification
 # tables, OS exclusion lists) live in ._kg_constants; re-exported here so every
 # existing ``from ._kg_common import <CONST>`` site is unaffected.
+from ..quiet import quiet
 from ._kg_constants import (  # noqa: E402
     _KG_DB_FORMAT_KEY,
     _KG_DB_FORMAT_VERSION,
@@ -67,7 +69,6 @@ from ._kg_constants import (  # noqa: E402
     WINDOWS_EXCLUDED_NAMES,
 )
 
-
 # Pure fs/path/hash/classification helpers → ._kg_fsutil (re-exported so the
 # computed __all__ below still forwards them to the graph mixins).
 from ._kg_fsutil import *  # noqa: E402,F401,F403
@@ -152,7 +153,7 @@ def chunk_strategy_for(filename: Any, *, content_type: str = "") -> str:
         if mime.startswith("text/html") or mime.startswith("text/plain"):
             return "prose"
     except Exception:
-        pass
+        quiet()
     return "plain"
 
 
@@ -736,7 +737,6 @@ _CONCEPT_STOP: set = {
     "yes",
     "not",
     "but",
-    "are",
     "all",
     "any",
     "out",

package/lattice_brain/graph/_kg_fsutil.py

@@ -11,7 +11,6 @@ them to the graph mixins), so existing call sites are unaffected.
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 import hashlib
 import math
 import os
@@ -19,10 +18,12 @@ import platform
 import re
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Tuple
 
+from ..quiet import quiet
+from ..utils import now_iso as _now
+from ..utils import parse_iso as _parse_iso
 from ._kg_constants import *  # noqa: F401,F403
-from ..utils import now_iso as _now, parse_iso as _parse_iso
 
 
 def _recency_score(
@@ -202,7 +203,7 @@ def _excluded_directory_reason(
             ):
                 return "user_library"
         except OSError:
-            pass
+            quiet()
         if _prefix_blocks(MACOS_EXCLUDED_PREFIXES):
             return "system_folder"
     if os_type == "linux":

package/lattice_brain/graph/discovery.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
-# ruff: noqa: F403,F405
+from ..quiet import quiet
 
+# ruff: noqa: F403,F405
 from ._kg_common import *  # noqa: F403,F401
 
 
@@ -59,7 +60,7 @@ class KnowledgeGraphDiscoveryMixin:
                     ):
                         add(volume.name, volume, "volume", recommended=False)
                 except OSError:
-                    pass
+                    quiet()
         elif os_type == "windows":
             for letter in "ABCDEFGHIJKLMNOPQRSTUVWXYZ":
                 drive = Path(f"{letter}:\\")
@@ -87,7 +88,7 @@ class KnowledgeGraphDiscoveryMixin:
                         ):
                             add(mounted.name, mounted, "volume", recommended=False)
                 except OSError:
-                    pass
+                    quiet()
 
         return {
             "os_type": os_type,

package/lattice_brain/graph/discovery_index.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 

package/lattice_brain/graph/documents.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
-# ruff: noqa: F403,F405
+from ..quiet import quiet
 
+# ruff: noqa: F403,F405
 from ._kg_common import *  # noqa: F403,F401
 
 
@@ -287,7 +288,7 @@ class KnowledgeGraphDocumentsMixin:
                                 }
                             )
                     except Exception:
-                        pass
+                        quiet()
                     result["images"].append(image_info)
         except Exception as exc:
             result["error"] = str(exc)

package/lattice_brain/graph/fusion.py

@@ -30,6 +30,8 @@ import os
 import re
 from typing import Any, Dict, Mapping, Optional
 
+from ..quiet import quiet
+
 QUERY_CLASSES = ("fact", "code", "person", "recency")
 
 FUSION_WEIGHTS_ENV = "LATTICEAI_FUSION_WEIGHTS"
@@ -118,6 +120,7 @@ def _env_overrides() -> Dict[str, Dict[str, float]]:
             try:
                 cleaned[key] = max(0.0, min(1.0, float(value)))
             except (TypeError, ValueError):
+                quiet()
                 continue
         if cleaned:
             overrides[str(cls)] = cleaned
@@ -137,6 +140,7 @@ def fusion_weight_table(
                         try:
                             table[cls][key] = max(0.0, min(1.0, float(value)))
                         except (TypeError, ValueError):
+                            quiet()
                             continue
     return table
 

package/lattice_brain/graph/ingest.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
-# ruff: noqa: F403,F405
+from ..quiet import quiet
+from ..sensitivity import stamp_sensitivity
 
+# ruff: noqa: F403,F405
 from ._kg_common import *  # noqa: F403,F401
 
 
@@ -36,7 +38,7 @@ def _triple_edge_metadata(triple: Dict[str, Any]) -> Dict[str, Any]:
         try:
             metadata["confidence"] = round(float(confidence), 4)
         except (TypeError, ValueError):
-            pass
+            quiet()
     return metadata
 
 
@@ -270,6 +272,14 @@ class KnowledgeGraphIngestMixin:
             "extracted": {k: v for k, v in (extracted or {}).items() if k != "content"},
             "structure": doc_meta,
         }
+
+        # Stamp never-leaves at ingestion. Before 10.2.0 the cloud filter looked
+        # for these flags and nothing in the product could set them, so the
+        # guard was unreachable. Deriving it from the path means a user who
+        # indexes a project folder does not have to remember that it contains a
+        # .env — the flag is on the node before any cloud turn can select it.
+        stamp_sensitivity(metadata, source_uri or str(path))
+
         full_text = f"{filename}\n{text}"
         concepts = _extract_concepts(full_text, limit=15)
         triples = _extract_triples(full_text, concepts)

package/lattice_brain/graph/projection.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
-# ruff: noqa: F403,F405
+from ..quiet import quiet
 
+# ruff: noqa: F403,F405
 from ._kg_common import *  # noqa: F403,F401
 
 # ── promotion review mode (review 2026-07-25 Wave 4) ─────────────────────────
@@ -364,7 +365,7 @@ class KnowledgeGraphProjectionMixin:
                 if EdgeType.from_legacy(leg_type).value == leg_type:
                     leg_type = ""
             except ValueError:
-                pass
+                quiet()
         norm_type = (
             EdgeType.from_legacy(edge_type).value if EdgeType is not None else edge_type
         )

package/lattice_brain/graph/provenance.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 
@@ -188,7 +187,8 @@ class KnowledgeGraphProvenanceMixin:
     def schema_versions(self) -> Dict[str, Any]:
         """Versions an exporter stamps and an importer validates against."""
         try:
-            from .schema import EMBED_DIM as _EMBED_DIM, KG_SCHEMA_V2_VERSION as _V2
+            from .schema import EMBED_DIM as _EMBED_DIM
+            from .schema import KG_SCHEMA_V2_VERSION as _V2
         except Exception:  # pragma: no cover - kg_schema always importable in practice
             _EMBED_DIM, _V2 = 1024, 2
         return {
@@ -421,7 +421,9 @@ class KnowledgeGraphProvenanceMixin:
         dest.parent.mkdir(parents=True, exist_ok=True)
         if dest.exists():
             dest.unlink()  # VACUUM INTO requires the target to not exist
-        conn = self._connect()
+        # Raw connection, not ``_connect()``: VACUUM cannot run inside a
+        # transaction, and ``_connect()`` wraps its block in one.
+        conn = self.storage_engine.connect()
         try:
             conn.execute("PRAGMA wal_checkpoint(FULL)")
             conn.execute("VACUUM INTO ?", (str(dest),))

package/lattice_brain/graph/rerank.py

@@ -111,7 +111,10 @@ def cross_encoder_rerank(
         return result
 
     ranked = list(candidates)
-    for item, score in zip(ranked, scores):
+    # strict=False on purpose: a cross-encoder that returns fewer scores than
+    # candidates leaves the tail un-reranked at its fused score, which is a
+    # better outcome than dropping the whole rerank.
+    for item, score in zip(ranked, scores, strict=False):
         item["rerank_score"] = float(score)
         # Surface the rerank score as the primary ranking key while keeping
         # the pre-rerank fused score under scores.fused for audit.

package/lattice_brain/graph/retrieval.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 # --- Compat seam (v9.9.5 decomposition) -------------------------------------

package/lattice_brain/graph/retrieval_docgen.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 

package/lattice_brain/graph/retrieval_reads.py

@@ -12,7 +12,6 @@ calls (e.g. ``search`` → ``filter_scoped_nodes``) behave exactly as before.
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 

package/lattice_brain/graph/retrieval_vector.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 

package/lattice_brain/graph/schema.py

@@ -43,13 +43,14 @@ print(store.stats())       # {"nodes": ..., "by_node_type": {...}, ...}
 from __future__ import annotations
 
 import json
-import os
 import logging
+import os
 import sqlite3
 from contextlib import contextmanager
 from enum import Enum
 from typing import Any, Dict, Optional
 
+from ..quiet import quiet
 
 # ── Schema version ──────────────────────────────────────────────────────────
 KG_SCHEMA_V2_VERSION = 2
@@ -129,7 +130,7 @@ class NodeType(str, Enum):
         try:
             return cls(m.upper())
         except ValueError:
-            pass
+            quiet()
         return _LEGACY_NODE_MAP.get(m.lower(), cls.CONCEPT)
 
 
@@ -189,7 +190,7 @@ class EdgeType(str, Enum):
         try:
             return cls(m.upper())
         except ValueError:
-            pass
+            quiet()
         return _LEGACY_EDGE_MAP.get(m.lower(), cls.MENTIONS)
 
 

package/lattice_brain/graph/store.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
-from .documents import KnowledgeGraphDocumentsMixin
 from .discovery import KnowledgeGraphDiscoveryMixin
 from .discovery_index import KnowledgeGraphLocalIndexMixin
+from .documents import KnowledgeGraphDocumentsMixin
 from .ingest import KnowledgeGraphIngestMixin
 from .projection import KnowledgeGraphProjectionMixin
 from .provenance import KnowledgeGraphProvenanceMixin
@@ -81,8 +80,23 @@ class KnowledgeGraphStore(
             return ("kgv2_nodes", "kgv2_edges")
         return ("nodes", "edges")
 
-    def _connect(self) -> sqlite3.Connection:
-        return self.storage_engine.connect()
+    @contextmanager
+    def _connect(self) -> Iterator[sqlite3.Connection]:
+        """Transactional connection that is closed when the block exits.
+
+        This used to return the raw connection, so ``with self._connect() as
+        conn`` inherited sqlite3's context-manager semantics: commit on success,
+        rollback on error, **never close**. Each of the ~58 call sites therefore
+        held a descriptor open until garbage collection reclaimed it. That is
+        invisible while CPython refcounting frees the local immediately, and
+        fatal as soon as anything keeps the frame alive — a coverage tracer, a
+        profiler, or a traceback captured by ``logger.exception``.
+
+        Use ``storage_engine.connect()`` directly only for statements that
+        cannot run inside a transaction, and close it yourself.
+        """
+        with self.storage_engine.session() as conn:
+            yield conn
 
     def _init_db(self) -> None:
         with self._connect() as conn:

package/lattice_brain/graph/write_master.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 # ruff: noqa: F403,F405
-
 from ._kg_common import *  # noqa: F403,F401
 
 
@@ -250,3 +249,49 @@ class KnowledgeGraphWriteMixin:
             text=text,
             metadata={**metadata, "parent_source_node": source_node},
         )
+
+    def set_node_sensitivity(
+        self,
+        node_id: str,
+        *,
+        local_only: bool,
+        reason: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Mark (or unmark) one node as never-leaving.
+
+        The cloud filter has always looked for this flag; until 10.2.0 nothing
+        could set it, so the guard was unreachable. This is the user-driven
+        half — ingestion stamps secret-bearing paths automatically, and this
+        covers everything a path cannot tell you, like a note whose *content*
+        is private.
+
+        Unmarking is allowed and audited by the caller: a user who flagged
+        something by mistake must be able to undo it, but the reason is cleared
+        with the flag so a stale justification cannot linger.
+        """
+        with self._connect() as conn:
+            row = conn.execute(
+                "SELECT metadata_json FROM nodes WHERE id=?", (node_id,)
+            ).fetchone()
+            if row is None:
+                return {"ok": False, "node_id": node_id, "reason": "node not found"}
+            try:
+                metadata = json.loads(row[0] or "{}")
+            except Exception:
+                metadata = {}
+            if local_only:
+                metadata["local_only"] = True
+                metadata["local_only_reason"] = reason or "marked by the user"
+            else:
+                metadata.pop("local_only", None)
+                metadata.pop("local_only_reason", None)
+            conn.execute(
+                "UPDATE nodes SET metadata_json=?, updated_at=? WHERE id=?",
+                (json.dumps(metadata, ensure_ascii=False), _now(), node_id),
+            )
+        return {
+            "ok": True,
+            "node_id": node_id,
+            "local_only": bool(local_only),
+            "reason": metadata.get("local_only_reason"),
+        }

package/lattice_brain/ingestion.py

@@ -360,6 +360,9 @@ from .ingestion_jobs import (  # noqa: E402,F401 — re-export for existing impo
     BackgroundIngestionJob,
     BackgroundIngestionQueue,
 )
+from .quiet import (  # noqa: E402 — imported after the module constants it depends on
+    quiet,  # noqa: E402 — imported after the module constants it depends on
+)
 
 
 @dataclass
@@ -577,7 +580,7 @@ class IngestionPipeline:
                     user_email,
                 )
             except Exception:  # noqa: BLE001 — audit must never break ingestion
-                pass
+                quiet()
 
         extraction_quality = self._assess_item_quality(
             item, source_type=source_type, text=quality_text, chunk_ids=chunk_ids,

package/lattice_brain/portability.py

@@ -20,10 +20,12 @@ import shutil
 import sqlite3
 import tempfile
 import zipfile
+from contextlib import closing
 from pathlib import Path, PurePosixPath
 from typing import Any, Dict, Optional
 
 from .archive import BrainArchivePaths, EncryptedBrainArchive
+from .quiet import quiet
 from .storage import (
     DockerPostgresWizard,
     PostgresEngine,
@@ -66,7 +68,7 @@ def _checkpoint_sqlite(db_path: Path) -> None:
     if not db_path.exists():
         return
     try:
-        with sqlite3.connect(str(db_path)) as conn:
+        with closing(sqlite3.connect(str(db_path))) as conn, conn:
             conn.execute("PRAGMA wal_checkpoint(FULL)")
     except sqlite3.Error:
         # Best-effort only. Existing sibling backup/restore still preserves
@@ -99,6 +101,7 @@ def _replace_sqlite_atomically(src: Path, dest: Path, backup_dir: Path) -> None:
             try:
                 shutil.copy2(sibling, backup)
             except FileNotFoundError:
+                quiet()
                 continue
             backups[sibling] = backup
         for sibling in _sqlite_siblings(dest)[1:]:
@@ -247,7 +250,7 @@ class KGPortabilityService:
                               "counts": (artifact.get("header") or {}).get("counts")},
                 )
             except Exception:
-                pass
+                quiet()
         return result
 
     def import_from_file(self, path, *, mode: str = "merge", dry_run: bool = False) -> Dict[str, Any]:

package/lattice_brain/quality.py

@@ -7,13 +7,15 @@ Pure Python Quality Layer for Lattice Brain (v6.4+ hardening)
 """
 
 from __future__ import annotations
+
+import hashlib
 import math
+import re
 import time
-import hashlib
+from collections import defaultdict
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional
-from collections import defaultdict
-import re
+
 
 # -----------------------------
 # 1. Embedding Fallback Labelling + Drift/Reindex Plan
@@ -41,7 +43,12 @@ class EmbeddingFallbackLabeller:
             conf = 0.3
         else:
             # Simple hash-based pseudo-label for determinism
-            h = hashlib.md5(str(embedding[:4]).encode()).hexdigest()[:8]
+            # Not a security hash: a short, stable label for grouping
+            # vectors in reports. usedforsecurity=False states that and
+            # keeps it working on FIPS builds where md5 is restricted.
+            h = hashlib.md5(
+                str(embedding[:4]).encode(), usedforsecurity=False
+            ).hexdigest()[:8]
             label = f"emb_cluster_{h}"
             conf = 0.85
 
@@ -63,7 +70,7 @@ class EmbeddingFallbackLabeller:
     def _cosine_distance(self, a: List[float], b: List[float]) -> float:
         if not a or not b or len(a) != len(b):
             return 1.0
-        dot = sum(x*y for x,y in zip(a,b))
+        dot = sum(x * y for x, y in zip(a, b, strict=True))
         na = math.sqrt(sum(x*x for x in a))
         nb = math.sqrt(sum(x*x for x in b))
         return 1.0 - (dot / (na*nb + 1e-9))

package/lattice_brain/quiet.py

@@ -0,0 +1,43 @@
+"""Deliberately-ignored exceptions, recorded instead of erased (Brain Core).
+
+Mirror of :mod:`latticeai.core.quiet`. Brain Core cannot import from the app
+package (``tests/unit/test_import_guard.py``), and the alternative — leaving
+Brain Core's suppressions silent while the app layer's are visible — would make
+the quieter half the harder one to debug.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import Optional
+
+logger = logging.getLogger("lattice_brain.suppressed")
+
+
+def quiet(reason: Optional[str] = None, *, level: int = logging.DEBUG) -> None:
+    """Record the exception currently being handled, then continue."""
+    exc_type, exc, tb = sys.exc_info()
+    if exc is None:
+        return
+    where = "<unknown>"
+    if tb is not None:
+        deepest = tb
+        while deepest.tb_next is not None:
+            deepest = deepest.tb_next
+        frame = deepest.tb_frame
+        where = f"{frame.f_code.co_filename}:{frame.f_lineno} in {frame.f_code.co_name}"
+    label = f" ({reason})" if reason else ""
+    if logger.isEnabledFor(level):
+        logger.log(
+            level,
+            "suppressed %s at %s%s: %s",
+            getattr(exc_type, "__name__", "Exception"),
+            where,
+            label,
+            exc,
+            exc_info=exc,
+        )
+
+
+__all__ = ["quiet"]