ltcai 4.3.3 → 4.5.1

package/lattice_brain/graph/store.py

@@ -0,0 +1,237 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+from .documents import KnowledgeGraphDocumentsMixin
+from .discovery import KnowledgeGraphDiscoveryMixin
+from .ingest import KnowledgeGraphIngestMixin
+from .projection import KnowledgeGraphProjectionMixin
+from .provenance import KnowledgeGraphProvenanceMixin
+from .retrieval import KnowledgeGraphRetrievalMixin
+from .write_master import KnowledgeGraphWriteMixin
+
+
+class KnowledgeGraphStore(
+    KnowledgeGraphProjectionMixin,
+    KnowledgeGraphWriteMixin,
+    KnowledgeGraphDiscoveryMixin,
+    KnowledgeGraphIngestMixin,
+    KnowledgeGraphProvenanceMixin,
+    KnowledgeGraphDocumentsMixin,
+    KnowledgeGraphRetrievalMixin,
+):
+    def __init__(
+        self,
+        db_path: Path,
+        blob_dir: Path,
+        embedder: Any = None,
+        storage_engine: Any = None,
+    ):
+        self.db_path = Path(db_path)
+        self.blob_dir = Path(blob_dir)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.blob_dir.mkdir(parents=True, exist_ok=True)
+        if storage_engine is None:
+            from ..storage import SQLiteEngine
+
+            storage_engine = SQLiteEngine(self.db_path)
+        storage_caps = storage_engine.capabilities()
+        if not storage_caps.available:
+            raise RuntimeError(storage_caps.reason or "Brain storage is unavailable.")
+        if storage_caps.engine != "sqlite":
+            raise RuntimeError(
+                "KnowledgeGraphStore currently requires SQLiteEngine. "
+                "Explicit non-SQLite storage must use the migration/scale tooling; "
+                "no SQLite fallback is attempted."
+            )
+        self.storage_engine = storage_engine
+        # The embedder is swappable behind a fixed interface
+        # (model_id/dim/embed/encode/decode/similarity). Defaults to the
+        # deterministic, offline hash model so the store works with no config;
+        # server_app injects a provider-backed embedder from Config.
+        self._embedding_model = (
+            embedder if embedder is not None else LocalEmbeddingModel()
+        )
+        self._v2_projection_available = False
+        self._init_db()
+        # Read graph queries from the v2 projection (kgv2_* views) when available.
+        # Toggle off (e.g. in tests) to compare against the legacy tables.
+        self._read_from_v2 = (
+            KGStoreV2 is not None
+            and _READ_FROM_V2_DEFAULT
+            and self._v2_projection_available
+        )
+
+    def _read_tables(self) -> tuple:
+        """Return (nodes_table, edges_table) for read queries.
+
+        Same read code runs against the legacy tables or the v2 reconstruction
+        views, so the two paths are equivalent by construction.
+        """
+        if self._read_from_v2:
+            return ("kgv2_nodes", "kgv2_edges")
+        return ("nodes", "edges")
+
+    def _connect(self) -> sqlite3.Connection:
+        return self.storage_engine.connect()
+
+    def _init_db(self) -> None:
+        with self._connect() as conn:
+            db_format = int(conn.execute("PRAGMA user_version").fetchone()[0] or 0)
+            if db_format > _KG_DB_FORMAT_VERSION:
+                raise RuntimeError(
+                    f"Knowledge Graph DB format {db_format} is newer than this build "
+                    f"({_KG_DB_FORMAT_VERSION}); restore a pre-upgrade backup or upgrade Lattice AI."
+                )
+            conn.executescript(
+                """
+                    CREATE TABLE IF NOT EXISTS graph_meta (
+                      key TEXT PRIMARY KEY,
+                      value TEXT NOT NULL
+                    );
+                    CREATE TABLE IF NOT EXISTS nodes (
+                      id TEXT PRIMARY KEY,
+                      type TEXT NOT NULL,
+                      title TEXT NOT NULL,
+                      summary TEXT,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      raw_json TEXT NOT NULL CHECK (json_valid(raw_json)),
+                      created_at TEXT NOT NULL,
+                      updated_at TEXT NOT NULL
+                    );
+                    CREATE TABLE IF NOT EXISTS edges (
+                      id TEXT PRIMARY KEY,
+                      from_node TEXT NOT NULL,
+                      to_node TEXT NOT NULL,
+                      type TEXT NOT NULL,
+                      weight REAL NOT NULL DEFAULT 1.0,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      created_at TEXT NOT NULL,
+                      UNIQUE(from_node, to_node, type),
+                      FOREIGN KEY(from_node) REFERENCES nodes(id) ON DELETE CASCADE,
+                      FOREIGN KEY(to_node) REFERENCES nodes(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS chunks (
+                      id TEXT PRIMARY KEY,
+                      source_node TEXT NOT NULL,
+                      text TEXT NOT NULL,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      created_at TEXT NOT NULL,
+                      FOREIGN KEY(source_node) REFERENCES nodes(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS knowledge_sources (
+                      id TEXT PRIMARY KEY,
+                      root_path TEXT NOT NULL UNIQUE,
+                      os_type TEXT NOT NULL,
+                      drive_id TEXT,
+                      label TEXT,
+                      status TEXT NOT NULL,
+                      include_ocr INTEGER NOT NULL DEFAULT 0,
+                      watch_enabled INTEGER NOT NULL DEFAULT 0,
+                      consent_json TEXT NOT NULL CHECK (json_valid(consent_json)),
+                      created_at TEXT NOT NULL,
+                      updated_at TEXT NOT NULL,
+                      last_scanned_at TEXT
+                    );
+                    CREATE TABLE IF NOT EXISTS local_file_index (
+                      id TEXT PRIMARY KEY,
+                      source_id TEXT NOT NULL,
+                      os_type TEXT NOT NULL,
+                      drive_id TEXT,
+                      root_path TEXT NOT NULL,
+                      file_path TEXT NOT NULL,
+                      relative_path TEXT NOT NULL,
+                      file_name TEXT NOT NULL,
+                      extension TEXT NOT NULL,
+                      size_bytes INTEGER,
+                      modified_at TEXT,
+                      sha256 TEXT,
+                      last_scanned_at TEXT,
+                      last_indexed_at TEXT,
+                      parser_type TEXT,
+                      status TEXT NOT NULL,
+                      error_message TEXT,
+                      graph_node_id TEXT,
+                      deleted INTEGER NOT NULL DEFAULT 0,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      UNIQUE(source_id, relative_path),
+                      FOREIGN KEY(source_id) REFERENCES knowledge_sources(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS vector_embeddings (
+                      item_id TEXT PRIMARY KEY,
+                      item_type TEXT NOT NULL,
+                      source_node TEXT NOT NULL,
+                      text_hash TEXT NOT NULL,
+                      embedding BLOB NOT NULL,
+                      embedding_dim INTEGER NOT NULL,
+                      embedding_model TEXT NOT NULL,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json)),
+                      indexed_at TEXT NOT NULL,
+                      FOREIGN KEY(source_node) REFERENCES nodes(id) ON DELETE CASCADE
+                    );
+                    CREATE TABLE IF NOT EXISTS vector_index_operations (
+                      id TEXT PRIMARY KEY,
+                      operation TEXT NOT NULL,
+                      status TEXT NOT NULL,
+                      requested_at TEXT NOT NULL,
+                      started_at TEXT,
+                      completed_at TEXT,
+                      items_total INTEGER NOT NULL DEFAULT 0,
+                      items_indexed INTEGER NOT NULL DEFAULT 0,
+                      items_skipped INTEGER NOT NULL DEFAULT 0,
+                      error_message TEXT,
+                      metadata_json TEXT NOT NULL CHECK (json_valid(metadata_json))
+                    );
+                    -- v3.6.0 Knowledge Graph First: per-ingestion provenance trail.
+                    -- Append-only audit of where every graph node came from, when it
+                    -- was captured, how it was processed, and whether it was embedded /
+                    -- linked / used by an agent. get_provenance() returns the latest row.
+                    CREATE TABLE IF NOT EXISTS ingestion_provenance (
+                      id TEXT PRIMARY KEY,
+                      node_id TEXT NOT NULL,
+                      source_type TEXT NOT NULL,
+                      source_uri TEXT,
+                      content_hash TEXT,
+                      title TEXT,
+                      pipeline TEXT NOT NULL,
+                      owner TEXT,
+                      workspace_id TEXT,
+                      captured_at TEXT,
+                      modified_at TEXT,
+                      embedded INTEGER NOT NULL DEFAULT 0,
+                      linked INTEGER NOT NULL DEFAULT 0,
+                      duplicate INTEGER NOT NULL DEFAULT 0,
+                      agent_used TEXT,
+                      chunk_count INTEGER NOT NULL DEFAULT 0,
+                      permissions_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(permissions_json)),
+                      metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)),
+                      created_at TEXT NOT NULL
+                    );
+                    CREATE INDEX IF NOT EXISTS idx_nodes_type ON nodes(type);
+                    CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_node);
+                    CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(to_node);
+                    CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_node);
+                    CREATE INDEX IF NOT EXISTS idx_knowledge_sources_root ON knowledge_sources(root_path);
+                    CREATE INDEX IF NOT EXISTS idx_local_file_index_source ON local_file_index(source_id);
+                    CREATE INDEX IF NOT EXISTS idx_local_file_index_status ON local_file_index(status);
+                    CREATE INDEX IF NOT EXISTS idx_local_file_index_graph_node ON local_file_index(graph_node_id);
+                    CREATE INDEX IF NOT EXISTS idx_vector_embeddings_type ON vector_embeddings(item_type);
+                    CREATE INDEX IF NOT EXISTS idx_vector_embeddings_source ON vector_embeddings(source_node);
+                    CREATE INDEX IF NOT EXISTS idx_vector_embeddings_model ON vector_embeddings(embedding_model);
+                    CREATE INDEX IF NOT EXISTS idx_vector_index_operations_requested ON vector_index_operations(requested_at);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_node ON ingestion_provenance(node_id);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_source_type ON ingestion_provenance(source_type);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_hash ON ingestion_provenance(content_hash);
+                    CREATE INDEX IF NOT EXISTS idx_provenance_created ON ingestion_provenance(created_at);
+                    """
+            )
+            conn.execute(
+                "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+                ("schema_version", str(GRAPH_SCHEMA_VERSION)),
+            )
+        self._init_v2_schema()
+        self._init_fts()
+
+
+__all__ = ["KnowledgeGraphStore"]

package/lattice_brain/graph/write_master.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphWriteMixin:
+    def _upsert_node(
+        self,
+        conn: sqlite3.Connection,
+        node_id: str,
+        node_type: str,
+        title: str,
+        summary: str = "",
+        metadata: Optional[Dict[str, Any]] = None,
+        raw: Optional[Dict[str, Any]] = None,
+        owner: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        visibility: Optional[str] = None,
+    ) -> str:
+        now = _now()
+        # v4 write-mastering: nodes_v2 is authoritative; the legacy nodes
+        # table is maintained as the compatibility projection.
+        title_s = title[:240]
+        summary_s = summary[:1000]
+        meta_json = _json(metadata)
+        self._v2_project_node(
+            conn,
+            node_id,
+            node_type,
+            title_s,
+            summary_s,
+            meta_json,
+            created_at=now,
+            updated_at=now,
+            owner=owner,
+            workspace_id=workspace_id,
+            visibility=visibility,
+            strict=True,
+        )
+        conn.execute(
+            """
+                INSERT INTO nodes(id, type, title, summary, metadata_json, raw_json, created_at, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(id) DO UPDATE SET
+                  title=excluded.title,
+                  summary=excluded.summary,
+                  metadata_json=excluded.metadata_json,
+                  raw_json=excluded.raw_json,
+                  updated_at=excluded.updated_at
+                """,
+            (node_id, node_type, title_s, summary_s, meta_json, _json(raw), now, now),
+        )
+        if node_type != "Chunk":
+            self._upsert_vector_item(
+                conn,
+                item_id=node_id,
+                item_type="node",
+                source_node=node_id,
+                text=self._vector_text_for_node(
+                    title=title_s, summary=summary_s, metadata=metadata
+                ),
+                metadata={"node_type": node_type, **(metadata or {})},
+            )
+        return node_id
+
+    def _upsert_edge(
+        self,
+        conn: sqlite3.Connection,
+        from_node: str,
+        to_node: str,
+        edge_type: str,
+        weight: float = 1.0,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        # v4 write door: every new edge stores the canonical EdgeType value —
+        # free-string types (e.g. '포함함', '언급함') are normalized here, so no
+        # caller can mint new legacy taxonomy. The original label survives in
+        # metadata.legacy_label for traceability.
+        if EdgeType is not None:
+            canonical = EdgeType.from_legacy(edge_type).value
+            if canonical != edge_type:
+                metadata = dict(metadata or {})
+                metadata.setdefault("legacy_label", edge_type)
+            edge_type = canonical
+        edge_id = f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}')[:24]}"
+        now = _now()
+        meta_json = _json(metadata)  # canonical string shared with the projection
+        self._v2_project_edge(
+            conn,
+            from_node,
+            to_node,
+            edge_type,
+            float(weight),
+            meta_json,
+            edge_id=edge_id,
+            created_at=now,
+            strict=True,
+        )
+        conn.execute(
+            """
+                INSERT INTO edges(id, from_node, to_node, type, weight, metadata_json, created_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(from_node, to_node, type) DO UPDATE SET
+                  weight=max(edges.weight, excluded.weight),
+                  metadata_json=excluded.metadata_json
+                """,
+            (edge_id, from_node, to_node, edge_type, float(weight), meta_json, now),
+        )
+        return edge_id
+
+    def _vector_text_for_node(
+        self,
+        *,
+        title: str,
+        summary: str = "",
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> str:
+        metadata = metadata or {}
+        meta_parts = []
+        for key in (
+            "filename",
+            "relative_path",
+            "file_path",
+            "conversation_id",
+            "source",
+            "category",
+            "ext",
+            "role",
+        ):
+            value = metadata.get(key)
+            if value:
+                meta_parts.append(str(value))
+        return _clean_text(
+            "\n".join([str(title or ""), str(summary or ""), " ".join(meta_parts)])
+        )
+
+    def _upsert_vector_item(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        item_id: str,
+        item_type: str,
+        source_node: str,
+        text: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> bool:
+        text = _clean_text(text)
+        if len(text) < 2:
+            conn.execute("DELETE FROM vector_embeddings WHERE item_id=?", (item_id,))
+            return False
+        text_hash = _sha256_text(text)
+        existing = conn.execute(
+            """
+                SELECT text_hash, embedding_dim, embedding_model
+                FROM vector_embeddings
+                WHERE item_id=?
+                """,
+            (item_id,),
+        ).fetchone()
+        if (
+            existing
+            and existing["text_hash"] == text_hash
+            and existing["embedding_dim"] == self._embedding_model.dim
+            and existing["embedding_model"] == self._embedding_model.model_id
+        ):
+            return False
+        embedding = self._embedding_model.encode(
+            self._embedding_model.embed(text[:50_000])
+        )
+        conn.execute(
+            """
+                INSERT INTO vector_embeddings(
+                  item_id, item_type, source_node, text_hash, embedding,
+                  embedding_dim, embedding_model, metadata_json, indexed_at
+                )
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(item_id) DO UPDATE SET
+                  item_type=excluded.item_type,
+                  source_node=excluded.source_node,
+                  text_hash=excluded.text_hash,
+                  embedding=excluded.embedding,
+                  embedding_dim=excluded.embedding_dim,
+                  embedding_model=excluded.embedding_model,
+                  metadata_json=excluded.metadata_json,
+                  indexed_at=excluded.indexed_at
+                """,
+            (
+                item_id,
+                item_type,
+                source_node,
+                text_hash,
+                embedding,
+                self._embedding_model.dim,
+                self._embedding_model.model_id,
+                _json(metadata),
+                _now(),
+            ),
+        )
+        return True
+
+    def _upsert_chunk(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        chunk_id: str,
+        source_node: str,
+        text: str,
+        metadata: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        metadata = metadata or {}
+        conn.execute(
+            "INSERT OR REPLACE INTO chunks(id, source_node, text, metadata_json, created_at) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (chunk_id, source_node, text, _json(metadata), _now()),
+        )
+        self._upsert_vector_item(
+            conn,
+            item_id=chunk_id,
+            item_type="chunk",
+            source_node=chunk_id,
+            text=text,
+            metadata={**metadata, "parent_source_node": source_node},
+        )

package/lattice_brain/identity.py

@@ -1,13 +1,11 @@
-from latticeai.brain.identity import (
-    DeviceIdentity,
-    fingerprint_of,
-    verify_manifest,
-    verify_signature,
-)
-
-__all__ = [
-    "DeviceIdentity",
-    "fingerprint_of",
-    "verify_manifest",
-    "verify_signature",
-]
+"""Compatibility shim: implementation moved to lattice_brain.graph.identity.
+
+This module aliases itself to the physical module so identity, singletons,
+and monkeypatching behave as if the old flat path were the real module.
+"""
+
+import sys
+
+from .graph import identity as _impl
+
+sys.modules[__name__] = _impl

package/lattice_brain/ingest.py

@@ -1 +1,11 @@
-from latticeai.brain.ingest import *  # noqa: F401,F403
+"""Compatibility shim: implementation moved to lattice_brain.graph.ingest.
+
+This module aliases itself to the physical module so identity, singletons,
+and monkeypatching behave as if the old flat path were the real module.
+"""
+
+import sys
+
+from .graph import ingest as _impl
+
+sys.modules[__name__] = _impl