ltcai 11.5.2 → 11.7.0

package/lattice_brain/graph/discovery_index/cleanup.py

@@ -1,182 +0,0 @@
-"""Removing a local file from the graph, and the scope checks around it.
-
-Deletes a file's graph node, sweeps the concepts left orphaned by it, and
-answers the two "is this row still good?" questions the scanner asks before
-skipping work. Moved verbatim out of ``discovery_index.py`` (v11.3.0).
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-# ruff: noqa: F403,F405
-from .._kg_common import *  # noqa: F403,F401
-
-# The cross-mixin surface (`_connect`, `_upsert_node`, …) is declared in
-# `_kg_contract.KnowledgeGraphCore`. It is a typing-only base: at runtime this
-# is `object`, so the MRO of `KnowledgeGraphStore` is unchanged.
-if TYPE_CHECKING:
-    from .._kg_contract import KnowledgeGraphCore as _Core
-else:
-    _Core = object
-
-
-class _LocalCleanupMixin(_Core):
-    """Graph deletion + orphan sweep. Composed into the public mixin."""
-
-    def _delete_local_file_graph(
-        self, conn: sqlite3.Connection, file_node_id: Optional[str]
-    ) -> None:
-        if not file_node_id:
-            return
-
-        file_row = conn.execute(
-            "SELECT metadata_json FROM nodes WHERE id=?",
-            (file_node_id,),
-        ).fetchone()
-        source_id = None
-        if file_row:
-            source_id = _safe_loads(file_row["metadata_json"]).get("source_id")
-
-        linked_rows = conn.execute(
-            """
-                SELECT n.id, n.type, n.metadata_json
-                FROM edges e
-                JOIN nodes n ON n.id=e.to_node
-                WHERE e.from_node=?
-                """,
-            (file_node_id,),
-        ).fetchall()
-        owned_ids: set = set()
-        auto_candidate_ids: set = set()
-        for row in linked_rows:
-            metadata = _safe_loads(row["metadata_json"])
-            if (
-                row["type"] in {"Chunk", "ImageText", "Section"}
-                or metadata.get("source_node") == file_node_id
-            ):
-                owned_ids.add(row["id"])
-            elif (
-                metadata.get("auto_extracted")
-                and metadata.get("source") == "local_folder"
-            ):
-                auto_candidate_ids.add(row["id"])
-
-        conn.execute("DELETE FROM chunks WHERE source_node=?", (file_node_id,))
-        conn.execute(
-            "DELETE FROM edges WHERE from_node=? OR to_node=?",
-            (file_node_id, file_node_id),
-        )
-        conn.execute("DELETE FROM nodes WHERE id=?", (file_node_id,))
-        self._v2_delete_nodes(conn, [file_node_id])
-
-        def delete_nodes(node_ids: set) -> None:
-            if not node_ids:
-                return
-            placeholders = ",".join("?" * len(node_ids))
-            params = list(node_ids)
-            conn.execute(
-                f"DELETE FROM chunks WHERE source_node IN ({placeholders})", params
-            )
-            conn.execute(
-                f"DELETE FROM edges WHERE from_node IN ({placeholders}) OR to_node IN ({placeholders})",
-                params * 2,
-            )
-            conn.execute(f"DELETE FROM nodes WHERE id IN ({placeholders})", params)
-            self._v2_delete_nodes(conn, params)
-
-        delete_nodes(owned_ids)
-
-        removable_auto_ids: set = set()
-        for node_id in auto_candidate_ids:
-            remaining_edges = conn.execute(
-                "SELECT from_node, to_node FROM edges WHERE from_node=? OR to_node=?",
-                (node_id, node_id),
-            ).fetchall()
-            if all(
-                (
-                    row["from_node"] in auto_candidate_ids
-                    and row["to_node"] in auto_candidate_ids
-                )
-                for row in remaining_edges
-            ):
-                removable_auto_ids.add(node_id)
-        delete_nodes(removable_auto_ids)
-        if source_id:
-            self._cleanup_local_graph_orphans(conn, str(source_id))
-
-    def _cleanup_local_graph_orphans(
-        self, conn: sqlite3.Connection, source_id: str
-    ) -> None:
-        while True:
-            folder_rows = conn.execute(
-                "SELECT id, metadata_json FROM nodes WHERE type='Folder'"
-            ).fetchall()
-            leaf_ids = []
-            for row in folder_rows:
-                metadata = _safe_loads(row["metadata_json"])
-                if metadata.get("source_id") != source_id:
-                    continue
-                has_children = conn.execute(
-                    "SELECT 1 FROM edges WHERE from_node=? LIMIT 1",
-                    (row["id"],),
-                ).fetchone()
-                if not has_children:
-                    leaf_ids.append(row["id"])
-            if not leaf_ids:
-                break
-            placeholders = ",".join("?" * len(leaf_ids))
-            conn.execute(
-                f"DELETE FROM edges WHERE from_node IN ({placeholders}) OR to_node IN ({placeholders})",
-                leaf_ids * 2,
-            )
-            conn.execute(f"DELETE FROM nodes WHERE id IN ({placeholders})", leaf_ids)
-            self._v2_delete_nodes(conn, leaf_ids)
-
-        for node_type in ("Drive", "Computer"):
-            rows = conn.execute(
-                "SELECT id FROM nodes WHERE type=?", (node_type,)
-            ).fetchall()
-            removable = []
-            for row in rows:
-                has_children = conn.execute(
-                    "SELECT 1 FROM edges WHERE from_node=? LIMIT 1",
-                    (row["id"],),
-                ).fetchone()
-                if not has_children:
-                    removable.append(row["id"])
-            if removable:
-                placeholders = ",".join("?" * len(removable))
-                conn.execute(
-                    f"DELETE FROM edges WHERE from_node IN ({placeholders}) OR to_node IN ({placeholders})",
-                    removable * 2,
-                )
-                conn.execute(
-                    f"DELETE FROM nodes WHERE id IN ({placeholders})", removable
-                )
-                self._v2_delete_nodes(conn, removable)
-
-    def _local_file_index_has_extracted_text(self, row: sqlite3.Row) -> bool:
-        metadata = _safe_loads(row["metadata_json"])
-        parser = metadata.get("parser") if isinstance(metadata, dict) else {}
-        if not isinstance(parser, dict):
-            return False
-        try:
-            return int(parser.get("extracted_chars") or 0) > 0
-        except (TypeError, ValueError):
-            return False
-
-    @staticmethod
-    def _node_matches_workspace(
-        conn: sqlite3.Connection,
-        node_id: Optional[str],
-        workspace_id: Optional[str],
-    ) -> bool:
-        """Return true only when the projected node has the expected scope."""
-        if not node_id:
-            return False
-        row = conn.execute(
-            "SELECT workspace_id FROM nodes_v2 WHERE id=?",
-            (node_id,),
-        ).fetchone()
-        return bool(row is not None and row["workspace_id"] == workspace_id)

package/lattice_brain/graph/discovery_index/extract.py

@@ -1,137 +0,0 @@
-"""Local file → text: the parsers behind local indexing.
-
-PDF/Word/Excel/PowerPoint text plus the image signal path (dimensions, OCR,
-and — only when a VLM exists — a caption). Moved verbatim out of
-``discovery_index.py`` (v11.3.0 decomposition).
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-# ruff: noqa: F403,F405
-from .._kg_common import *  # noqa: F403,F401
-
-# The cross-mixin surface (`_connect`, `_upsert_node`, …) is declared in
-# `_kg_contract.KnowledgeGraphCore`. It is a typing-only base: at runtime this
-# is `object`, so the MRO of `KnowledgeGraphStore` is unchanged.
-if TYPE_CHECKING:
-    from .._kg_contract import KnowledgeGraphCore as _Core
-else:
-    _Core = object
-
-
-class _LocalExtractMixin(_Core):
-    """File-text and image-signal extraction. Composed into the public mixin."""
-
-    def _extract_local_file_text(
-        self, path: Path, category: str, *, include_ocr: bool
-    ) -> Tuple[str, Dict[str, Any]]:
-        ext = path.suffix.lower()
-        meta: Dict[str, Any] = {"parser": _parser_type_for_category(category, ext)}
-        text = ""
-        if category in {"text", "code"} or ext == ".csv":
-            text = path.read_text(encoding="utf-8", errors="replace")
-        elif ext == ".pdf":
-            import pdfplumber
-
-            with pdfplumber.open(str(path)) as pdf:
-                meta["pages"] = len(pdf.pages)
-                text = "\n\n".join((page.extract_text() or "") for page in pdf.pages)
-        elif ext == ".docx":
-            from docx import Document
-
-            doc = Document(str(path))
-            paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
-            table_lines = []
-            for table in doc.tables:
-                for row in table.rows:
-                    cells = [_clean_text(cell.text) for cell in row.cells]
-                    if any(cells):
-                        table_lines.append("\t".join(cells))
-            meta["paragraphs"] = len(paragraphs)
-            meta["tables"] = len(doc.tables)
-            meta["table_rows"] = len(table_lines)
-            text = "\n\n".join([*paragraphs, *table_lines])
-        elif ext == ".xlsx":
-            from openpyxl import load_workbook
-
-            wb = load_workbook(str(path), read_only=True, data_only=True)
-            rows_all = []
-            non_empty_rows = 0
-            non_empty_cells = 0
-            char_count = 0
-            for ws in wb.worksheets:
-                sheet_rows = []
-                for row in ws.iter_rows(values_only=True):
-                    cells = [
-                        str(cell).strip() if cell is not None else "" for cell in row
-                    ]
-                    if not any(cells):
-                        continue
-                    line = "\t".join(cells)
-                    non_empty_rows += 1
-                    non_empty_cells += sum(1 for cell in cells if cell)
-                    sheet_rows.append(line)
-                    char_count += len(line) + 1
-                    if char_count > 200_000:
-                        break
-                if sheet_rows:
-                    rows_all.append(f"[Sheet: {ws.title}]")
-                    rows_all.extend(sheet_rows)
-                if char_count > 200_000:
-                    break
-            meta["sheets"] = len(wb.worksheets)
-            meta["rows"] = non_empty_rows
-            meta["cells"] = non_empty_cells
-            text = "\n".join(rows_all)
-        elif ext == ".pptx":
-            from pptx import Presentation
-
-            prs = Presentation(str(path))
-            slides_text = []
-            for index, slide in enumerate(prs.slides, 1):
-                parts = []
-                for shape in slide.shapes:
-                    if getattr(shape, "has_text_frame", False):
-                        slide_text = shape.text_frame.text.strip()
-                        if slide_text:
-                            parts.append(slide_text)
-                if parts:
-                    slides_text.append(f"[Slide {index}]\n" + "\n".join(parts))
-            meta["slides"] = len(prs.slides)
-            meta["text_slides"] = len(slides_text)
-            text = "\n\n".join(slides_text)
-        elif category == "image":
-            text = self._extract_image_signals(path, meta, include_ocr=include_ocr)
-        return text[:200_000], meta
-
-    def _extract_image_signals(
-        self, path: Path, meta: Dict[str, Any], *, include_ocr: bool
-    ) -> str:
-        """Dimensions, OCR text, and — only if a VLM exists — a caption.
-
-        Until v11.1.0 this path always attached a ``vision_caption`` built out
-        of the filename and the pixel dimensions (``Image pic.png (PNG 12x8)``)
-        and used it as the retrieval text. Nothing downstream could tell that
-        string apart from something a vision model had actually said about the
-        picture, so every screenshot in the graph carried a fake description.
-
-        Now the caption comes from the injected port and from nowhere else. A
-        picture with no OCR text and no model still gets indexed — under its
-        filename, which is a fact — and ``caption_status`` says why there is no
-        caption.
-        """
-        from ...multimodal import MultimodalPorts, extract_image_facts
-
-        ports = getattr(self, "multimodal_ports", None) or MultimodalPorts()
-        facts = extract_image_facts(str(path), ports=ports, ocr=include_ocr)
-        meta.update(facts.as_metadata())
-        meta["ocr_enabled"] = bool(include_ocr)
-        if facts.ocr_text:
-            meta["ocr_chars"] = len(facts.ocr_text)
-        if facts.ocr_status == "failed":
-            meta["ocr_error"] = facts.ocr_detail
-        if not facts.readable:
-            return ""
-        return facts.index_text() or path.name