ltcai 9.9.4 → 9.9.9

package/lattice_brain/graph/retrieval_reads.py

@@ -0,0 +1,459 @@
+"""Non-search read surface of the knowledge graph (v9.9.5 decomposition).
+
+Byte-identical move-only split out of :mod:`lattice_brain.graph.retrieval`,
+which now keeps the search surface (``search`` / ``hybrid_search`` /
+``context_for_query``). This mixin owns the document listing, workspace
+scoping reads, node/neighbor/relationship lookups, graph traversal, and
+store statistics. Mixins share ``self`` on
+:class:`lattice_brain.graph.store.KnowledgeGraphStore`, so cross-mixin
+calls (e.g. ``search`` → ``filter_scoped_nodes``) behave exactly as before.
+"""
+
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphReadsMixin:
+    def list_documents(self, limit: int = 200) -> Dict[str, Any]:
+        """List ingested ``Document`` nodes with their ingest + index state.
+
+        Powers the Files view: every accepted upload and every indexed local
+        document becomes a ``Document`` node. A document is reported ``indexed``
+        once its retrieval chunks exist (searchable in Chat / Hybrid Search).
+        """
+        limit = max(1, min(int(limit or 200), 1000))
+        nt, _ = self._read_tables()
+        documents: List[Dict[str, Any]] = []
+        with self._connect() as conn:
+            rows = conn.execute(
+                f"SELECT id, title, summary, metadata_json, created_at, updated_at "
+                f"FROM {nt} WHERE type='Document' ORDER BY updated_at DESC, id ASC LIMIT ?",
+                (limit,),
+            ).fetchall()
+            for row in rows:
+                meta = _safe_loads(row["metadata_json"]) or {}
+                extracted = meta.get("extracted") or {}
+                node_id = row["id"]
+                chunk_count = conn.execute(
+                    "SELECT COUNT(*) AS c FROM chunks WHERE source_node=?",
+                    (node_id,),
+                ).fetchone()["c"]
+                if not chunk_count:
+                    # Legacy projections represented chunks as graph nodes and
+                    # linked them only through metadata_json. Keep read
+                    # compatibility without making the fragile LIKE path the
+                    # primary query.
+                    chunk_count = conn.execute(
+                        f"SELECT COUNT(*) AS c FROM {nt} WHERE type='Chunk' AND metadata_json LIKE ?",
+                        (f"%{node_id}%",),
+                    ).fetchone()["c"]
+                documents.append(
+                    {
+                        "id": node_id,
+                        "filename": meta.get("filename") or row["title"],
+                        "ext": meta.get("ext"),
+                        "mime_type": meta.get("mime_type"),
+                        "bytes": meta.get("bytes"),
+                        "sha256": meta.get("sha256"),
+                        "uploader": meta.get("uploader"),
+                        "chars": extracted.get("chars"),
+                        "chunks": int(chunk_count or 0),
+                        "indexed": int(chunk_count or 0) > 0,
+                        "ingest_state": "indexed"
+                        if int(chunk_count or 0) > 0
+                        else "ingested",
+                        "created_at": row["created_at"],
+                        "updated_at": row["updated_at"],
+                    }
+                )
+        return {
+            "documents": documents,
+            "total": len(documents),
+            "generated_at": datetime.now().isoformat(timespec="seconds"),
+        }
+
+    def workspaces_of(self, node_ids) -> Dict[str, Optional[str]]:
+        """Map known node ids to their workspace scope.
+
+        ``None`` is returned only for a row that is explicitly present in the
+        authoritative v2 projection with a NULL workspace.  Missing ids remain
+        missing, and projection/query failures propagate so callers can fail
+        closed instead of mistaking every candidate for legacy-global data.
+        """
+        ids = [str(i) for i in node_ids if i]
+        if not ids:
+            return {}
+        placeholders = ",".join("?" for _ in ids)
+        with self._connect() as conn:
+            return {
+                row["id"]: row["workspace_id"]
+                for row in conn.execute(
+                    f"SELECT id, workspace_id FROM nodes_v2 WHERE id IN ({placeholders})",
+                    ids,
+                ).fetchall()
+            }
+
+    def filter_scoped_nodes(
+        self,
+        items,
+        allowed_workspaces,
+        *,
+        id_key: str = "id",
+        include_legacy_global: bool = False,
+    ):
+        """Drop items scoped to a workspace the caller is not a member of.
+
+        ``allowed_workspaces=None`` means no scoping (single-user / no-auth
+        mode). In scoped/multi-user mode, unknown ids are private and
+        legacy-global rows require the explicit ``include_legacy_global=True``
+        compatibility opt-in.
+        """
+        candidates = list(items)
+        if allowed_workspaces is None:
+            return candidates
+        allowed = {str(workspace_id) for workspace_id in allowed_workspaces if workspace_id}
+        scopes = self.workspaces_of([item.get(id_key) for item in candidates])
+        visible = []
+        for item in candidates:
+            node_id = str(item.get(id_key) or "")
+            if not node_id or node_id not in scopes:
+                # Unknown/unprojected rows are never treated as public.
+                continue
+            workspace_id = scopes[node_id]
+            if workspace_id is None:
+                if include_legacy_global:
+                    visible.append(item)
+            elif str(workspace_id) in allowed:
+                visible.append(item)
+        return visible
+
+    def neighbors(
+        self,
+        node_id: str,
+        *,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        """Return direct neighbors (1-hop) of a node."""
+        if allowed_workspaces is not None and not self.filter_scoped_nodes(
+            [{"id": node_id}],
+            allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        ):
+            raise ValueError(f"graph node not found: {node_id}")
+        nt, et = self._read_tables()
+        with self._connect() as conn:
+            edge_rows = conn.execute(
+                f"SELECT from_node, to_node, type, weight FROM {et} WHERE from_node=? OR to_node=? ORDER BY id ASC",
+                (node_id, node_id),
+            ).fetchall()
+            neighbor_ids: set = set()
+            edges = []
+            for row in edge_rows:
+                neighbor_ids.add(row["from_node"])
+                neighbor_ids.add(row["to_node"])
+                edges.append(
+                    {
+                        "from": row["from_node"],
+                        "to": row["to_node"],
+                        "type": row["type"],
+                        "weight": row["weight"],
+                    }
+                )
+            neighbor_ids.discard(node_id)
+            nodes = []
+            if neighbor_ids:
+                placeholders = ",".join("?" * len(neighbor_ids))
+                nodes = [
+                    {
+                        "id": row["id"],
+                        "type": row["type"],
+                        "title": row["title"],
+                        "summary": row["summary"],
+                        "metadata": _safe_loads(row["metadata_json"]),
+                    }
+                    for row in conn.execute(
+                        f"SELECT id, type, title, summary, metadata_json FROM {nt} WHERE id IN ({placeholders}) ORDER BY id ASC",
+                        list(neighbor_ids),
+                    )
+                ]
+        if allowed_workspaces is not None:
+            nodes = self.filter_scoped_nodes(
+                nodes,
+                allowed_workspaces,
+                include_legacy_global=include_legacy_global,
+            )
+            kept = {node.get("id") for node in nodes}
+            edges = [
+                edge for edge in edges
+                if (edge.get("from") == node_id or edge.get("from") in kept)
+                and (edge.get("to") == node_id or edge.get("to") in kept)
+            ]
+        return {"node_id": node_id, "neighbors": nodes, "edges": edges}
+
+    def get_node(
+        self,
+        node_id: str,
+        *,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        node_id = str(node_id or "").strip()
+        if not node_id:
+            raise ValueError("node_id required")
+        nt, et = self._read_tables()
+        with self._connect() as conn:
+            row = conn.execute(
+                f"""
+                    SELECT id, type, title, summary, metadata_json, updated_at
+                    FROM {nt}
+                    WHERE id=?
+                    """,
+                (node_id,),
+            ).fetchone()
+            if not row:
+                raise ValueError(f"graph node not found: {node_id}")
+            degree = conn.execute(
+                f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
+                (node_id, node_id),
+            ).fetchone()["c"]
+        node = {
+            "id": row["id"],
+            "type": row["type"],
+            "title": row["title"],
+            "summary": row["summary"],
+            "metadata": _safe_loads(row["metadata_json"]),
+            "updated_at": row["updated_at"],
+            "degree": degree,
+        }
+        if allowed_workspaces is not None and not self.filter_scoped_nodes(
+            [node],
+            allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        ):
+            raise ValueError(f"graph node not found: {node_id}")
+        return node
+
+    def relationship_search(
+        self,
+        *,
+        query: str = "",
+        node_id: str = "",
+        relationship_type: str = "",
+        limit: int = 30,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        query = str(query or "").strip()
+        node_id = str(node_id or "").strip()
+        relationship_type = str(relationship_type or "").strip()
+        limit = max(1, min(int(limit or 30), 200))
+        nt, et = self._read_tables()
+        where = []
+        params: List[Any] = []
+        if node_id:
+            where.append("(e.from_node=? OR e.to_node=?)")
+            params.extend([node_id, node_id])
+        if relationship_type:
+            where.append("e.type LIKE ?")
+            params.append(f"%{relationship_type}%")
+        if query:
+            where.append(
+                "(e.type LIKE ? OR e.metadata_json LIKE ? OR src.title LIKE ? OR dst.title LIKE ? OR src.summary LIKE ? OR dst.summary LIKE ?)"
+            )
+            params.extend([f"%{query}%"] * 6)
+        where_sql = "WHERE " + " AND ".join(where) if where else ""
+        with self._connect() as conn:
+            rows = conn.execute(
+                f"""
+                    SELECT
+                      e.id, e.from_node, e.to_node, e.type, e.weight, e.metadata_json, e.created_at,
+                      src.type AS source_type, src.title AS source_title, src.summary AS source_summary,
+                      src.metadata_json AS source_metadata,
+                      dst.type AS target_type, dst.title AS target_title, dst.summary AS target_summary,
+                      dst.metadata_json AS target_metadata
+                    FROM {et} e
+                    JOIN {nt} src ON src.id=e.from_node
+                    JOIN {nt} dst ON dst.id=e.to_node
+                    {where_sql}
+                    ORDER BY e.weight DESC, e.created_at DESC, e.id ASC
+                    LIMIT ?
+                    """,
+                (*params, limit),
+            ).fetchall()
+        relationships = [
+                {
+                    "id": row["id"],
+                    "type": row["type"],
+                    "weight": row["weight"],
+                    "metadata": _safe_loads(row["metadata_json"]),
+                    "created_at": row["created_at"],
+                    "source": {
+                        "id": row["from_node"],
+                        "type": row["source_type"],
+                        "title": row["source_title"],
+                        "summary": row["source_summary"],
+                        "metadata": _safe_loads(row["source_metadata"]),
+                    },
+                    "target": {
+                        "id": row["to_node"],
+                        "type": row["target_type"],
+                        "title": row["target_title"],
+                        "summary": row["target_summary"],
+                        "metadata": _safe_loads(row["target_metadata"]),
+                    },
+                }
+                for row in rows
+            ]
+        if allowed_workspaces is not None:
+            kept = []
+            for rel in relationships:
+                endpoints = [
+                    {"id": (rel.get("source") or {}).get("id")},
+                    {"id": (rel.get("target") or {}).get("id")},
+                ]
+                if len(
+                    self.filter_scoped_nodes(
+                        endpoints,
+                        allowed_workspaces,
+                        include_legacy_global=include_legacy_global,
+                    )
+                ) == 2:
+                    kept.append(rel)
+            relationships = kept
+        return {
+            "query": query,
+            "node_id": node_id,
+            "relationship_type": relationship_type,
+            "relationships": relationships,
+        }
+
+    def traverse(
+        self,
+        node_id: str,
+        *,
+        depth: int = 1,
+        limit: int = 100,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        node_id = str(node_id or "").strip()
+        if not node_id:
+            raise ValueError("node_id required")
+        if allowed_workspaces is not None and not self.filter_scoped_nodes(
+            [{"id": node_id}],
+            allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        ):
+            raise ValueError(f"graph node not found: {node_id}")
+        depth = max(0, min(int(depth or 1), 4))
+        limit = max(1, min(int(limit or 100), 500))
+        nt, et = self._read_tables()
+        visited = {node_id}
+        frontier = {node_id}
+        edges_by_id: Dict[str, Dict[str, Any]] = {}
+        with self._connect() as conn:
+            for _ in range(depth):
+                if not frontier or len(visited) >= limit:
+                    break
+                placeholders = ",".join("?" * len(frontier))
+                rows = conn.execute(
+                    f"""
+                        SELECT id, from_node, to_node, type, weight, metadata_json
+                        FROM {et}
+                        WHERE from_node IN ({placeholders}) OR to_node IN ({placeholders})
+                        ORDER BY weight DESC, id ASC
+                        LIMIT ?
+                        """,
+                    (*frontier, *frontier, limit * 3),
+                ).fetchall()
+                next_frontier = set()
+                for row in rows:
+                    edges_by_id[row["id"]] = {
+                        "id": row["id"],
+                        "from": row["from_node"],
+                        "to": row["to_node"],
+                        "type": row["type"],
+                        "weight": row["weight"],
+                        "metadata": _safe_loads(row["metadata_json"]),
+                    }
+                    for candidate in (row["from_node"], row["to_node"]):
+                        if candidate not in visited and len(visited) < limit:
+                            visited.add(candidate)
+                            next_frontier.add(candidate)
+                frontier = next_frontier
+            placeholders = ",".join("?" * len(visited))
+            node_rows = conn.execute(
+                f"""
+                    SELECT id, type, title, summary, metadata_json, updated_at
+                    FROM {nt}
+                    WHERE id IN ({placeholders})
+                    ORDER BY updated_at DESC, id ASC
+                    """,
+                list(visited),
+            ).fetchall()
+        nodes = [
+                {
+                    "id": row["id"],
+                    "type": row["type"],
+                    "title": row["title"],
+                    "summary": row["summary"],
+                    "metadata": _safe_loads(row["metadata_json"]),
+                    "updated_at": row["updated_at"],
+                }
+                for row in node_rows
+            ]
+        edges = list(edges_by_id.values())
+        if allowed_workspaces is not None:
+            nodes = self.filter_scoped_nodes(
+                nodes,
+                allowed_workspaces,
+                include_legacy_global=include_legacy_global,
+            )
+            kept = {node.get("id") for node in nodes}
+            edges = [edge for edge in edges if edge.get("from") in kept and edge.get("to") in kept]
+        return {"root": node_id, "depth": depth, "nodes": nodes, "edges": edges}
+
+    def stats(self) -> Dict[str, Any]:
+        nt, et = self._read_tables()
+        with self._connect() as conn:
+            node_counts = {
+                row["type"]: row["count"]
+                for row in conn.execute(
+                    f"SELECT type, COUNT(*) AS count FROM {nt} GROUP BY type"
+                )
+            }
+            edge_counts = {
+                row["type"]: row["count"]
+                for row in conn.execute(
+                    f"SELECT type, COUNT(*) AS count FROM {et} GROUP BY type"
+                )
+            }
+            local_sources = conn.execute(
+                "SELECT COUNT(*) AS c FROM knowledge_sources"
+            ).fetchone()["c"]
+            local_file_status = {
+                row["status"]: row["count"]
+                for row in conn.execute(
+                    "SELECT status, COUNT(*) AS count FROM local_file_index GROUP BY status"
+                )
+            }
+        v2 = None
+        if KGStoreV2 is not None:
+            try:
+                v2 = KGStoreV2(self.db_path).stats()
+            except Exception as e:
+                v2 = {"available": False, "error": str(e)}
+        return {
+            "db_path": str(self.db_path),
+            "schema_version": GRAPH_SCHEMA_VERSION,
+            "v2_schema_available": KGStoreV2 is not None,
+            "nodes": node_counts,
+            "edges": edge_counts,
+            "local_sources": local_sources,
+            "local_file_status": local_file_status,
+            "v2": v2,
+        }

package/lattice_brain/graph/retrieval_vector.py

@@ -657,6 +657,7 @@ class KnowledgeGraphVectorMixin:
                       n.type AS node_type, n.title AS node_title, n.summary AS node_summary,
                       n.metadata_json AS node_metadata, n.updated_at AS node_updated_at,
                       c.text AS chunk_text, c.source_node AS parent_node_id,
+                      c.metadata_json AS chunk_metadata,
                       pn.type AS parent_type, pn.title AS parent_title,
                       pn.summary AS parent_summary, pn.metadata_json AS parent_metadata,
                       pn.updated_at AS parent_updated_at
@@ -690,6 +691,13 @@ class KnowledgeGraphVectorMixin:
             )
             parent_metadata = _safe_loads(row["parent_metadata"])
             node_metadata = _safe_loads(row["node_metadata"])
+            # Citation precision (review 2026-07-27 P1 #4): a chunk hit used to
+            # cite only its parent document, so a 200-page PDF answered with
+            # "from report.pdf". The chunk's own provenance (section heading,
+            # page, offset) now rides along, and `locator` is the one-line
+            # human form — absent when the chunk carries no such metadata.
+            chunk_metadata = _safe_loads(row["chunk_metadata"]) if is_chunk else {}
+            locator = citation_locator(chunk_metadata)
             scored.append(
                 {
                     "id": row["item_id"],
@@ -708,6 +716,8 @@ class KnowledgeGraphVectorMixin:
                         "vector": _safe_loads(row["vector_metadata"]),
                         "parent_node_id": row["parent_node_id"],
                         "parent_type": row["parent_type"],
+                        **({"chunk": chunk_metadata} if chunk_metadata else {}),
+                        **({"locator": locator} if locator else {}),
                     },
                     "updated_at": row["parent_updated_at"]
                     if is_chunk and row["parent_updated_at"]

package/lattice_brain/graph/store.py

@@ -11,6 +11,7 @@ from .projection import KnowledgeGraphProjectionMixin
 from .provenance import KnowledgeGraphProvenanceMixin
 from .retrieval import KnowledgeGraphRetrievalMixin
 from .retrieval_docgen import KnowledgeGraphDocGenMixin
+from .retrieval_reads import KnowledgeGraphReadsMixin
 from .retrieval_vector import KnowledgeGraphVectorMixin
 from .write_master import KnowledgeGraphWriteMixin
 
@@ -24,6 +25,7 @@ class KnowledgeGraphStore(
     KnowledgeGraphProvenanceMixin,
     KnowledgeGraphDocumentsMixin,
     KnowledgeGraphRetrievalMixin,
+    KnowledgeGraphReadsMixin,
     KnowledgeGraphVectorMixin,
     KnowledgeGraphDocGenMixin,
 ):

package/lattice_brain/ingestion.py

@@ -42,10 +42,9 @@ from __future__ import annotations
 import fnmatch
 import hashlib
 import os
-import threading
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Tuple
 
 from .runtime.hooks import dispatch_tool
 from .utils import utc_now_iso
@@ -353,110 +352,14 @@ def _matches_ignore(
     return False
 
 
-# --- Large candidate 1 slice: incremental / background ingestion support ---
-JOB_ERRORS_CAP = 50  # per-job error records kept (failed count keeps counting)
-
-
-@dataclass
-class BackgroundIngestionJob:
-    """Job descriptor + progress state for background/incremental indexing.
-
-    ``done_indices`` tracks per-item completion so an interrupted or partially
-    failed job can be *resumed* from the remaining items instead of restarting.
-    ``errors`` is capped at ``max_errors`` records; ``failed`` keeps counting.
-    """
-    job_id: str
-    items: List[IngestionItem]
-    status: str = "queued"  # queued | running | completed | failed | partial
-    created_at: str = field(default_factory=utc_now_iso)
-    updated_at: str = field(default_factory=utc_now_iso)
-    processed: int = 0
-    failed: int = 0
-    total: int = 0
-    errors: List[Dict[str, Any]] = field(default_factory=list)
-    incremental: bool = True
-    user_email: Optional[str] = None
-    max_errors: int = JOB_ERRORS_CAP
-    done_indices: Set[int] = field(default_factory=set)
-
-    def touch(self) -> None:
-        self.updated_at = utc_now_iso()
-
-    def record_error(self, index: int, item: IngestionItem, detail: Any) -> None:
-        self.failed += 1
-        if len(self.errors) < self.max_errors:
-            self.errors.append({
-                "index": index,
-                "source": item.source_uri or item.path or item.title or item.source_type,
-                "detail": str(detail)[:500],
-            })
-
-    def remaining_indices(self) -> List[int]:
-        return [i for i in range(len(self.items)) if i not in self.done_indices]
-
-    def as_dict(self) -> Dict[str, Any]:
-        """Frozen job schema consumed by ``/api/ingestion/jobs*``."""
-        return {
-            "job_id": self.job_id,
-            "status": self.status,
-            "total": self.total,
-            "processed": self.processed,
-            "failed": self.failed,
-            "errors": list(self.errors),
-            "created_at": self.created_at,
-            "updated_at": self.updated_at,
-        }
-
-
-class BackgroundIngestionQueue:
-    """Simple in-memory queue for background incremental ingestion.
-
-    For large corpus: this is the seam where a real scheduler / worker pool
-    (celery, rq, or internal thread) can be plugged later without changing callers.
-    Supports incremental (skip duplicates) vs force reindex.
-    """
-    def __init__(self) -> None:
-        self._jobs: Dict[str, BackgroundIngestionJob] = {}
-        self._counter = 0
-        self._lock = threading.Lock()
-
-    def schedule(
-        self,
-        items: List[IngestionItem],
-        *,
-        incremental: bool = True,
-        user_email: Optional[str] = None,
-    ) -> BackgroundIngestionJob:
-        with self._lock:
-            self._counter += 1
-            job_id = f"bg_ingest_{self._counter:04d}"
-        job = BackgroundIngestionJob(
-            job_id=job_id,
-            items=items,
-            total=len(items),
-            incremental=incremental,
-            user_email=user_email,
-        )
-        # annotate items for downstream
-        for it in job.items:
-            # attach flag without breaking dataclass defaults (use metadata)
-            it.metadata = {**it.metadata, "incremental": incremental, "bg_job": job_id}
-        with self._lock:
-            self._jobs[job_id] = job
-        return job
-
-    def get(self, job_id: str) -> Optional[BackgroundIngestionJob]:
-        return self._jobs.get(job_id)
-
-    def list_pending(self) -> List[BackgroundIngestionJob]:
-        return [j for j in self._jobs.values() if j.status == "queued"]
-
-    def list_recent(self, limit: int = 20) -> List[BackgroundIngestionJob]:
-        """Most recent jobs first (insertion order is schedule order)."""
-        limit = max(1, int(limit))
-        with self._lock:
-            jobs = list(self._jobs.values())
-        return list(reversed(jobs))[:limit]
+# Background job scheduling + progress lives in its own module (v9.9.6):
+# the pipeline owns "ingest one item", the queue owns "schedule many and
+# report progress". Re-exported so every existing import keeps working.
+from .ingestion_jobs import (  # noqa: E402,F401 — re-export for existing importers
+    JOB_ERRORS_CAP,
+    BackgroundIngestionJob,
+    BackgroundIngestionQueue,
+)
 
 
 @dataclass