ltcai 9.9.3 → 9.9.5

package/lattice_brain/graph/retrieval_policy.py

@@ -0,0 +1,174 @@
+"""Single retrieval policy consulted by both hybrid fusion layers.
+
+Review 2026-07-25 (UX/harness/loop/KG-RAG), Wave 0.2 — "RetrievalPolicy
+단일화" + "query rewrite" + "recency age decay". Before this module the two
+hybrid entry points (``KnowledgeGraphStore.hybrid_search`` two-channel
+alpha fusion and ``SearchService.hybrid_search`` three-channel weight
+fusion) each resolved :func:`lattice_brain.graph.fusion.fusion_profile`
+independently and had no shared place to hang retrieval-wide behavior.
+This module is that single place:
+
+* :func:`rewrite_query` — deterministic, rule-based ko/en normalization
+  (NO LLM). Strips a conservative fixed list of leading/trailing politeness
+  fillers ("~좀 알려줘", "궁금해", "뭐였지", "what is", "please", …) and
+  collapses repeated whitespace. Code-class queries pass through untouched
+  (except whitespace collapse) because exact identifiers/filenames ARE the
+  retrieval signal. ``LATTICEAI_QUERY_REWRITE=0`` disables rewriting.
+* :func:`resolve_policy` — composes the rewrite with
+  :func:`~lattice_brain.graph.fusion.fusion_profile` (the weight table is
+  NOT duplicated here) and adds ``recency_half_life_days``, which is
+  non-``None`` ONLY for the ``recency`` query class — the honest signal
+  that age decay applies exactly where the fusion layers wire it.
+
+Both functions are pure, deterministic, and never raise; the ``fact``
+query class resolves to the historical default weights, so fact-class
+behavior stays byte-compatible. Scores are never combined with ``or``
+(score ``0.0`` is falsy but valid).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Dict, List, Mapping, Optional
+
+from .fusion import classify_query, fusion_profile
+
+QUERY_REWRITE_ENV = "LATTICEAI_QUERY_REWRITE"
+
+# Half-life (days) for the recency-class age decay. Matches the default of
+# lattice_brain.graph._kg_fsutil._recency_score, which the fusion layers use
+# to compute the actual multiplier — this module only declares the policy.
+RECENCY_HALF_LIFE_DAYS = 14.0
+
+# A filler strip is applied only when the remaining query keeps at least this
+# many characters — never rewrite a query down to (near-)nothing.
+_MIN_REMAINDER_CHARS = 4
+
+_WS_RE = re.compile(r"\s+")
+
+# Conservative trailing Korean politeness/filler phrases ("… 좀 알려줘",
+# "… 궁금해", "… 뭐였지"). Anchored at end-of-string, must start the query or
+# follow whitespace so word interiors are never clipped.
+_KO_FILLER_TAIL_RE = re.compile(
+    r"(?:^|(?<=\s))(?:좀\s+)?"
+    r"(?:알려\s*줘요?|알려\s*주세요|알려\s*줄래요?"
+    r"|말해\s*줘요?|말해\s*주세요"
+    r"|설명해\s*줘요?|설명해\s*주세요"
+    r"|궁금해요?|궁금합니다"
+    r"|뭐였지|뭐였더라|뭐더라|뭐지|뭐야"
+    r")\s*[?!.…~]*\s*$"
+)
+
+# Conservative leading English filler ("what is …", "tell me about …",
+# optionally prefixed with "please").
+_EN_FILLER_LEAD_RE = re.compile(
+    r"^(?:please\s+)?(?:tell\s+me\s+about|what\s+is|what\s+was|what\s+are|what's)\s+",
+    re.IGNORECASE,
+)
+
+# Conservative trailing English politeness ("…, please?").
+_EN_FILLER_TAIL_RE = re.compile(r"[,\s]*\bplease\b\s*[?!.]*\s*$", re.IGNORECASE)
+
+_FILLER_RULES = (
+    ("strip_filler_ko", _KO_FILLER_TAIL_RE),
+    ("strip_filler_en_leading", _EN_FILLER_LEAD_RE),
+    ("strip_filler_en_trailing", _EN_FILLER_TAIL_RE),
+)
+
+
+def _rewrite_enabled() -> bool:
+    raw = os.getenv(QUERY_REWRITE_ENV, "").strip().lower()
+    return raw not in {"0", "false", "no", "off"}
+
+
+def rewrite_query(query: Any) -> Dict[str, Any]:
+    """Deterministic rule-based ko/en query normalization (never raises).
+
+    Returns ``{"original": str, "rewritten": str, "rules": [rule names]}``.
+    Only three conservative rules exist: whitespace collapse, trailing
+    Korean filler strip, and leading/trailing English filler strip — each
+    filler strip applies only when the remainder keeps >= 4 characters.
+    Code-class queries (exact identifiers/filenames are the signal) are
+    returned unchanged apart from whitespace collapse. The
+    ``LATTICEAI_QUERY_REWRITE=0`` kill-switch disables rewriting entirely
+    (``rewritten == original``). Empty input yields empty output.
+    """
+    try:
+        text = str(query or "").strip()
+    except Exception:  # noqa: BLE001 — rewrite must never raise
+        return {"original": "", "rewritten": "", "rules": []}
+    if not text:
+        return {"original": "", "rewritten": "", "rules": []}
+    original = text
+    if not _rewrite_enabled():
+        return {"original": original, "rewritten": original, "rules": []}
+
+    rules: List[str] = []
+    rewritten = _WS_RE.sub(" ", text).strip()
+    if rewritten != original:
+        rules.append("collapse_whitespace")
+
+    if classify_query(original) == "code":
+        # Exact identifiers/filenames are the retrieval signal — never touch
+        # them (whitespace collapse is the only permitted normalization).
+        return {"original": original, "rewritten": rewritten, "rules": rules}
+
+    for name, pattern in _FILLER_RULES:
+        candidate = pattern.sub("", rewritten).strip()
+        if candidate != rewritten and len(candidate) >= _MIN_REMAINDER_CHARS:
+            rewritten = candidate
+            rules.append(name)
+    return {"original": original, "rewritten": rewritten, "rules": rules}
+
+
+def resolve_policy(
+    query: Any,
+    *,
+    overrides: Optional[Mapping[str, Mapping[str, float]]] = None,
+) -> Dict[str, Any]:
+    """Resolve the full retrieval policy for ``query`` (never raises).
+
+    Composes :func:`rewrite_query` with
+    :func:`~lattice_brain.graph.fusion.fusion_profile` (classification runs
+    on the ORIGINAL query, so ``query_class`` / ``weights`` / ``alpha`` are
+    byte-identical to calling ``fusion_profile`` directly, as both hybrid
+    layers did before this module). Returns::
+
+        {
+          "query_class": "fact" | "code" | "person" | "recency",
+          "weights": {"keyword", "vector", "graph"},   # service fusion
+          "alpha": float,                              # graph-layer fusion
+          "original_query": str,
+          "search_query": str,       # the rewritten form to search with
+          "rewrite_rules": [str],
+          "recency_half_life_days": float | None,      # 14.0 only for recency
+        }
+
+    ``recency_half_life_days`` is non-``None`` only for the ``recency``
+    class — the honest contract that age decay applies exactly where the
+    fusion layers wire it, and nowhere else.
+    """
+    rewrite = rewrite_query(query)
+    profile = fusion_profile(rewrite["original"], overrides=overrides)
+    query_class = profile["query_class"]
+    search_query = rewrite["rewritten"] if rewrite["rewritten"] else rewrite["original"]
+    return {
+        "query_class": query_class,
+        "weights": dict(profile["weights"]),
+        "alpha": float(profile["alpha"]),
+        "original_query": rewrite["original"],
+        "search_query": search_query,
+        "rewrite_rules": list(rewrite["rules"]),
+        "recency_half_life_days": (
+            RECENCY_HALF_LIFE_DAYS if query_class == "recency" else None
+        ),
+    }
+
+
+__all__ = [
+    "QUERY_REWRITE_ENV",
+    "RECENCY_HALF_LIFE_DAYS",
+    "resolve_policy",
+    "rewrite_query",
+]

package/lattice_brain/graph/retrieval_reads.py

@@ -0,0 +1,459 @@
+"""Non-search read surface of the knowledge graph (v9.9.5 decomposition).
+
+Byte-identical move-only split out of :mod:`lattice_brain.graph.retrieval`,
+which now keeps the search surface (``search`` / ``hybrid_search`` /
+``context_for_query``). This mixin owns the document listing, workspace
+scoping reads, node/neighbor/relationship lookups, graph traversal, and
+store statistics. Mixins share ``self`` on
+:class:`lattice_brain.graph.store.KnowledgeGraphStore`, so cross-mixin
+calls (e.g. ``search`` → ``filter_scoped_nodes``) behave exactly as before.
+"""
+
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphReadsMixin:
+    def list_documents(self, limit: int = 200) -> Dict[str, Any]:
+        """List ingested ``Document`` nodes with their ingest + index state.
+
+        Powers the Files view: every accepted upload and every indexed local
+        document becomes a ``Document`` node. A document is reported ``indexed``
+        once its retrieval chunks exist (searchable in Chat / Hybrid Search).
+        """
+        limit = max(1, min(int(limit or 200), 1000))
+        nt, _ = self._read_tables()
+        documents: List[Dict[str, Any]] = []
+        with self._connect() as conn:
+            rows = conn.execute(
+                f"SELECT id, title, summary, metadata_json, created_at, updated_at "
+                f"FROM {nt} WHERE type='Document' ORDER BY updated_at DESC, id ASC LIMIT ?",
+                (limit,),
+            ).fetchall()
+            for row in rows:
+                meta = _safe_loads(row["metadata_json"]) or {}
+                extracted = meta.get("extracted") or {}
+                node_id = row["id"]
+                chunk_count = conn.execute(
+                    "SELECT COUNT(*) AS c FROM chunks WHERE source_node=?",
+                    (node_id,),
+                ).fetchone()["c"]
+                if not chunk_count:
+                    # Legacy projections represented chunks as graph nodes and
+                    # linked them only through metadata_json. Keep read
+                    # compatibility without making the fragile LIKE path the
+                    # primary query.
+                    chunk_count = conn.execute(
+                        f"SELECT COUNT(*) AS c FROM {nt} WHERE type='Chunk' AND metadata_json LIKE ?",
+                        (f"%{node_id}%",),
+                    ).fetchone()["c"]
+                documents.append(
+                    {
+                        "id": node_id,
+                        "filename": meta.get("filename") or row["title"],
+                        "ext": meta.get("ext"),
+                        "mime_type": meta.get("mime_type"),
+                        "bytes": meta.get("bytes"),
+                        "sha256": meta.get("sha256"),
+                        "uploader": meta.get("uploader"),
+                        "chars": extracted.get("chars"),
+                        "chunks": int(chunk_count or 0),
+                        "indexed": int(chunk_count or 0) > 0,
+                        "ingest_state": "indexed"
+                        if int(chunk_count or 0) > 0
+                        else "ingested",
+                        "created_at": row["created_at"],
+                        "updated_at": row["updated_at"],
+                    }
+                )
+        return {
+            "documents": documents,
+            "total": len(documents),
+            "generated_at": datetime.now().isoformat(timespec="seconds"),
+        }
+
+    def workspaces_of(self, node_ids) -> Dict[str, Optional[str]]:
+        """Map known node ids to their workspace scope.
+
+        ``None`` is returned only for a row that is explicitly present in the
+        authoritative v2 projection with a NULL workspace.  Missing ids remain
+        missing, and projection/query failures propagate so callers can fail
+        closed instead of mistaking every candidate for legacy-global data.
+        """
+        ids = [str(i) for i in node_ids if i]
+        if not ids:
+            return {}
+        placeholders = ",".join("?" for _ in ids)
+        with self._connect() as conn:
+            return {
+                row["id"]: row["workspace_id"]
+                for row in conn.execute(
+                    f"SELECT id, workspace_id FROM nodes_v2 WHERE id IN ({placeholders})",
+                    ids,
+                ).fetchall()
+            }
+
+    def filter_scoped_nodes(
+        self,
+        items,
+        allowed_workspaces,
+        *,
+        id_key: str = "id",
+        include_legacy_global: bool = False,
+    ):
+        """Drop items scoped to a workspace the caller is not a member of.
+
+        ``allowed_workspaces=None`` means no scoping (single-user / no-auth
+        mode). In scoped/multi-user mode, unknown ids are private and
+        legacy-global rows require the explicit ``include_legacy_global=True``
+        compatibility opt-in.
+        """
+        candidates = list(items)
+        if allowed_workspaces is None:
+            return candidates
+        allowed = {str(workspace_id) for workspace_id in allowed_workspaces if workspace_id}
+        scopes = self.workspaces_of([item.get(id_key) for item in candidates])
+        visible = []
+        for item in candidates:
+            node_id = str(item.get(id_key) or "")
+            if not node_id or node_id not in scopes:
+                # Unknown/unprojected rows are never treated as public.
+                continue
+            workspace_id = scopes[node_id]
+            if workspace_id is None:
+                if include_legacy_global:
+                    visible.append(item)
+            elif str(workspace_id) in allowed:
+                visible.append(item)
+        return visible
+
+    def neighbors(
+        self,
+        node_id: str,
+        *,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        """Return direct neighbors (1-hop) of a node."""
+        if allowed_workspaces is not None and not self.filter_scoped_nodes(
+            [{"id": node_id}],
+            allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        ):
+            raise ValueError(f"graph node not found: {node_id}")
+        nt, et = self._read_tables()
+        with self._connect() as conn:
+            edge_rows = conn.execute(
+                f"SELECT from_node, to_node, type, weight FROM {et} WHERE from_node=? OR to_node=? ORDER BY id ASC",
+                (node_id, node_id),
+            ).fetchall()
+            neighbor_ids: set = set()
+            edges = []
+            for row in edge_rows:
+                neighbor_ids.add(row["from_node"])
+                neighbor_ids.add(row["to_node"])
+                edges.append(
+                    {
+                        "from": row["from_node"],
+                        "to": row["to_node"],
+                        "type": row["type"],
+                        "weight": row["weight"],
+                    }
+                )
+            neighbor_ids.discard(node_id)
+            nodes = []
+            if neighbor_ids:
+                placeholders = ",".join("?" * len(neighbor_ids))
+                nodes = [
+                    {
+                        "id": row["id"],
+                        "type": row["type"],
+                        "title": row["title"],
+                        "summary": row["summary"],
+                        "metadata": _safe_loads(row["metadata_json"]),
+                    }
+                    for row in conn.execute(
+                        f"SELECT id, type, title, summary, metadata_json FROM {nt} WHERE id IN ({placeholders}) ORDER BY id ASC",
+                        list(neighbor_ids),
+                    )
+                ]
+        if allowed_workspaces is not None:
+            nodes = self.filter_scoped_nodes(
+                nodes,
+                allowed_workspaces,
+                include_legacy_global=include_legacy_global,
+            )
+            kept = {node.get("id") for node in nodes}
+            edges = [
+                edge for edge in edges
+                if (edge.get("from") == node_id or edge.get("from") in kept)
+                and (edge.get("to") == node_id or edge.get("to") in kept)
+            ]
+        return {"node_id": node_id, "neighbors": nodes, "edges": edges}
+
+    def get_node(
+        self,
+        node_id: str,
+        *,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        node_id = str(node_id or "").strip()
+        if not node_id:
+            raise ValueError("node_id required")
+        nt, et = self._read_tables()
+        with self._connect() as conn:
+            row = conn.execute(
+                f"""
+                    SELECT id, type, title, summary, metadata_json, updated_at
+                    FROM {nt}
+                    WHERE id=?
+                    """,
+                (node_id,),
+            ).fetchone()
+            if not row:
+                raise ValueError(f"graph node not found: {node_id}")
+            degree = conn.execute(
+                f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
+                (node_id, node_id),
+            ).fetchone()["c"]
+        node = {
+            "id": row["id"],
+            "type": row["type"],
+            "title": row["title"],
+            "summary": row["summary"],
+            "metadata": _safe_loads(row["metadata_json"]),
+            "updated_at": row["updated_at"],
+            "degree": degree,
+        }
+        if allowed_workspaces is not None and not self.filter_scoped_nodes(
+            [node],
+            allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        ):
+            raise ValueError(f"graph node not found: {node_id}")
+        return node
+
+    def relationship_search(
+        self,
+        *,
+        query: str = "",
+        node_id: str = "",
+        relationship_type: str = "",
+        limit: int = 30,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        query = str(query or "").strip()
+        node_id = str(node_id or "").strip()
+        relationship_type = str(relationship_type or "").strip()
+        limit = max(1, min(int(limit or 30), 200))
+        nt, et = self._read_tables()
+        where = []
+        params: List[Any] = []
+        if node_id:
+            where.append("(e.from_node=? OR e.to_node=?)")
+            params.extend([node_id, node_id])
+        if relationship_type:
+            where.append("e.type LIKE ?")
+            params.append(f"%{relationship_type}%")
+        if query:
+            where.append(
+                "(e.type LIKE ? OR e.metadata_json LIKE ? OR src.title LIKE ? OR dst.title LIKE ? OR src.summary LIKE ? OR dst.summary LIKE ?)"
+            )
+            params.extend([f"%{query}%"] * 6)
+        where_sql = "WHERE " + " AND ".join(where) if where else ""
+        with self._connect() as conn:
+            rows = conn.execute(
+                f"""
+                    SELECT
+                      e.id, e.from_node, e.to_node, e.type, e.weight, e.metadata_json, e.created_at,
+                      src.type AS source_type, src.title AS source_title, src.summary AS source_summary,
+                      src.metadata_json AS source_metadata,
+                      dst.type AS target_type, dst.title AS target_title, dst.summary AS target_summary,
+                      dst.metadata_json AS target_metadata
+                    FROM {et} e
+                    JOIN {nt} src ON src.id=e.from_node
+                    JOIN {nt} dst ON dst.id=e.to_node
+                    {where_sql}
+                    ORDER BY e.weight DESC, e.created_at DESC, e.id ASC
+                    LIMIT ?
+                    """,
+                (*params, limit),
+            ).fetchall()
+        relationships = [
+                {
+                    "id": row["id"],
+                    "type": row["type"],
+                    "weight": row["weight"],
+                    "metadata": _safe_loads(row["metadata_json"]),
+                    "created_at": row["created_at"],
+                    "source": {
+                        "id": row["from_node"],
+                        "type": row["source_type"],
+                        "title": row["source_title"],
+                        "summary": row["source_summary"],
+                        "metadata": _safe_loads(row["source_metadata"]),
+                    },
+                    "target": {
+                        "id": row["to_node"],
+                        "type": row["target_type"],
+                        "title": row["target_title"],
+                        "summary": row["target_summary"],
+                        "metadata": _safe_loads(row["target_metadata"]),
+                    },
+                }
+                for row in rows
+            ]
+        if allowed_workspaces is not None:
+            kept = []
+            for rel in relationships:
+                endpoints = [
+                    {"id": (rel.get("source") or {}).get("id")},
+                    {"id": (rel.get("target") or {}).get("id")},
+                ]
+                if len(
+                    self.filter_scoped_nodes(
+                        endpoints,
+                        allowed_workspaces,
+                        include_legacy_global=include_legacy_global,
+                    )
+                ) == 2:
+                    kept.append(rel)
+            relationships = kept
+        return {
+            "query": query,
+            "node_id": node_id,
+            "relationship_type": relationship_type,
+            "relationships": relationships,
+        }
+
+    def traverse(
+        self,
+        node_id: str,
+        *,
+        depth: int = 1,
+        limit: int = 100,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        node_id = str(node_id or "").strip()
+        if not node_id:
+            raise ValueError("node_id required")
+        if allowed_workspaces is not None and not self.filter_scoped_nodes(
+            [{"id": node_id}],
+            allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        ):
+            raise ValueError(f"graph node not found: {node_id}")
+        depth = max(0, min(int(depth or 1), 4))
+        limit = max(1, min(int(limit or 100), 500))
+        nt, et = self._read_tables()
+        visited = {node_id}
+        frontier = {node_id}
+        edges_by_id: Dict[str, Dict[str, Any]] = {}
+        with self._connect() as conn:
+            for _ in range(depth):
+                if not frontier or len(visited) >= limit:
+                    break
+                placeholders = ",".join("?" * len(frontier))
+                rows = conn.execute(
+                    f"""
+                        SELECT id, from_node, to_node, type, weight, metadata_json
+                        FROM {et}
+                        WHERE from_node IN ({placeholders}) OR to_node IN ({placeholders})
+                        ORDER BY weight DESC, id ASC
+                        LIMIT ?
+                        """,
+                    (*frontier, *frontier, limit * 3),
+                ).fetchall()
+                next_frontier = set()
+                for row in rows:
+                    edges_by_id[row["id"]] = {
+                        "id": row["id"],
+                        "from": row["from_node"],
+                        "to": row["to_node"],
+                        "type": row["type"],
+                        "weight": row["weight"],
+                        "metadata": _safe_loads(row["metadata_json"]),
+                    }
+                    for candidate in (row["from_node"], row["to_node"]):
+                        if candidate not in visited and len(visited) < limit:
+                            visited.add(candidate)
+                            next_frontier.add(candidate)
+                frontier = next_frontier
+            placeholders = ",".join("?" * len(visited))
+            node_rows = conn.execute(
+                f"""
+                    SELECT id, type, title, summary, metadata_json, updated_at
+                    FROM {nt}
+                    WHERE id IN ({placeholders})
+                    ORDER BY updated_at DESC, id ASC
+                    """,
+                list(visited),
+            ).fetchall()
+        nodes = [
+                {
+                    "id": row["id"],
+                    "type": row["type"],
+                    "title": row["title"],
+                    "summary": row["summary"],
+                    "metadata": _safe_loads(row["metadata_json"]),
+                    "updated_at": row["updated_at"],
+                }
+                for row in node_rows
+            ]
+        edges = list(edges_by_id.values())
+        if allowed_workspaces is not None:
+            nodes = self.filter_scoped_nodes(
+                nodes,
+                allowed_workspaces,
+                include_legacy_global=include_legacy_global,
+            )
+            kept = {node.get("id") for node in nodes}
+            edges = [edge for edge in edges if edge.get("from") in kept and edge.get("to") in kept]
+        return {"root": node_id, "depth": depth, "nodes": nodes, "edges": edges}
+
+    def stats(self) -> Dict[str, Any]:
+        nt, et = self._read_tables()
+        with self._connect() as conn:
+            node_counts = {
+                row["type"]: row["count"]
+                for row in conn.execute(
+                    f"SELECT type, COUNT(*) AS count FROM {nt} GROUP BY type"
+                )
+            }
+            edge_counts = {
+                row["type"]: row["count"]
+                for row in conn.execute(
+                    f"SELECT type, COUNT(*) AS count FROM {et} GROUP BY type"
+                )
+            }
+            local_sources = conn.execute(
+                "SELECT COUNT(*) AS c FROM knowledge_sources"
+            ).fetchone()["c"]
+            local_file_status = {
+                row["status"]: row["count"]
+                for row in conn.execute(
+                    "SELECT status, COUNT(*) AS count FROM local_file_index GROUP BY status"
+                )
+            }
+        v2 = None
+        if KGStoreV2 is not None:
+            try:
+                v2 = KGStoreV2(self.db_path).stats()
+            except Exception as e:
+                v2 = {"available": False, "error": str(e)}
+        return {
+            "db_path": str(self.db_path),
+            "schema_version": GRAPH_SCHEMA_VERSION,
+            "v2_schema_available": KGStoreV2 is not None,
+            "nodes": node_counts,
+            "edges": edge_counts,
+            "local_sources": local_sources,
+            "local_file_status": local_file_status,
+            "v2": v2,
+        }