ltcai 11.5.2 → 11.7.0

package/lattice_brain/graph/retrieval/signals.py

@@ -1,95 +0,0 @@
-"""Honest context-quality and multimodal signals for a result set.
-
-Plain functions over an already-computed match list — no store, no I/O — so
-every mixin in the package (and the chat surfaces outside it) can build the
-same signal the same way.
-"""
-
-from __future__ import annotations
-
-# ruff: noqa: F403,F405
-from .._kg_common import *  # noqa: F403,F401
-
-#: Node types that are a *thing you can look at or listen to*, not prose. A
-#: match of one of these means the answer rests on more than text.
-MULTIMODAL_NODE_TYPES = ("Image", "ImageText")
-
-
-def multimodal_signal(matches: Iterable[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
-    """``{"images": n, "types": [...]}`` when a result set includes pictures.
-
-    ``None`` when it does not: the context-quality contract stays four keys
-    wide for the ordinary all-text case, and a caller that sees the key knows
-    it means something rather than having to compare a zero.
-    """
-    images = 0
-    seen: List[str] = []
-    for match in matches:
-        node_type = str(match.get("type") or "")
-        if node_type in MULTIMODAL_NODE_TYPES:
-            images += 1
-            if node_type not in seen:
-                seen.append(node_type)
-    if not images:
-        return None
-    return {"images": images, "types": seen}
-
-
-def context_quality_signal(
-    mode: str,
-    nodes: int,
-    *,
-    reason: Optional[str] = None,
-    vector: Optional[Dict[str, Any]] = None,
-    multimodal: Optional[Dict[str, Any]] = None,
-) -> Dict[str, Any]:
-    """Honest RAG context-quality signal (v9.8.0, additive contract).
-
-    Shape consumed by the chat metadata channel:
-    ``{"mode": "hybrid"|"lexical_only"|"none", "nodes": int, "limited": bool,
-    "reason": str|None}``. ``nodes == 0`` always collapses ``mode`` to
-    ``"none"``; ``limited`` is true whenever the context is thin (0–1 nodes)
-    or the vector side fell back to lexical-only retrieval. ``reason`` is a
-    short human-readable Korean phrase, only present when limited.
-
-    ``vector`` (v11.1.0) carries the vector channel's own honesty block —
-    which backend scored, whether it was approximate, whether the candidate
-    scan was truncated. "hybrid, 6 nodes" describes two different answers
-    depending on those bits, and the caller that has to say "I did not find
-    it" deserves to know which one it got. The key is present **only when
-    there is a caveat to report**: an exact, complete vector scan is the
-    contract's baseline assumption, so annotating it would be noise, and the
-    four-key shape stays exactly what existing consumers pin.
-
-    ``multimodal`` (v11.1.0) follows the same present-only-when-true rule and
-    says that part of this context is a picture. "6 nodes" reads differently
-    when two of them are screenshots whose text came out of OCR, and the
-    surface that has to explain the answer deserves to know.
-    """
-    nodes = max(0, int(nodes or 0))
-    mode = str(mode or "none")
-    if nodes == 0:
-        mode = "none"
-    if mode not in ("hybrid", "lexical_only", "none"):
-        mode = "lexical_only"
-    limited = nodes <= 1 or mode != "hybrid"
-    if reason is None and limited:
-        if nodes == 0:
-            reason = "그래프에서 관련 지식을 찾지 못했습니다"
-        elif mode == "lexical_only":
-            reason = "벡터 검색을 사용할 수 없어 키워드 검색 결과만 사용했습니다"
-        else:
-            reason = "그래프 기반 컨텍스트가 제한적입니다"
-    if not limited:
-        reason = None
-    signal: Dict[str, Any] = {
-        "mode": mode,
-        "nodes": nodes,
-        "limited": limited,
-        "reason": reason,
-    }
-    if vector is not None:
-        signal["vector"] = dict(vector)
-    if multimodal is not None:
-        signal["multimodal"] = dict(multimodal)
-    return signal

package/lattice_brain/graph/retrieval_docgen.py

@@ -1,253 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-# ruff: noqa: F403,F405
-from ._kg_common import *  # noqa: F403,F401
-
-# The cross-mixin surface (`_connect`, `_upsert_node`, …) is declared in
-# `_kg_contract.KnowledgeGraphCore`. It is a typing-only base: at runtime this
-# is `object`, so the MRO of `KnowledgeGraphStore` is unchanged.
-if TYPE_CHECKING:
-    from ._kg_contract import KnowledgeGraphCore as _Core
-else:
-    _Core = object
-
-
-
-class KnowledgeGraphDocGenMixin(_Core):
-    """Multi-hop retrieval specialised for document generation, split out
-    of retrieval. Composed into KnowledgeGraphStore alongside the other
-    retrieval mixins; shared instance means sibling helpers resolve via MRO.
-    """
-
-    def search_for_document_generation(
-        self,
-        query: str,
-        limit: int = 10,
-        *,
-        allowed_workspaces=None,
-        include_legacy_global: bool = False,
-    ) -> List[Dict[str, Any]]:
-        """Hybrid retrieval optimized for document generation.
-
-        Scoring: 0.5*text_relevance + 0.3*graph_relationship + 0.2*recency
-        Returns nodes with rich context for document generation prompts.
-        """
-        query = str(query or "").strip()
-        if not query:
-            return []
-        limit = max(1, min(int(limit or 10), 50))
-        terms = _topic_candidates(query, limit=12)
-        now = datetime.now()
-        nt, et = self._read_tables()
-
-        with self._connect() as conn:
-            candidate_rows = []
-            seen_ids = set()
-
-            # `query` is non-empty here — the early return above took the blank case.
-            q = f"%{query}%"
-            rows = conn.execute(
-                f"""
-                    SELECT id, type, title, summary, metadata_json, updated_at
-                    FROM {nt}
-                    WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
-                      AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
-                                   'Spreadsheet', 'Image', 'ImageText', 'Audio',
-                                   'Chat', 'Decision', 'Task', 'Concept',
-                                   'Feature', 'Page', 'Slide')
-                    ORDER BY updated_at DESC, id ASC
-                    LIMIT ?
-                    """,
-                (q, q, q, limit * 5),
-            ).fetchall()
-            for row in rows:
-                if row["id"] not in seen_ids:
-                    seen_ids.add(row["id"])
-                    candidate_rows.append(row)
-
-            for term in terms:
-                t = f"%{term}%"
-                rows = conn.execute(
-                    f"""
-                        SELECT id, type, title, summary, metadata_json, updated_at
-                        FROM {nt}
-                        WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
-                          AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
-                                       'Spreadsheet', 'Image', 'ImageText', 'Audio',
-                                       'Chat', 'Decision', 'Task', 'Concept',
-                                       'Feature', 'Page', 'Slide')
-                        ORDER BY updated_at DESC, id ASC
-                        LIMIT ?
-                        """,
-                    (t, t, t, limit * 3),
-                ).fetchall()
-                for row in rows:
-                    if row["id"] not in seen_ids:
-                        seen_ids.add(row["id"])
-                        candidate_rows.append(row)
-
-            scored_results = []
-            for row in candidate_rows:
-                haystack = (
-                    f"{row['title']} {row['summary']} {row['metadata_json']}".lower()
-                )
-
-                text_hits = sum(1 for term in terms if term.lower() in haystack)
-                text_score = min(1.0, text_hits / max(len(terms), 1))
-
-                edge_count = conn.execute(
-                    f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
-                    (row["id"], row["id"]),
-                ).fetchone()["c"]
-                graph_score = min(1.0, math.log1p(edge_count) / 4.0)
-
-                recency = _recency_score(
-                    row["updated_at"], now=now, half_life_days=14.0
-                )
-
-                doc_type_boost = (
-                    1.2
-                    if row["type"]
-                    in (
-                        "Document",
-                        "File",
-                        "SlideDeck",
-                        "Decision",
-                    )
-                    else 1.0
-                )
-
-                hybrid_score = (
-                    0.5 * text_score + 0.3 * graph_score + 0.2 * recency
-                ) * doc_type_boost
-
-                meta = _safe_loads(row["metadata_json"])
-                neighbor_concepts = []
-                neighbor_rows = conn.execute(
-                    f"""
-                        SELECT n.id, n.title, n.type FROM {et} e
-                        JOIN {nt} n ON n.id = CASE WHEN e.from_node = ? THEN e.to_node ELSE e.from_node END
-                        WHERE (e.from_node = ? OR e.to_node = ?)
-                          AND n.type IN ('Concept', 'Feature', 'Decision', 'Task')
-                        LIMIT 8
-                        """,
-                    (row["id"], row["id"], row["id"]),
-                ).fetchall()
-                for nr in neighbor_rows:
-                    neighbor_concepts.append({"id": nr["id"], "title": nr["title"], "type": nr["type"]})
-
-                scored_results.append(
-                    {
-                        "id": row["id"],
-                        "type": row["type"],
-                        "title": row["title"],
-                        "summary": row["summary"],
-                        "metadata": meta,
-                        "updated_at": row["updated_at"],
-                        "hybrid_score": round(hybrid_score, 4),
-                        "scores": {
-                            "text": round(text_score, 4),
-                            "graph": round(graph_score, 4),
-                            "recency": round(recency, 4),
-                        },
-                        "related_concepts": neighbor_concepts,
-                    }
-                )
-
-            if allowed_workspaces is not None:
-                scored_results = self.filter_scoped_nodes(
-                    scored_results,
-                    allowed_workspaces,
-                    include_legacy_global=include_legacy_global,
-                )
-                for item in scored_results:
-                    item["related_concepts"] = self.filter_scoped_nodes(
-                        item.get("related_concepts", []),
-                        allowed_workspaces,
-                        include_legacy_global=include_legacy_global,
-                    )
-            scored_results.sort(key=lambda x: x["hybrid_score"], reverse=True)
-            return scored_results[:limit]
-
-    def multi_hop_context(
-        self,
-        node_ids: List[str],
-        max_hops: int = 2,
-        *,
-        allowed_workspaces=None,
-        include_legacy_global: bool = False,
-    ) -> Dict[str, Any]:
-        """Multi-hop graph traversal from seed nodes for richer context."""
-        visited_nodes = set()
-        visited_edges = set()
-        all_nodes = []
-        all_edges = []
-        frontier = set(node_ids)
-        nt, et = self._read_tables()
-
-        with self._connect() as conn:
-            for hop in range(max_hops):
-                if not frontier:
-                    break
-                next_frontier = set()
-                for nid in frontier:
-                    if nid in visited_nodes:
-                        continue
-                    visited_nodes.add(nid)
-                    row = conn.execute(
-                        f"SELECT id, type, title, summary, metadata_json, updated_at FROM {nt} WHERE id=?",
-                        (nid,),
-                    ).fetchone()
-                    if row:
-                        all_nodes.append(
-                            {
-                                "id": row["id"],
-                                "type": row["type"],
-                                "title": row["title"],
-                                "summary": row["summary"],
-                                "metadata": _safe_loads(row["metadata_json"]),
-                                "hop": hop,
-                            }
-                        )
-                    edge_rows = conn.execute(
-                        f"""
-                            SELECT id, from_node, to_node, type, weight
-                            FROM {et} WHERE from_node=? OR to_node=?
-                            ORDER BY id ASC
-                            """,
-                        (nid, nid),
-                    ).fetchall()
-                    for er in edge_rows:
-                        if er["id"] not in visited_edges:
-                            visited_edges.add(er["id"])
-                            all_edges.append(
-                                {
-                                    "from": er["from_node"],
-                                    "to": er["to_node"],
-                                    "type": er["type"],
-                                    "weight": er["weight"],
-                                }
-                            )
-                            other = (
-                                er["to_node"]
-                                if er["from_node"] == nid
-                                else er["from_node"]
-                            )
-                            if other not in visited_nodes:
-                                next_frontier.add(other)
-                frontier = next_frontier
-
-        if allowed_workspaces is not None:
-            all_nodes = self.filter_scoped_nodes(
-                all_nodes,
-                allowed_workspaces,
-                include_legacy_global=include_legacy_global,
-            )
-            visible_ids = {node.get("id") for node in all_nodes}
-            all_edges = [
-                edge for edge in all_edges
-                if edge.get("from") in visible_ids and edge.get("to") in visible_ids
-            ]
-        return {"nodes": all_nodes, "edges": all_edges}

package/lattice_brain/graph/retrieval_policy.py

@@ -1,180 +0,0 @@
-"""Single retrieval policy consulted by both hybrid fusion layers.
-
-Review 2026-07-25 (UX/harness/loop/KG-RAG), Wave 0.2 — "RetrievalPolicy
-단일화" + "query rewrite" + "recency age decay". Before this module the two
-hybrid entry points (``KnowledgeGraphStore.hybrid_search`` two-channel
-alpha fusion and ``SearchService.hybrid_search`` three-channel weight
-fusion) each resolved :func:`lattice_brain.graph.fusion.fusion_profile`
-independently and had no shared place to hang retrieval-wide behavior.
-This module is that single place:
-
-* :func:`rewrite_query` — deterministic, rule-based ko/en normalization
-  (NO LLM). Strips a conservative fixed list of leading/trailing politeness
-  fillers ("~좀 알려줘", "궁금해", "뭐였지", "what is", "please", …) and
-  collapses repeated whitespace. Code-class queries pass through untouched
-  (except whitespace collapse) because exact identifiers/filenames ARE the
-  retrieval signal. ``LATTICEAI_QUERY_REWRITE=0`` disables rewriting.
-* :func:`resolve_policy` — composes the rewrite with
-  :func:`~lattice_brain.graph.fusion.fusion_profile` (the weight table is
-  NOT duplicated here) and adds ``recency_half_life_days``, which is
-  non-``None`` ONLY for the ``recency`` query class — the honest signal
-  that age decay applies exactly where the fusion layers wire it.
-
-Both functions are pure, deterministic, and never raise; the ``fact``
-query class resolves to the historical default weights, so fact-class
-behavior stays byte-compatible. Scores are never combined with ``or``
-(score ``0.0`` is falsy but valid).
-"""
-
-from __future__ import annotations
-
-import os
-import re
-from typing import Any, Dict, List, Mapping, Optional
-
-from .fusion import classify_query, fusion_profile
-
-QUERY_REWRITE_ENV = "LATTICEAI_QUERY_REWRITE"
-
-# Half-life (days) for the recency-class age decay. Matches the default of
-# lattice_brain.graph._kg_fsutil._recency_score, which the fusion layers use
-# to compute the actual multiplier — this module only declares the policy.
-RECENCY_HALF_LIFE_DAYS = 14.0
-
-# A filler strip is applied only when the remaining query keeps at least this
-# many characters — never rewrite a query down to (near-)nothing.
-_MIN_REMAINDER_CHARS = 4
-
-_WS_RE = re.compile(r"\s+")
-
-# Conservative trailing Korean politeness/filler phrases ("… 좀 알려줘",
-# "… 궁금해", "… 뭐였지"). Anchored at end-of-string, must start the query or
-# follow whitespace so word interiors are never clipped.
-_KO_FILLER_TAIL_RE = re.compile(
-    r"(?:^|(?<=\s))(?:좀\s+)?"
-    r"(?:알려\s*줘요?|알려\s*주세요|알려\s*줄래요?"
-    r"|말해\s*줘요?|말해\s*주세요"
-    r"|설명해\s*줘요?|설명해\s*주세요"
-    r"|궁금해요?|궁금합니다"
-    r"|뭐였지|뭐였더라|뭐더라|뭐지|뭐야"
-    r")\s*[?!.…~]*\s*$"
-)
-
-# Conservative leading English filler ("what is …", "tell me about …",
-# optionally prefixed with "please").
-_EN_FILLER_LEAD_RE = re.compile(
-    r"^(?:please\s+)?(?:tell\s+me\s+about|what\s+is|what\s+was|what\s+are|what's)\s+",
-    re.IGNORECASE,
-)
-
-# Conservative trailing English politeness ("…, please?").
-_EN_FILLER_TAIL_RE = re.compile(r"[,\s]*\bplease\b\s*[?!.]*\s*$", re.IGNORECASE)
-
-_FILLER_RULES = (
-    ("strip_filler_ko", _KO_FILLER_TAIL_RE),
-    ("strip_filler_en_leading", _EN_FILLER_LEAD_RE),
-    ("strip_filler_en_trailing", _EN_FILLER_TAIL_RE),
-)
-
-
-def _rewrite_enabled() -> bool:
-    raw = os.getenv(QUERY_REWRITE_ENV, "").strip().lower()
-    return raw not in {"0", "false", "no", "off"}
-
-
-def rewrite_query(query: Any) -> Dict[str, Any]:
-    """Deterministic rule-based ko/en query normalization (never raises).
-
-    Returns ``{"original": str, "rewritten": str, "rules": [rule names]}``.
-    Only three conservative rules exist: whitespace collapse, trailing
-    Korean filler strip, and leading/trailing English filler strip — each
-    filler strip applies only when the remainder keeps >= 4 characters.
-    Code-class queries (exact identifiers/filenames are the signal) are
-    returned unchanged apart from whitespace collapse. The
-    ``LATTICEAI_QUERY_REWRITE=0`` kill-switch disables rewriting entirely
-    (``rewritten == original``). Empty input yields empty output.
-    """
-    try:
-        text = str(query or "").strip()
-    except Exception:  # noqa: BLE001 — rewrite must never raise
-        return {"original": "", "rewritten": "", "rules": []}
-    if not text:
-        return {"original": "", "rewritten": "", "rules": []}
-    original = text
-    if not _rewrite_enabled():
-        return {"original": original, "rewritten": original, "rules": []}
-
-    rules: List[str] = []
-    rewritten = _WS_RE.sub(" ", text).strip()
-    if rewritten != original:
-        rules.append("collapse_whitespace")
-
-    if classify_query(original) == "code":
-        # Exact identifiers/filenames are the retrieval signal — never touch
-        # them (whitespace collapse is the only permitted normalization).
-        return {"original": original, "rewritten": rewritten, "rules": rules}
-
-    for name, pattern in _FILLER_RULES:
-        candidate = pattern.sub("", rewritten).strip()
-        if candidate != rewritten and len(candidate) >= _MIN_REMAINDER_CHARS:
-            rewritten = candidate
-            rules.append(name)
-    return {"original": original, "rewritten": rewritten, "rules": rules}
-
-
-def resolve_policy(
-    query: Any,
-    *,
-    overrides: Optional[Mapping[str, Mapping[str, float]]] = None,
-) -> Dict[str, Any]:
-    """Resolve the full retrieval policy for ``query`` (never raises).
-
-    Composes :func:`rewrite_query` with
-    :func:`~lattice_brain.graph.fusion.fusion_profile` (classification runs
-    on the ORIGINAL query, so ``query_class`` / ``weights`` / ``alpha`` are
-    byte-identical to calling ``fusion_profile`` directly, as both hybrid
-    layers did before this module). Returns::
-
-        {
-          "query_class": "fact" | "code" | "person" | "recency",
-          "weights": {"keyword", "vector", "graph"},   # service fusion
-          "alpha": float,                              # graph-layer fusion
-          "fusion_strategy": "alpha" | "rrf",          # how they combine
-          "original_query": str,
-          "search_query": str,       # the rewritten form to search with
-          "rewrite_rules": [str],
-          "recency_half_life_days": float | None,      # 14.0 only for recency
-        }
-
-    ``fusion_strategy`` is ``"alpha"`` for every class unless
-    ``LATTICEAI_FUSION_STRATEGY`` says otherwise, so the default policy is
-    byte-identical to the pre-11.1.0 one.
-
-    ``recency_half_life_days`` is non-``None`` only for the ``recency``
-    class — the honest contract that age decay applies exactly where the
-    fusion layers wire it, and nowhere else.
-    """
-    rewrite = rewrite_query(query)
-    profile = fusion_profile(rewrite["original"], overrides=overrides)
-    query_class = profile["query_class"]
-    search_query = rewrite["rewritten"] if rewrite["rewritten"] else rewrite["original"]
-    return {
-        "query_class": query_class,
-        "weights": dict(profile["weights"]),
-        "alpha": float(profile["alpha"]),
-        "fusion_strategy": str(profile["strategy"]),
-        "original_query": rewrite["original"],
-        "search_query": search_query,
-        "rewrite_rules": list(rewrite["rules"]),
-        "recency_half_life_days": (
-            RECENCY_HALF_LIFE_DAYS if query_class == "recency" else None
-        ),
-    }
-
-
-__all__ = [
-    "QUERY_REWRITE_ENV",
-    "RECENCY_HALF_LIFE_DAYS",
-    "resolve_policy",
-    "rewrite_query",
-]