ltcai 11.5.2 → 11.7.0

package/lattice_brain/graph/fusion.py

@@ -1,395 +0,0 @@
-"""Query-class aware retrieval fusion weights (backlog #5, review §7.2 E).
-
-Hybrid retrieval fuses keyword/lexical, vector, and graph channels. A single
-static weight set treats "이 함수 왜 죽어?" and "어제 회의에서 뭐 결정했지?"
-identically, which is measurably wrong: code questions live on exact
-identifiers (lexical), person questions lean on relationship edges (graph),
-recency questions need fresher lexical hits over semantic neighbors.
-
-This module is deliberately tiny and deterministic:
-
-* :func:`classify_query` — pure ko/en heuristics → ``fact | code | person |
-  recency`` (``fact`` is the fallback class and its weights are identical to
-  the historical defaults, so fact-class behavior is byte-compatible).
-* :data:`DEFAULT_FUSION_WEIGHTS` — the per-class weight table. ``keyword`` /
-  ``vector`` / ``graph`` feed the three-channel service fusion
-  (``SearchService.hybrid_search``); ``alpha`` is the vector share for the
-  two-channel graph-layer fusion (``KnowledgeGraphStore.hybrid_search``).
-* :func:`fusion_profile` — classify + resolve weights, honoring the
-  ``LATTICEAI_FUSION_WEIGHTS`` env override (JSON, partial per-class merges,
-  e.g. ``{"code": {"alpha": 0.2}}``).
-
-Never uses ``or`` on scores (score 0 is falsy but valid) and never raises:
-a malformed override is ignored, an unknown class falls back to ``fact``.
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import re
-from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple
-
-from ..gates import FeatureGate
-from ..quiet import quiet
-
-QUERY_CLASSES = ("fact", "code", "person", "recency")
-
-FUSION_WEIGHTS_ENV = "LATTICEAI_FUSION_WEIGHTS"
-
-# ── fusion strategy (v11.1.0) ────────────────────────────────────────────────
-# Linear alpha fusion needs the two channels' scores to be comparable. They
-# are not: the lexical channel scores 1/rank while the vector channel scores a
-# max-normalized cosine, so a query where every cosine lands near 0.9 flattens
-# the vector channel to noise. Reciprocal Rank Fusion ignores score magnitudes
-# entirely and fuses *positions*, which is exactly the property that survives
-# incomparable scales.
-#
-# It is opt-in, per query class, and off everywhere by default: alpha fusion
-# is what every existing ranking assertion in this repo describes, and a
-# ranking change dressed up as a bug fix is how retrieval quality regressions
-# get shipped. Turn it on with LATTICEAI_FUSION_STRATEGY=rrf (all classes) or
-# a JSON object like {"code": "rrf"} (per class).
-FUSION_STRATEGY_ENV = "LATTICEAI_FUSION_STRATEGY"
-#: The one-switch form of the per-class table above, for the settings panel:
-#: "combine by rank instead of score", everywhere. It is applied as the *base*
-#: of the table, so the per-class ``LATTICEAI_FUSION_STRATEGY`` config — the
-#: more specific statement — still wins over it, and an install that touched
-#: neither is byte-identical to what it was.
-FUSION_RRF_ENV = "LATTICEAI_FUSION_RRF"
-FUSION_STRATEGIES = ("alpha", "rrf")
-DEFAULT_FUSION_STRATEGY: Dict[str, str] = dict.fromkeys(QUERY_CLASSES, "alpha")
-FUSION_RRF_GATE = FeatureGate(
-    FUSION_RRF_ENV,
-    default=False,
-    name="fusion_rrf",
-    detail="Search channels are combined by rank rather than by score.",
-)
-#: The smoothing constant from the original RRF paper (Cormack et al., 2009).
-#: Larger k flattens the curve, so rank 1 wins by less.
-DEFAULT_RRF_K = 60
-
-# Per-class fusion weights. "fact" mirrors the pre-fusion-gate defaults
-# (SearchService DEFAULT_HYBRID_WEIGHTS + graph-layer alpha=0.6) so the
-# fallback class introduces zero behavioral drift.
-DEFAULT_FUSION_WEIGHTS: Dict[str, Dict[str, float]] = {
-    "fact": {"keyword": 0.35, "vector": 0.40, "graph": 0.25, "alpha": 0.60},
-    # Code recall lives on exact identifiers/tokens → lexical dominates.
-    "code": {"keyword": 0.55, "vector": 0.25, "graph": 0.20, "alpha": 0.35},
-    # Person questions are relationship questions → graph channel leads.
-    "person": {"keyword": 0.30, "vector": 0.30, "graph": 0.40, "alpha": 0.45},
-    # Recency questions want fresh literal hits over semantic neighbors.
-    "recency": {"keyword": 0.45, "vector": 0.35, "graph": 0.20, "alpha": 0.50},
-}
-
-# ── query-class heuristics (pure, ko/en) ─────────────────────────────────────
-
-_CODE_FENCE_RE = re.compile(r"```|`[^`\n]+`")
-_CODE_IDENT_RE = re.compile(
-    # ``\b`` fails between an identifier and an attached Korean particle
-    # ("ingest_folder가"), so use explicit ASCII-boundary lookbehinds instead.
-    r"[A-Za-z_][A-Za-z0-9_]*\(\)"                      # call syntax foo()
-    r"|(?<![A-Za-z0-9_])[a-z0-9]+_[a-z0-9_]+"          # snake_case
-    r"|(?<![A-Za-z0-9_])[a-z]+[A-Z][A-Za-z0-9]*"       # camelCase
-    r"|(?<![A-Za-z0-9_])[\w-]+\.(?:py|js|jsx|ts|tsx|json|yaml|yml|css|html|sql|sh|go|rs|java|rb)(?![A-Za-z0-9])"
-)
-_CODE_WORD_RE = re.compile(
-    r"\b(?:def|class|import|function|traceback|exception|stack\s*trace|"
-    r"bug|error|null|undefined|api|sql|regex)\b"
-    r"|코드|함수|버그|에러|오류|스택|컴파일|빌드\s*실패|구현",
-    re.IGNORECASE,
-)
-_PERSON_RE = re.compile(
-    r"\b(?:who|whom|whose)\b|누구|어떤\s*사람|담당자|팀원|동료|만난\s*사람"
-    r"|[\w.+-]+@[\w-]+\.[\w.]+"        # an email address names a person
-    r"|[가-힣]{2,4}\s*(?:님|씨)(?:\s|$|[을를이가은는의??.!,])",
-    re.IGNORECASE,
-)
-_RECENCY_RE = re.compile(
-    r"최근|어제|오늘|그저께|방금|아까|지난\s*주|지난주|지난\s*달|지난달"
-    r"|이번\s*주|이번주|이번\s*달|이번달"
-    r"|\b(?:recent|recently|yesterday|today|latest|last\s+(?:week|month|night|meeting))\b",
-    re.IGNORECASE,
-)
-
-
-def classify_query(query: Any) -> str:
-    """Classify a retrieval query into fact / code / person / recency.
-
-    Precedence: code (strongest structural signal) → recency (explicit time
-    words) → person → fact. Deterministic, never raises, empty → fact.
-    """
-    text = str(query or "").strip()
-    if not text:
-        return "fact"
-    if _CODE_FENCE_RE.search(text) or _CODE_IDENT_RE.search(text) or _CODE_WORD_RE.search(text):
-        return "code"
-    if _RECENCY_RE.search(text):
-        return "recency"
-    if _PERSON_RE.search(text):
-        return "person"
-    return "fact"
-
-
-def _env_overrides() -> Dict[str, Dict[str, float]]:
-    """Parse LATTICEAI_FUSION_WEIGHTS (JSON) → per-class partial overrides."""
-    raw = os.getenv(FUSION_WEIGHTS_ENV, "").strip()
-    if not raw:
-        return {}
-    try:
-        payload = json.loads(raw)
-    except (ValueError, TypeError):
-        return {}
-    if not isinstance(payload, dict):
-        return {}
-    overrides: Dict[str, Dict[str, float]] = {}
-    for cls, table in payload.items():
-        if cls not in DEFAULT_FUSION_WEIGHTS or not isinstance(table, dict):
-            continue
-        cleaned: Dict[str, float] = {}
-        for key, value in table.items():
-            if key not in ("keyword", "vector", "graph", "alpha"):
-                continue
-            try:
-                cleaned[key] = max(0.0, min(1.0, float(value)))
-            except (TypeError, ValueError):
-                quiet()
-                continue
-        if cleaned:
-            overrides[str(cls)] = cleaned
-    return overrides
-
-
-def fusion_weight_table(
-    overrides: Optional[Mapping[str, Mapping[str, float]]] = None,
-) -> Dict[str, Dict[str, float]]:
-    """Full per-class table: defaults ← env override ← caller override."""
-    table = {cls: dict(weights) for cls, weights in DEFAULT_FUSION_WEIGHTS.items()}
-    for source in (_env_overrides(), overrides or {}):
-        for cls, partial in source.items():
-            if cls in table and isinstance(partial, Mapping):
-                for key, value in partial.items():
-                    if key in table[cls]:
-                        try:
-                            table[cls][key] = max(0.0, min(1.0, float(value)))
-                        except (TypeError, ValueError):
-                            quiet()
-                            continue
-    return table
-
-
-def _env_strategy_overrides() -> Dict[str, str]:
-    """Parse ``LATTICEAI_FUSION_STRATEGY`` → per-class strategy overrides.
-
-    Two accepted forms: a bare strategy name (``rrf``) applies to every class,
-    a JSON object (``{"code": "rrf"}``) applies per class. Anything else —
-    a typo, malformed JSON, an unknown strategy — is ignored rather than
-    raising, so a bad env var degrades to the default fusion instead of
-    taking every search down.
-    """
-    raw = os.getenv(FUSION_STRATEGY_ENV, "").strip()
-    if not raw:
-        return {}
-    if raw.lower() in FUSION_STRATEGIES:
-        return dict.fromkeys(QUERY_CLASSES, raw.lower())
-    try:
-        payload = json.loads(raw)
-    except (ValueError, TypeError):
-        return {}
-    if not isinstance(payload, dict):
-        return {}
-    return {
-        str(cls): str(value).lower()
-        for cls, value in payload.items()
-        if cls in DEFAULT_FUSION_STRATEGY and str(value).lower() in FUSION_STRATEGIES
-    }
-
-
-def fusion_strategy_table(
-    overrides: Optional[Mapping[str, str]] = None,
-) -> Dict[str, str]:
-    """Full per-class table: defaults ← simple switch ← env override ← caller.
-
-    The simple switch (:data:`FUSION_RRF_GATE`, the settings panel's "combine by
-    rank") sits *under* the per-class env config on purpose: a person who wrote
-    ``{"code": "alpha"}`` said something specific, and a single global flag must
-    not overrule it.
-    """
-    table = dict(DEFAULT_FUSION_STRATEGY)
-    if FUSION_RRF_GATE.enabled():
-        table = dict.fromkeys(QUERY_CLASSES, "rrf")
-    for source in (_env_strategy_overrides(), overrides or {}):
-        for cls, value in source.items():
-            if cls in table and str(value).lower() in FUSION_STRATEGIES:
-                table[cls] = str(value).lower()
-    return table
-
-
-def rrf_score(rank: int, *, k: int = DEFAULT_RRF_K) -> float:
-    """One channel's contribution for an item it ranked at ``rank`` (1-based)."""
-    return 1.0 / (int(k) + max(1, int(rank)))
-
-
-def rrf_fuse(
-    rankings: Mapping[str, Sequence[str]],
-    *,
-    k: int = DEFAULT_RRF_K,
-    weights: Optional[Mapping[str, float]] = None,
-) -> Dict[str, float]:
-    """Reciprocal Rank Fusion over per-channel id rankings.
-
-    ``rankings`` maps a channel name to its ids in rank order (best first).
-    An id absent from a channel simply contributes nothing from it — RRF has
-    no notion of a "missing score" to guess at, which is the other reason it
-    survives channels that fail independently.
-
-    ``weights`` optionally scales a channel's contribution (a channel missing
-    from the mapping weighs 1.0). Deterministic and pure.
-    """
-    fused: Dict[str, float] = {}
-    for channel, ordered in rankings.items():
-        weight = 1.0 if weights is None else float(weights.get(channel, 1.0))
-        for rank, item_id in enumerate(ordered, start=1):
-            fused[item_id] = fused.get(item_id, 0.0) + weight * rrf_score(rank, k=k)
-    return fused
-
-
-# ── graph traversal candidate expansion (v11.1.0) ────────────────────────────
-# Both retrieval channels can only return what they matched. A note that
-# never says "postgres" but is one CONTAINS edge away from the one that does
-# is unreachable by either — and it is often the answer. Expansion walks one
-# hop out from the top hits and offers those neighbours as candidates.
-#
-# It is capped, off by default, and reports its own counts, because the
-# failure mode is dilution: an unbounded expansion turns a precise answer into
-# a tour of the graph.
-GRAPH_EXPANSION_ENV = "LATTICEAI_GRAPH_EXPANSION"
-#: Resolved when asked so the settings panel can move it without a restart; the
-#: env parsing is the same set of words the hand-written check used.
-GRAPH_EXPANSION_GATE = FeatureGate(
-    GRAPH_EXPANSION_ENV,
-    default=False,
-    name="graph_expansion",
-    detail="Memories one link away from a hit are offered as extra candidates.",
-)
-DEFAULT_EXPANSION_SEEDS = 3
-DEFAULT_EXPANSION_CAP = 5
-#: Expanded candidates inherit a damped share of their seed's score: they are
-#: related to a hit, they are not themselves a hit.
-EXPANSION_DECAY = 0.5
-
-
-def graph_expansion_enabled() -> bool:
-    """True when the expansion gate opts in (default: off)."""
-    return GRAPH_EXPANSION_GATE.enabled()
-
-
-def expand_with_neighbors(
-    seeds: Sequence[Tuple[str, float]],
-    neighbors_fn: Callable[[str], Any],
-    *,
-    exclude: Optional[Sequence[str]] = None,
-    cap: int = DEFAULT_EXPANSION_CAP,
-) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
-    """One-hop neighbours of ``seeds`` as extra candidates + an honest report.
-
-    ``seeds`` is ``[(node_id, score)]`` best first; ``neighbors_fn`` is the
-    store's ``neighbors``. Returns ``(candidates, report)`` where each
-    candidate carries its originating seed and a decayed score, and the report
-    states how many seeds were walked, how many candidates were kept, and
-    whether the cap bit. A neighbour lookup that fails is skipped and counted,
-    never raised: expansion is an optimization, not a dependency.
-    """
-    known = {str(item) for item in (exclude or ())}
-    cap = max(0, int(cap))
-    candidates: List[Dict[str, Any]] = []
-    failures = 0
-    for seed_id, seed_score in seeds:
-        if len(candidates) >= cap:
-            break
-        try:
-            payload = neighbors_fn(seed_id) or {}
-        except Exception:  # noqa: BLE001 — a bad seed must not fail the search
-            failures += 1
-            continue
-        for node in payload.get("neighbors") or []:
-            node_id = str(node.get("id") or "")
-            if not node_id or node_id in known:
-                continue
-            known.add(node_id)
-            candidates.append(
-                {
-                    "node": node,
-                    "seed": str(seed_id),
-                    "score": round(float(seed_score) * EXPANSION_DECAY, 6),
-                }
-            )
-            if len(candidates) >= cap:
-                break
-    report = {
-        "enabled": True,
-        "seeds": len(seeds),
-        "added": len(candidates),
-        "cap": cap,
-        "truncated": len(candidates) >= cap,
-        "failed_seeds": failures,
-    }
-    return candidates, report
-
-
-def fusion_profile(
-    query: Any,
-    *,
-    overrides: Optional[Mapping[str, Mapping[str, float]]] = None,
-    strategies: Optional[Mapping[str, str]] = None,
-) -> Dict[str, Any]:
-    """Classify ``query`` and return its resolved fusion weights.
-
-    Returns ``{"query_class": str, "weights": {keyword, vector, graph},
-    "alpha": float, "strategy": "alpha"|"rrf"}``. ``weights`` feeds the
-    three-channel service fusion; ``alpha`` feeds the two-channel graph-layer
-    fusion; ``strategy`` selects *how* those channels combine and is
-    ``"alpha"`` for every class unless configured otherwise.
-    """
-    query_class = classify_query(query)
-    table = fusion_weight_table(overrides)
-    resolved = table.get(query_class)
-    if resolved is None:  # defensive — classify_query only emits known classes
-        query_class = "fact"
-        resolved = table["fact"]
-    return {
-        "query_class": query_class,
-        "weights": {
-            "keyword": resolved["keyword"],
-            "vector": resolved["vector"],
-            "graph": resolved["graph"],
-        },
-        "alpha": resolved["alpha"],
-        "strategy": fusion_strategy_table(strategies)[query_class],
-    }
-
-
-__all__ = [
-    "DEFAULT_EXPANSION_CAP",
-    "DEFAULT_EXPANSION_SEEDS",
-    "DEFAULT_FUSION_STRATEGY",
-    "DEFAULT_FUSION_WEIGHTS",
-    "DEFAULT_RRF_K",
-    "EXPANSION_DECAY",
-    "FUSION_RRF_ENV",
-    "FUSION_RRF_GATE",
-    "FUSION_STRATEGIES",
-    "FUSION_STRATEGY_ENV",
-    "FUSION_WEIGHTS_ENV",
-    "GRAPH_EXPANSION_ENV",
-    "GRAPH_EXPANSION_GATE",
-    "QUERY_CLASSES",
-    "classify_query",
-    "expand_with_neighbors",
-    "fusion_profile",
-    "fusion_strategy_table",
-    "fusion_weight_table",
-    "graph_expansion_enabled",
-    "rrf_fuse",
-    "rrf_score",
-]

package/lattice_brain/graph/identity.py

@@ -1,175 +0,0 @@
-"""Device identity — the sovereignty primitive.
-
-Every Lattice installation owns an Ed25519 keypair. Exports are signed by
-it, peers pair against its public key, and imported knowledge records which
-device it came from. The private key never leaves the machine: it lives in
-the OS keyring when one is available, otherwise in a 0600 file under the
-data directory (the storage backend is reported honestly).
-"""
-
-from __future__ import annotations
-
-import base64
-import hashlib
-import json
-import logging
-import os
-from pathlib import Path
-from typing import Any, Dict, Optional
-
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric.ed25519 import (
-    Ed25519PrivateKey,
-    Ed25519PublicKey,
-)
-
-_KEYRING_SERVICE = "lattice-ai-device-identity"
-_KEYRING_ENTRY = "ed25519-private-key"
-
-
-def _b64(data: bytes) -> str:
-    return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=")
-
-
-def _unb64(text: str) -> bytes:
-    padded = text + "=" * (-len(text) % 4)
-    return base64.urlsafe_b64decode(padded.encode("ascii"))
-
-
-def _keyring_opt_in() -> bool:
-    """Keyring storage is opt-in (LATTICEAI_DEVICE_KEY_KEYRING=1).
-
-    OS keychain access can block or prompt during startup/tests; the default
-    is a 0600 file under the data dir, and ``describe()`` reports which
-    backend holds the key — no silent security theater either way.
-    """
-    return os.getenv("LATTICEAI_DEVICE_KEY_KEYRING", "").strip() in {"1", "true", "yes"}
-
-
-class DeviceIdentity:
-    """Loads-or-creates the installation's Ed25519 keypair."""
-
-    def __init__(self, data_dir: Path, *, use_keyring: Optional[bool] = None):
-        if use_keyring is None:
-            use_keyring = _keyring_opt_in()
-        self._data_dir = Path(data_dir)
-        self._key_file = self._data_dir / "device_identity.key"
-        self._private: Ed25519PrivateKey
-        self.storage: str  # "keyring" | "file"
-        self._load_or_create(use_keyring)
-
-    # ── key material ───────────────────────────────────────────────────────
-    def _load_or_create(self, use_keyring: bool) -> None:
-        raw: Optional[bytes] = None
-        backend = "file"
-        if use_keyring:
-            try:
-                import keyring
-
-                stored = keyring.get_password(_KEYRING_SERVICE, _KEYRING_ENTRY)
-                if stored:
-                    raw = _unb64(stored)
-                    backend = "keyring"
-            except Exception as exc:
-                logging.debug("device identity: keyring unavailable (%s)", exc)
-        if raw is None and self._key_file.exists():
-            raw = _unb64(self._key_file.read_text().strip())
-            backend = "file"
-        if raw is None:
-            key = Ed25519PrivateKey.generate()
-            raw = key.private_bytes(
-                serialization.Encoding.Raw,
-                serialization.PrivateFormat.Raw,
-                serialization.NoEncryption(),
-            )
-            backend = self._persist_new(raw, use_keyring)
-        self._private = Ed25519PrivateKey.from_private_bytes(raw)
-        self.storage = backend
-
-    def _persist_new(self, raw: bytes, use_keyring: bool) -> str:
-        if use_keyring:
-            try:
-                import keyring
-
-                keyring.set_password(_KEYRING_SERVICE, _KEYRING_ENTRY, _b64(raw))
-                return "keyring"
-            except Exception as exc:
-                logging.debug("device identity: keyring store failed (%s); using file", exc)
-        self._data_dir.mkdir(parents=True, exist_ok=True)
-        self._key_file.write_text(_b64(raw))
-        os.chmod(self._key_file, 0o600)
-        return "file"
-
-    # ── public surface ─────────────────────────────────────────────────────
-    @property
-    def public_key_b64(self) -> str:
-        return _b64(
-            self._private.public_key().public_bytes(
-                serialization.Encoding.Raw, serialization.PublicFormat.Raw
-            )
-        )
-
-    @property
-    def fingerprint(self) -> str:
-        """Short human-comparable id: sha256 of the raw public key."""
-        raw = self._private.public_key().public_bytes(
-            serialization.Encoding.Raw, serialization.PublicFormat.Raw
-        )
-        digest = hashlib.sha256(raw).hexdigest()
-        return ":".join(digest[i : i + 4] for i in range(0, 16, 4))
-
-    def describe(self) -> Dict[str, Any]:
-        return {
-            "fingerprint": self.fingerprint,
-            "public_key": self.public_key_b64,
-            "algorithm": "ed25519",
-            "storage": self.storage,
-        }
-
-    # ── signing ────────────────────────────────────────────────────────────
-    def sign(self, payload: bytes) -> str:
-        return _b64(self._private.sign(payload))
-
-    def sign_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
-        """Detached signature over the canonical JSON of a manifest."""
-        canonical = json.dumps(manifest, sort_keys=True, ensure_ascii=False).encode("utf-8")
-        return {
-            "algorithm": "ed25519",
-            "public_key": self.public_key_b64,
-            "fingerprint": self.fingerprint,
-            "signature": self.sign(canonical),
-        }
-
-
-def fingerprint_of(public_key_b64: str) -> str:
-    """Human-comparable fingerprint of an Ed25519 public key.
-
-    Raises ValueError when the input is not a valid key — the pairing flow
-    uses this as its validation gate.
-    """
-    raw = _unb64(public_key_b64)
-    Ed25519PublicKey.from_public_bytes(raw)  # validates; raises on garbage
-    digest = hashlib.sha256(raw).hexdigest()
-    return ":".join(digest[i : i + 4] for i in range(0, 16, 4))
-
-
-def verify_signature(public_key_b64: str, payload: bytes, signature_b64: str) -> bool:
-    """True iff ``signature`` is valid for ``payload`` under the given key."""
-    try:
-        key = Ed25519PublicKey.from_public_bytes(_unb64(public_key_b64))
-        key.verify(_unb64(signature_b64), payload)
-        return True
-    except Exception:
-        return False
-
-
-def verify_manifest(manifest: Dict[str, Any], signature_block: Dict[str, Any]) -> bool:
-    canonical = json.dumps(manifest, sort_keys=True, ensure_ascii=False).encode("utf-8")
-    return verify_signature(
-        str(signature_block.get("public_key") or ""),
-        canonical,
-        str(signature_block.get("signature") or ""),
-    )
-
-
-__all__ = ["DeviceIdentity", "verify_signature", "verify_manifest"]