@mindrian_os/cli 1.13.0-beta.10 → 1.13.0-beta.44

package/lib/core/rs_hybrid.py

@@ -1,586 +0,0 @@
-#!/usr/bin/env python3
-"""
-rs_hybrid.py -- Room x External Unified Corpus (Phase 89 Plan 89-05)
-=====================================================================
-
-Mode C builder: unifies a user room's artifacts with an external research
-corpus into a single indexable corpus, plus a cross-corpus post-filter
-that keeps only pairs where one side is a room artifact and the other is
-an external document.
-
-Architecture (Plan 89-05):
-  - Room side: walk the room filesystem (skip lazygraph, obsidian, etc.),
-    reuse the discover_artifacts pattern from scripts/rs-engine.py so Mode
-    A and Mode C see the same artifact shape.
-  - External side: reuse the Pinecone rs-external cache from Plan 89-03.
-    Cold path fetches via lib.core.rs_corpus and upserts; warm path reads
-    cached multilingual-e5-large vectors directly.
-  - Unified corpus: flat list of dicts with `origin` in {room, external}
-    and `global_id` that is globally unique across both sides.
-  - origin_mask: numpy bool array, True=room, False=external. Enables O(1)
-    cross-corpus pair filtering in rs-engine.
-
-Public API:
-  build_unified_corpus(room_path, topic, external_target=2000)
-      -> (corpus, origin_mask, metadata)
-  filter_cross_corpus_pairs(pairs, origin_mask, topk)
-      -> list[dict]  pairs where origin_mask[i] != origin_mask[j]
-
-Part 8 Graph Boundary (Canon, mandatory):
-  Zero user-content egress. Room artifacts are read locally and used as
-  LSA / semantic inputs in-process only. The external corpus metadata
-  stored in Pinecone (via rs_cache.upsert_corpus) is strictly public
-  OpenAlex/arXiv/Tavily data. Room content never touches Pinecone.
-
-Three-surface coverage (CLAUDE.md tri-polar rule):
-  CLI     Consumed by scripts/rs-engine.py --mode hybrid.
-  Desktop ReverseSalientAgent invokes rs-engine.py as a child process
-          (Plan 89-07 wires the conversational trigger).
-  Cowork  When COWORK=1, the hybrid results symlink into 00_Context/
-          at the rs-engine CLI layer (unchanged from Mode A/B).
-
-License: BSL-1.1 (see LICENSE at repo root).
-  Licensed under the Business Source License 1.1. Change Date 2029-04-24.
-  Change License Apache 2.0.
-"""
-
-from __future__ import annotations
-
-import os
-import re
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-try:
-    import numpy as np
-except ImportError:
-    print(
-        "rs_hybrid requires numpy. Run: pip install -r requirements-hsi.txt",
-        file=sys.stderr,
-    )
-    raise
-
-
-# --- Shared constants mirrored from scripts/rs-engine.py --------------------
-
-# Files we never treat as artifacts (room identity / state, not content).
-SKIP_FILES = {"STATE.md", "ROOM.md", "MINTO.md"}
-# Directories we never descend into (metadata / tooling, not room artifacts).
-SKIP_DIRS = {".lazygraph", ".git", ".mindrian", "node_modules", ".obsidian"}
-# Minimum body length to treat a .md file as an artifact. Matches
-# scripts/rs-engine.py:discover_artifacts and scripts/compute-hsi.py so the
-# Mode A and Mode C corpora see identical artifact inclusion rules.
-MIN_BODY_CHARS = 50
-
-# Maximum external docs we will fold into the unified corpus. Plan 89-05
-# canonical value is 2000 (aligns with Pinecone rs-external namespace cap
-# MAX_NAMESPACE_VECTORS = 10_000 / 5, leaving room for multiple topics).
-DEFAULT_EXTERNAL_TARGET = 2000
-
-# External target is bounded so a misconfigured caller cannot balloon the
-# corpus into a 50k-vector LSA fit that blows memory.
-MAX_EXTERNAL_TARGET = 5000
-
-
-# --- Frontmatter + body helpers (mirror scripts/rs-engine.py) ---------------
-
-def _parse_frontmatter_skip(content: str) -> str:
-    """Strip a YAML frontmatter block if present; return the rest."""
-    fm_match = re.match(r"^---\n[\s\S]*?\n---\n?", content)
-    if fm_match:
-        return content[fm_match.end():]
-    return content
-
-
-def _extract_title(content: str, filepath: Path) -> str:
-    match = re.search(r"^# (.+)$", content, re.MULTILINE)
-    if match:
-        return match.group(1).strip()
-    return filepath.stem.replace("-", " ").title()
-
-
-# --- Room artifact loader ---------------------------------------------------
-
-def _load_room_artifacts(room_path: Path) -> List[Dict[str, Any]]:
-    """Walk room_path for .md artifacts. Returns room-origin dicts.
-
-    Each artifact carries:
-      - global_id: f"room::{room_id}::{artifact_id}"  (unique across corpus)
-      - origin:    "room"
-      - room_id:   basename of the room directory
-      - artifact_id: relative path stem (section/name)
-      - section:   first path segment under the room
-      - title:     first H1 or humanized filename
-      - text:      post-frontmatter body (the LSA / semantic input)
-
-    Pattern reuse: mirrors scripts/rs-engine.py:discover_artifacts byte-for-
-    byte so single-room Mode A and hybrid Mode C file the same artifact the
-    same way. Plan 89-01 Deviation #2 (no SQLite artifacts table) carries
-    through -- we walk the filesystem.
-    """
-    artifacts: List[Dict[str, Any]] = []
-    room_id = room_path.name
-
-    for root, dirs, files in os.walk(room_path):
-        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
-        rel_root = Path(root).relative_to(room_path)
-        if str(rel_root) == ".":
-            # Skip room-root files -- not part of any section.
-            continue
-        section = str(rel_root).split(os.sep)[0]
-        for fname in sorted(files):
-            if not fname.endswith(".md"):
-                continue
-            if fname in SKIP_FILES:
-                continue
-            fpath = Path(root) / fname
-            try:
-                content = fpath.read_text(encoding="utf-8")
-            except (OSError, UnicodeDecodeError):
-                continue
-            body = _parse_frontmatter_skip(content).strip()
-            if len(body) < MIN_BODY_CHARS:
-                continue
-            artifact_id = str(rel_root / Path(fname).stem).replace(os.sep, "/")
-            title = _extract_title(content, fpath)
-            artifacts.append({
-                "global_id": f"room::{room_id}::{artifact_id}",
-                "origin": "room",
-                "room_id": room_id,
-                "artifact_id": artifact_id,
-                "section": section,
-                "title": title,
-                "path": str(fpath.relative_to(room_path)),
-                "text": body,
-            })
-    return artifacts
-
-
-# --- External loader (via rs_cache) -----------------------------------------
-
-def _load_external_records(
-    topic: str,
-    external_target: int,
-) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
-    """Fetch (cold) or read (warm) the external corpus via Plan 89-03 cache.
-
-    Returns (records, cache_meta).
-      records:    list of {id, values, metadata} as returned by rs_cache.
-      cache_meta: {cache_mode, cache_age_days, cache_namespace, cache_ttl_days,
-                   corpus_size}.
-
-    Routes:
-      - If PINECONE_API_KEY is missing or RS_EMBEDDING_MODEL=minilm:
-        returns ([], {"cache_mode": "bypass", ...}). Caller falls back to
-        local MiniLM + rs_corpus fetch (Plan 89-02 path).
-      - If cache is warm (age <= TTL): skip fetch, return cached records.
-      - If cold: fetch via rs_corpus, upsert to Pinecone, re-read so the
-        semantic vectors are the server-side multilingual-e5-large ones.
-
-    Consumes Plan 89-03's public API verbatim. No new Pinecone calls live
-    here; this module is a composition layer.
-    """
-    try:
-        from lib.core.rs_cache import (
-            namespace_slug,
-            get_namespace_freshness,
-            upsert_corpus,
-            fetch_all_from_namespace,
-            is_fresh,
-            TTL_DAYS,
-        )
-        from lib.core.rs_corpus import fetch_corpus
-    except Exception as e:
-        return [], {
-            "cache_mode": "bypass",
-            "cache_age_days": None,
-            "cache_namespace": None,
-            "cache_ttl_days": None,
-            "corpus_size": 0,
-            "skip_reason": f"rs_cache import failed: {e}",
-        }
-
-    # Bypass check: the Plan 89-03 bypass conditions.
-    api_key = os.environ.get("PINECONE_API_KEY")
-    model_env = os.environ.get("RS_EMBEDDING_MODEL", "").strip().lower()
-    if not api_key:
-        return [], {
-            "cache_mode": "bypass",
-            "cache_age_days": None,
-            "cache_namespace": None,
-            "cache_ttl_days": None,
-            "corpus_size": 0,
-            "skip_reason": "PINECONE_API_KEY not set",
-        }
-    if model_env == "minilm":
-        return [], {
-            "cache_mode": "bypass",
-            "cache_age_days": None,
-            "cache_namespace": None,
-            "cache_ttl_days": None,
-            "corpus_size": 0,
-            "skip_reason": "RS_EMBEDDING_MODEL=minilm (user opted out)",
-        }
-
-    ns = namespace_slug(topic)
-    try:
-        age_days = get_namespace_freshness(ns)
-    except Exception as e:
-        print(
-            f"[rs-hybrid] freshness check failed ({e}); treating as cold.",
-            file=sys.stderr,
-        )
-        age_days = None
-
-    if is_fresh(age_days, ttl_days=TTL_DAYS):
-        # WARM path.
-        print(
-            f"[rs-hybrid] Cache hit (age={age_days:.1f} days, ttl={TTL_DAYS}). "
-            f"Using Pinecone namespace {ns}.",
-            file=sys.stderr,
-        )
-        try:
-            records = fetch_all_from_namespace(ns, limit=external_target)
-            return records, {
-                "cache_mode": "warm",
-                "cache_age_days": round(age_days, 2) if age_days is not None else None,
-                "cache_namespace": ns,
-                "cache_ttl_days": TTL_DAYS,
-                "corpus_size": len(records),
-            }
-        except Exception as e:
-            print(
-                f"[rs-hybrid] warm-path fetch failed ({e}); dropping to cold.",
-                file=sys.stderr,
-            )
-
-    # COLD path.
-    age_label = "unset" if age_days is None else f"{age_days:.1f} days"
-    print(
-        f"[rs-hybrid] Cold path: fetching external corpus for {topic!r} "
-        f"(cache age={age_label}).",
-        file=sys.stderr,
-    )
-    try:
-        # Overshoot fetch target so dedup + abstract-present filter still
-        # yields near-external_target usable docs (same pattern as Mode B).
-        fetch_target = max(external_target * 2, external_target + 200)
-        docs = fetch_corpus(topic, target_n=fetch_target)
-    except Exception as e:
-        print(f"[rs-hybrid] corpus fetch failed ({e}).", file=sys.stderr)
-        return [], {
-            "cache_mode": "bypass",
-            "cache_age_days": None,
-            "cache_namespace": ns,
-            "cache_ttl_days": TTL_DAYS,
-            "corpus_size": 0,
-            "skip_reason": f"fetch_corpus failed: {e}",
-        }
-
-    if len(docs) < 2:
-        return [], {
-            "cache_mode": "bypass",
-            "cache_age_days": None,
-            "cache_namespace": ns,
-            "cache_ttl_days": TTL_DAYS,
-            "corpus_size": len(docs),
-            "skip_reason": "cold fetch returned < 2 usable docs",
-        }
-
-    try:
-        upsert_corpus(topic, docs)
-        records = fetch_all_from_namespace(ns, limit=external_target)
-    except Exception as e:
-        print(
-            f"[rs-hybrid] upsert/fetch failed ({e}); dropping to bypass.",
-            file=sys.stderr,
-        )
-        return [], {
-            "cache_mode": "bypass",
-            "cache_age_days": None,
-            "cache_namespace": ns,
-            "cache_ttl_days": TTL_DAYS,
-            "corpus_size": 0,
-            "skip_reason": f"upsert/fetch failed: {e}",
-        }
-
-    return records, {
-        "cache_mode": "cold",
-        "cache_age_days": 0.0,
-        "cache_namespace": ns,
-        "cache_ttl_days": TTL_DAYS,
-        "corpus_size": len(records),
-    }
-
-
-def _external_records_to_dicts(records: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """Shape rs_cache records into unified-corpus dicts (origin=external).
-
-    Fields mirrored from Plan 89-02 / 89-03 Mode B output so downstream
-    bridge-writer (Plan 89-06) can resolve both sides via the existing
-    resolvePairIdentity() without per-mode branches.
-
-    Attached _vector field carries the cached multilingual-e5-large
-    embedding when present; rs-engine will prefer it over local MiniLM to
-    keep warm/cold semantic consistency with Mode B.
-    """
-    out: List[Dict[str, Any]] = []
-    for rec in records:
-        rid = rec.get("id") or ""
-        meta = rec.get("metadata", {}) or {}
-        abstract = (meta.get("abstract") or "").strip()
-        if not abstract:
-            continue
-        source = (meta.get("source") or "external").strip() or "external"
-        title = (meta.get("title") or "").strip() or rid
-        doi = (meta.get("doi") or "").strip() or None
-        year = meta.get("year") or None
-        # Build a stable artifact-style id that matches Mode B cold-path shape.
-        eid_slug = re.sub(r"[^a-zA-Z0-9]+", "-", rid).strip("-")
-        if not eid_slug:
-            eid_slug = re.sub(r"[^a-zA-Z0-9]+", "-", title or "untitled").strip("-")
-        external_id = f"{source}/{eid_slug}"
-        values = rec.get("values") or []
-        out.append({
-            "global_id": f"external::{external_id}",
-            "origin": "external",
-            "external_id": external_id,
-            "pinecone_id": rid,
-            "source": source,
-            "title": title,
-            "doi": doi,
-            "year": year,
-            "path": f"https://doi.org/{doi}" if doi else rid,
-            "text": abstract,
-            "_vector": list(values) if values else [],
-        })
-    return out
-
-
-def _external_docs_to_dicts(docs: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """Bypass-path shape: raw rs_corpus docs -> unified-corpus dicts.
-
-    Used when Pinecone is unavailable and the caller cold-fetched via
-    rs_corpus.fetch_corpus directly. Vectors are not attached (caller embeds
-    locally via MiniLM).
-    """
-    out: List[Dict[str, Any]] = []
-    for doc in docs:
-        abstract = (doc.get("abstract") or "").strip()
-        if not abstract:
-            continue
-        rid = doc.get("external_id") or ""
-        source = (doc.get("source") or "external").strip() or "external"
-        title = (doc.get("title") or "").strip() or rid
-        doi = (doc.get("doi") or "").strip() or None
-        year = doc.get("year")
-        eid_slug = re.sub(r"[^a-zA-Z0-9]+", "-", rid).strip("-")
-        if not eid_slug:
-            eid_slug = re.sub(r"[^a-zA-Z0-9]+", "-", title or "untitled").strip("-")
-        external_id = f"{source}/{eid_slug}"
-        out.append({
-            "global_id": f"external::{external_id}",
-            "origin": "external",
-            "external_id": external_id,
-            "pinecone_id": rid,
-            "source": source,
-            "title": title,
-            "doi": doi,
-            "year": year,
-            "path": doc.get("url") or rid,
-            "text": abstract,
-            "_vector": [],  # bypass path -- no cached vector.
-        })
-    return out
-
-
-# --- Public builder ---------------------------------------------------------
-
-def build_unified_corpus(
-    room_path: str,
-    topic: str,
-    external_target: int = DEFAULT_EXTERNAL_TARGET,
-) -> Tuple[List[Dict[str, Any]], "np.ndarray", Dict[str, Any]]:
-    """Build a unified (room + external) corpus for Mode C.
-
-    Returns:
-      corpus:      flat list of dicts. Room artifacts first (preserved
-                   discover order), then external docs (Pinecone namespace
-                   order or fetch order on bypass path).
-      origin_mask: np.ndarray[bool], True=room, False=external. Same length
-                   as corpus. Used by filter_cross_corpus_pairs.
-      metadata:    {room_count, external_count, room_id, topic, topic_slug,
-                    cache_mode, cache_age_days, cache_namespace,
-                    cache_ttl_days, external_target, fetched_at}.
-
-    Edge cases:
-      - Empty room: returns ([], array([]), {...room_count=0...}). Caller
-        should short-circuit (no cross-corpus pair is constructible).
-      - Empty external: same pattern. Callers should emit a clear "no
-        external corpus available" message rather than running the LSA.
-      - Bypass path with rs_corpus importable: falls through to cold fetch
-        via rs_corpus.fetch_corpus and shapes docs directly.
-      - Bypass path with rs_corpus NOT importable: returns empty external
-        side (caller decides what to do; typically short-circuits).
-
-    external_target is clamped to MAX_EXTERNAL_TARGET so misconfigured
-    callers cannot request 50k docs and blow memory.
-    """
-    # Clamp external_target defensively.
-    external_target = max(1, min(int(external_target), MAX_EXTERNAL_TARGET))
-
-    room_path_obj = Path(room_path).resolve()
-    if not room_path_obj.is_dir():
-        raise ValueError(f"rs_hybrid: room path is not a directory: {room_path_obj}")
-
-    room_artifacts = _load_room_artifacts(room_path_obj)
-    room_id = room_path_obj.name
-
-    # External side.
-    records, cache_meta = _load_external_records(topic, external_target)
-    if records:
-        external_dicts = _external_records_to_dicts(records)
-    else:
-        # Attempt the Plan 89-02 bypass path: fetch via rs_corpus directly
-        # if it is available. If rs_corpus import failed upstream, the
-        # cache_meta carries skip_reason and we return an empty external
-        # side -- caller decides whether to abort or continue with only
-        # room artifacts (which cross-corpus filter will discard anyway).
-        external_dicts = []
-        try:
-            from lib.core.rs_corpus import fetch_corpus
-            print(
-                "[rs-hybrid] Bypass path: fetching via rs_corpus (no Pinecone).",
-                file=sys.stderr,
-            )
-            fetch_target = max(external_target * 2, external_target + 200)
-            docs = fetch_corpus(topic, target_n=fetch_target)
-            external_dicts = _external_docs_to_dicts(docs[:external_target])
-            # Update cache_meta corpus_size to reflect actual bypass-path
-            # usable count.
-            cache_meta["corpus_size"] = len(external_dicts)
-        except Exception as e:
-            print(
-                f"[rs-hybrid] bypass-path fetch also failed ({e}); "
-                f"proceeding with empty external corpus.",
-                file=sys.stderr,
-            )
-
-    # Cap external side at external_target explicitly (defense-in-depth in
-    # case cache returned more than requested).
-    external_dicts = external_dicts[:external_target]
-
-    corpus: List[Dict[str, Any]] = list(room_artifacts) + list(external_dicts)
-
-    # Build origin_mask. True=room, False=external. Matches plan contract.
-    origin_mask = np.array(
-        [c["origin"] == "room" for c in corpus],
-        dtype=bool,
-    )
-
-    # Resolve topic slug the same way Mode B does so the hybrid output
-    # path aligns with research/{topic-slug}/ conventions.
-    try:
-        from lib.core.rs_corpus import topic_slug
-        slug = topic_slug(topic)
-    except Exception:
-        slug = re.sub(r"[^a-z0-9]+", "-", (topic or "").lower()).strip("-")
-
-    metadata: Dict[str, Any] = {
-        "room_count": int(origin_mask.sum()),
-        "external_count": int((~origin_mask).sum()),
-        "room_id": room_id,
-        "room_path": str(room_path_obj),
-        "topic": topic,
-        "topic_slug": slug,
-        "external_target": external_target,
-        "fetched_at": datetime.now(timezone.utc).isoformat(),
-    }
-    # Surface the 89-03 cache provenance so rs-engine can write it to
-    # .rs-engine-results.json and downstream consumers see warm/cold mode.
-    metadata.update({k: v for k, v in cache_meta.items() if k != "corpus_size"})
-
-    return corpus, origin_mask, metadata
-
-
-# --- Cross-corpus pair filter -----------------------------------------------
-
-def filter_cross_corpus_pairs(
-    pairs: Sequence[Any],
-    origin_mask: "np.ndarray",
-    topk: int,
-) -> List[Dict[str, Any]]:
-    """Keep only pairs where one side is room and other is external.
-
-    Accepts either:
-      - abs_diff_topk tuple output: list of (i, j, signed, abs_diff) tuples.
-      - Dict-shaped pairs already carrying i/j (future-proofing; not used by
-        the current rs-engine call site but allows unit testing with mocks).
-
-    Returns a list of dicts augmented with room_side and external_side
-    (integer indices into the unified corpus). The bridge-writer (Plan
-    89-06) consumes these through resolvePairIdentity.
-
-    Stops as soon as `topk` cross-corpus pairs have been collected. The
-    caller should request abs_diff_topk with an overshoot multiplier so
-    intra-corpus pairs (room-room or external-external) can be discarded
-    without starving the final result below topk.
-    """
-    out: List[Dict[str, Any]] = []
-    topk = max(1, int(topk))
-
-    for p in pairs:
-        # Normalize input shape.
-        if isinstance(p, dict):
-            i = int(p.get("i", -1))
-            j = int(p.get("j", -1))
-            signed = p.get("signed_diff", p.get("signed"))
-            absv = p.get("abs_diff", p.get("abs"))
-        else:
-            try:
-                i, j, signed, absv = p
-            except (ValueError, TypeError):
-                continue
-            i, j = int(i), int(j)
-
-        if i < 0 or j < 0:
-            continue
-        if i >= len(origin_mask) or j >= len(origin_mask):
-            continue
-        if bool(origin_mask[i]) == bool(origin_mask[j]):
-            # Same origin -- intra-corpus pair. Discard.
-            continue
-
-        # Canonical orientation: room_side is the room index, external_side
-        # is the external index. Matches plan Detailed Steps snippet.
-        if bool(origin_mask[i]):
-            room_side, external_side = i, j
-        else:
-            room_side, external_side = j, i
-
-        out.append({
-            "i": i,
-            "j": j,
-            "signed_diff": float(signed) if signed is not None else 0.0,
-            "abs_diff": float(absv) if absv is not None else 0.0,
-            "room_side": room_side,
-            "external_side": external_side,
-        })
-
-        if len(out) >= topk:
-            break
-
-    return out
-
-
-__all__ = [
-    "build_unified_corpus",
-    "filter_cross_corpus_pairs",
-    "DEFAULT_EXTERNAL_TARGET",
-    "MAX_EXTERNAL_TARGET",
-    "SKIP_FILES",
-    "SKIP_DIRS",
-    "MIN_BODY_CHARS",
-]