@mindrian_os/install 1.13.0-beta.11

package/lib/core/rs_math.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+"""
+rs-math.py -- Shared math helpers for the Reverse Salient Engine (Phase 89)
+==============================================================================
+
+Ports the authoritative Hughes 1983 / Kwan 2023 reverse-salient algorithm into
+reusable pure functions. Consumed by:
+  - scripts/rs-engine.py (Plan 89-01, Mode A single-room)
+  - scripts/rs-engine.py Mode B/C (Plans 89-04, 89-05)
+
+Authoritative source: .planning/phases/89-reverse-salient-engine/source/lsa.py
+(Newton Kwan et al., 2023). The distinguishing property of this algorithm is
+TOPIC-KEYWORD-MEMBERSHIP COUNTING + signed abs-diff detection -- NOT
+cosine-on-SVD, NOT MiniLM cosine. Do not replace these with more modern
+defaults; ALGORITHM-SOURCE.md documents why the keyword-membership signal is
+load-bearing.
+
+License: BSL-1.1 (see LICENSE at repo root).
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, List, Sequence, Tuple
+
+import numpy as np
+
+
+# ---------------------------------------------------------------------------
+# Step 1: TF-IDF + Truncated SVD
+# ---------------------------------------------------------------------------
+
+def build_tfidf_svd(
+    texts: Sequence[str],
+    n_components: int = 80,
+    max_features: int = 2000,
+    max_df: float = 0.5,
+    random_state: int = 256,
+) -> Tuple[object, object, object]:
+    """Fit TF-IDF + TruncatedSVD on a corpus.
+
+    Ports source/lsa.py lines 39-55 faithfully. Parameter choices
+    (2000 features, max_df=0.5, 80 components, n_iter=10, seed=256) are taken
+    verbatim from the authoritative source. For corpora smaller than 80 docs,
+    n_components is clamped to max(1, N-1) to avoid SVD underflow.
+
+    Returns (vectorizer, svd_model, X) where X is the sparse TF-IDF matrix.
+    """
+    # Imported lazily so rs-math.py can be imported without sklearn for tests
+    # that exercise only abs_diff_topk / classify_direction.
+    from sklearn.decomposition import TruncatedSVD
+    from sklearn.feature_extraction.text import TfidfVectorizer
+
+    vectorizer = TfidfVectorizer(
+        stop_words="english",
+        max_features=max_features,
+        max_df=max_df,
+        smooth_idf=True,
+    )
+    X = vectorizer.fit_transform(texts)
+
+    n_rows, n_terms = X.shape
+    effective_components = max(1, min(n_components, n_rows - 1, n_terms - 1))
+    svd_model = TruncatedSVD(
+        n_components=effective_components,
+        algorithm="randomized",
+        n_iter=10,
+        random_state=random_state,
+    )
+    svd_model.fit(X)
+
+    return vectorizer, svd_model, X
+
+
+# ---------------------------------------------------------------------------
+# Step 2: Extract top-k keywords per SVD component
+# ---------------------------------------------------------------------------
+
+def extract_topic_keywords(svd_model, terms: Sequence[str], top_k: int = 7) -> List[List[str]]:
+    """For each SVD component, return the top_k terms by weight descending.
+
+    Ports source/lsa.py lines 59-72 faithfully. `terms` is typically the
+    output of `vectorizer.get_feature_names_out()`.
+    """
+    topics: List[List[str]] = []
+    for comp in svd_model.components_:
+        sorted_terms = sorted(
+            zip(terms, comp), key=lambda pair: pair[1], reverse=True
+        )[:top_k]
+        topics.append([t for t, _ in sorted_terms])
+    return topics
+
+
+# ---------------------------------------------------------------------------
+# Step 3: Topic-keyword-membership counting (the authoritative signature)
+# ---------------------------------------------------------------------------
+
+def count_topic_membership(
+    tokenized_papers: Sequence[Sequence[str]],
+    topics: Sequence[Sequence[str]],
+) -> np.ndarray:
+    """Count how many of each paper's tokens appear in each topic's keyword set.
+
+    Ports source/lsa.py lines 80-98 faithfully.
+
+    CRITICAL: this is NOT cosine-on-SVD. It measures whether papers share
+    topic-level keywords (top 7 terms per SVD component). ALGORITHM-SOURCE.md
+    line 72 warns that swapping this for cosine-on-SVD changes the signal
+    entirely. Do not optimize this.
+
+    Returns a (n_papers, n_topics) float32 matrix.
+    """
+    n_papers = len(tokenized_papers)
+    n_topics = len(topics)
+
+    # Precompute topic keyword sets for O(1) membership lookup per token.
+    topic_sets = [set(t) for t in topics]
+
+    counts = np.zeros((n_papers, n_topics), dtype=np.float32)
+    for i, tokens in enumerate(tokenized_papers):
+        for word in tokens:
+            for j, kw_set in enumerate(topic_sets):
+                if word in kw_set:
+                    counts[i, j] += 1.0
+    return counts
+
+
+# ---------------------------------------------------------------------------
+# Step 4: Row-normalize + pairwise L1 distance + invert to similarity
+# ---------------------------------------------------------------------------
+
+def normalize_and_l1_similarity(topic_count_matrix: np.ndarray) -> np.ndarray:
+    """Row-normalize, compute pairwise L1 distance, invert, and rescale.
+
+    Ports source/lsa.py lines 102-126 faithfully. Steps:
+      1. Row-normalize (each paper becomes a topic distribution summing to 1)
+      2. Pairwise L1 distance via broadcast
+      3. Invert: sim = max(dist) - dist (closer pairs score higher)
+      4. Rescale to [0, 1] centered at 0.5
+
+    Diagonal entries should be ~1.0 (self-similarity maximum) after rescaling.
+
+    Returns a (N, N) float32 similarity matrix.
+    """
+    matrix = np.asarray(topic_count_matrix, dtype=np.float32)
+    n = matrix.shape[0]
+
+    row_sums = matrix.sum(axis=1)
+    # Papers with zero keyword hits would divide by zero; substitute 1.0 so the
+    # row stays all-zeros after normalization (most-dissimilar bucket). This
+    # matches the np.nan_to_num behavior in source/lsa.py line 106.
+    safe_row_sums = np.where(row_sums == 0.0, 1.0, row_sums)
+    normalized = (matrix.T / safe_row_sums).T
+
+    # Pairwise L1 distance: |normalized[i] - normalized[j]| summed across
+    # topics, same broadcast trick as the authoritative source.
+    diff = np.abs(normalized[:, None, :] - normalized[None, :, :])
+    sum_matrix = diff.sum(axis=2).astype(np.float32)
+
+    # Invert so that smaller distances become larger similarities.
+    sum_matrix = sum_matrix.max() - sum_matrix
+
+    # Rescale to [0, 1] centered at 0.5 (authoritative step).
+    rng = sum_matrix.max() - sum_matrix.min()
+    if rng == 0:
+        # Degenerate corpus (all identical) -- return identity-like matrix so
+        # abs_diff_topk still produces deterministic output.
+        return np.eye(n, dtype=np.float32)
+    midpoint = (sum_matrix.max() + sum_matrix.min()) / 2.0
+    return ((sum_matrix - midpoint) / rng) + 0.5
+
+
+# ---------------------------------------------------------------------------
+# Step 5: abs-diff top-k iterative argmax with symmetric cleanup
+# ---------------------------------------------------------------------------
+
+def abs_diff_topk(
+    lsa_matrix: np.ndarray,
+    semantic_matrix: np.ndarray,
+    k: int = 1000,
+    skip_diagonal: bool = True,
+) -> List[Tuple[int, int, float, float]]:
+    """Return the top-k pairs by |semantic - lsa| differential.
+
+    Ports source/comparison.py lines 130-147 faithfully. The whole innovation
+    detection is this single signal: where do semantic and structural
+    similarity disagree most?
+
+    Each returned tuple is (i, j, signed_diff, abs_diff) where:
+      - signed_diff = semantic_matrix[i, j] - lsa_matrix[i, j]
+      - abs_diff    = |signed_diff|
+      - signed_diff > 0 -> structural_transfer (different words, similar meaning)
+      - signed_diff < 0 -> semantic_implementation (same words, different meaning)
+
+    `skip_diagonal=True` zeroes out self-pairs before iteration so
+    (i, i) never wins. Symmetric cleanup ensures (j, i) is suppressed when
+    (i, j) is selected, preventing duplicate reporting.
+    """
+    lsa = np.asarray(lsa_matrix, dtype=np.float64)
+    sem = np.asarray(semantic_matrix, dtype=np.float64)
+    if lsa.shape != sem.shape:
+        raise ValueError(
+            f"shape mismatch: lsa {lsa.shape} vs semantic {sem.shape}"
+        )
+    n = lsa.shape[0]
+    if n < 2:
+        return []
+
+    signed_diff = sem - lsa
+    abs_diff = np.abs(signed_diff).copy()
+
+    if skip_diagonal:
+        np.fill_diagonal(abs_diff, -np.inf)
+
+    # Upper triangle only: symmetric matrix means (i,j) == (j,i); the
+    # authoritative source handles this by suppressing the mirror after each
+    # pick. We precompute mask so argmax never returns lower-triangle.
+    iu = np.tril_indices(n, k=-1)
+    abs_diff[iu] = -np.inf
+
+    results: List[Tuple[int, int, float, float]] = []
+    k = max(0, min(k, (n * (n - 1)) // 2))
+    for _ in range(k):
+        flat_index = int(np.argmax(abs_diff))
+        i, j = np.unravel_index(flat_index, abs_diff.shape)
+        max_val = float(abs_diff[i, j])
+        if max_val == -np.inf or max_val <= 0.0:
+            break
+        results.append((int(i), int(j), float(signed_diff[i, j]), float(max_val)))
+        # Symmetric cleanup: source/comparison.py lines 145-146 subtract the
+        # max-val so the mirror never wins next iteration. We mask to -inf
+        # directly since we already restrict to upper triangle.
+        abs_diff[i, j] = -np.inf
+        abs_diff[j, i] = -np.inf
+
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Step 6: Direction classification
+# ---------------------------------------------------------------------------
+
+def classify_direction(signed_diff: float) -> str:
+    """Classify an RS pair by the sign of (semantic - lsa).
+
+    Based on ALGORITHM-SOURCE.md lines 150-153:
+      - signed > 0 -> structural_transfer      (different keywords, similar meaning)
+      - signed <= 0 -> semantic_implementation (same keywords, different meaning)
+
+    The zero case is rare in practice (post-rescaling); we bucket it with
+    semantic_implementation for deterministic output.
+    """
+    return "structural_transfer" if float(signed_diff) > 0.0 else "semantic_implementation"
+
+
+# ---------------------------------------------------------------------------
+# Convenience: full LSA pipeline on raw text corpus
+# ---------------------------------------------------------------------------
+
+def build_lsa_matrix(texts: Sequence[str], n_components: int = 80, top_k: int = 7) -> np.ndarray:
+    """Run the full authoritative LSA pipeline end-to-end.
+
+    Equivalent to running build_tfidf_svd + extract_topic_keywords +
+    count_topic_membership + normalize_and_l1_similarity in sequence. Provided
+    as a single entry point so scripts/rs-engine.py and plans 89-04/89-05 can
+    call one function and not re-wire the pipeline.
+
+    Tokenization mirrors source/lsa.py lines 14-17: simple whitespace split on
+    the raw text, so topic-keyword matching compares against the same tokens
+    that TF-IDF saw. Do not pre-strip punctuation (see ALGORITHM-SOURCE.md
+    Pitfall 1 in RESEARCH.md line 569-573).
+    """
+    tokenized = [t.split() for t in texts]
+    vec, svd, _X = build_tfidf_svd(texts, n_components=n_components)
+    topics = extract_topic_keywords(svd, vec.get_feature_names_out(), top_k=top_k)
+    counts = count_topic_membership(tokenized, topics)
+    return normalize_and_l1_similarity(counts)
+
+
+__all__ = [
+    "build_tfidf_svd",
+    "extract_topic_keywords",
+    "count_topic_membership",
+    "normalize_and_l1_similarity",
+    "abs_diff_topk",
+    "classify_direction",
+    "build_lsa_matrix",
+]

package/lib/core/rs_rooms.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+"""
+rs_rooms.py -- Multi-room artifact loader for the Reverse Salient Engine
+==============================================================================
+
+Plan 89-04 Mode A extension: load artifacts from N user rooms simultaneously
+and tag every artifact with its source room_id so the engine can filter
+pairs to cross-room bridges only. Rediscovers the Yissum x CU x Hopkins
+connections the user found manually.
+
+Design notes:
+  - Room artifacts live on the filesystem (room/section/artifact.md), NOT in
+    a SQLite `artifacts` table. The Phase 89 baseline (Plan 89-01, scripts/
+    rs-engine.py:discover_artifacts) walks the filesystem because
+    lib/core/lazygraph-ops.cjs has no such table. This module follows that
+    same pattern per-room and aggregates into one flat corpus.
+  - global_id = "{room_id}::{artifact_id}" guarantees uniqueness across
+    rooms even when two rooms name their artifacts identically. This is the
+    plan's explicit risk-mitigation.
+  - Missing/unreadable rooms are skipped with a stderr warning; the engine
+    continues with whatever rooms successfully loaded. Mode B/C and Mode A
+    single-room require artifacts; cross-room requires >= 2 rooms actually
+    contributing artifacts for any cross-room pair to exist.
+
+Authoritative source invariants preserved:
+  - Empty body filter (>= 50 chars of real content, matching discover_artifacts).
+  - Skip dirs (.git, .lazygraph, .mindrian, node_modules, .obsidian).
+  - Skip files (STATE.md, ROOM.md, MINTO.md -- metadata, not venture content).
+  - Section = first-segment of relative path (same as discover_artifacts).
+
+License: BSL-1.1 (see LICENSE at repo root).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Sequence
+
+
+# --- Constants (kept in sync with scripts/rs-engine.py discover_artifacts) ---
+
+SKIP_FILES = {"STATE.md", "ROOM.md", "MINTO.md"}
+SKIP_DIRS = {".lazygraph", ".git", ".mindrian", "node_modules", ".obsidian"}
+MIN_BODY_CHARS = 50
+
+
+# --- Helpers (minimal local copies so rs_rooms does not import from scripts/) -
+
+def _extract_title(content: str, filepath: Path) -> str:
+    match = re.search(r"^# (.+)$", content, re.MULTILINE)
+    if match:
+        return match.group(1).strip()
+    return filepath.stem.replace("-", " ").title()
+
+
+def _extract_body(content: str) -> str:
+    fm_match = re.match(r"^---\n[\s\S]*?\n---\n?", content)
+    if fm_match:
+        return content[fm_match.end():]
+    return content
+
+
+# --- Public API -------------------------------------------------------------
+
+def load_multi_room_corpus(room_paths: Sequence[str]) -> List[Dict]:
+    """Load artifacts from N rooms, tagging each with its source room_id.
+
+    Args:
+      room_paths: list of filesystem paths, each pointing to one room root.
+        Room root = the directory that contains section subdirectories
+        (e.g. `~/rooms/yissum-deeptech`, not `~/rooms/yissum-deeptech/room`).
+
+    Returns:
+      Flat list of artifact dicts. Each artifact carries:
+        - global_id      : f"{room_id}::{artifact_id}" (unique across corpus)
+        - room_id        : basename of the room path (e.g. "yissum-deeptech")
+        - artifact_id    : per-room id (section/stem), same shape as 89-01
+        - section        : first path segment under the room root
+        - title          : H1 header or humanized filename
+        - text           : body text (post-frontmatter strip)
+        - path           : absolute filesystem path to the source .md
+        - source_room_path: absolute path to the room root (debugging)
+
+    Behavior:
+      - Missing rooms (no directory) print a stderr note and are skipped.
+        Not fatal: the engine may still have enough artifacts from other rooms.
+      - Rooms with zero usable artifacts (empty sections, files < 50 chars)
+        are silently skipped.
+      - Artifact order is deterministic: rooms in input order, files sorted
+        by filesystem walk. This matters because index-based pair lookups
+        in rs-engine.py assume the artifact list is stable across runs.
+    """
+    corpus: List[Dict] = []
+    seen_room_ids: Dict[str, int] = {}
+
+    for rp in room_paths:
+        room_path = Path(rp).resolve()
+        if not room_path.exists():
+            print(
+                f"[rs-rooms] Skip {rp}: path does not exist",
+                file=sys.stderr,
+            )
+            continue
+        if not room_path.is_dir():
+            print(
+                f"[rs-rooms] Skip {rp}: not a directory",
+                file=sys.stderr,
+            )
+            continue
+
+        room_id = room_path.name
+        # Disambiguate if two different paths share a basename. The second
+        # occurrence gets a suffix so global_id uniqueness survives.
+        seen_count = seen_room_ids.get(room_id, 0)
+        if seen_count > 0:
+            room_id = f"{room_id}-{seen_count + 1}"
+        seen_room_ids[room_path.name] = seen_count + 1
+
+        room_contribution = _load_single_room(room_path, room_id)
+        if not room_contribution:
+            print(
+                f"[rs-rooms] Skip {rp}: no usable artifacts (>= {MIN_BODY_CHARS} chars)",
+                file=sys.stderr,
+            )
+            continue
+        corpus.extend(room_contribution)
+
+    return corpus
+
+
+def _load_single_room(room_path: Path, room_id: str) -> List[Dict]:
+    """Walk one room and return artifact dicts tagged with room_id."""
+    artifacts: List[Dict] = []
+    for root, dirs, files in os.walk(room_path):
+        dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
+        rel_root = Path(root).relative_to(room_path)
+        if str(rel_root) == ".":
+            # Skip room-root files -- not part of any section.
+            continue
+        section = str(rel_root).split(os.sep)[0]
+        for fname in sorted(files):
+            if not fname.endswith(".md"):
+                continue
+            if fname in SKIP_FILES:
+                continue
+            fpath = Path(root) / fname
+            try:
+                content = fpath.read_text(encoding="utf-8")
+            except (OSError, UnicodeDecodeError):
+                continue
+            body = _extract_body(content).strip()
+            if len(body) < MIN_BODY_CHARS:
+                continue
+            artifact_id = str(rel_root / Path(fname).stem).replace(os.sep, "/")
+            global_id = f"{room_id}::{artifact_id}"
+            artifacts.append({
+                "global_id": global_id,
+                "room_id": room_id,
+                "artifact_id": artifact_id,
+                "section": section,
+                "title": _extract_title(content, fpath),
+                "text": body,
+                "path": str(fpath),
+                "source_room_path": str(room_path),
+            })
+    return artifacts
+
+
+def summarize_corpus(corpus: Sequence[Dict]) -> Dict[str, int]:
+    """Return {room_id: artifact_count} for a loaded corpus.
+
+    Used by the engine to warn when a single room dominates the corpus
+    (>= 95% share would skew LSA toward that room's vocabulary and make
+    cross-room pair discovery unreliable). The 5% warning threshold is
+    documented in the plan's Risks section.
+    """
+    counts: Dict[str, int] = {}
+    for art in corpus:
+        rid = art.get("room_id") or "unknown"
+        counts[rid] = counts.get(rid, 0) + 1
+    return counts
+
+
+__all__ = [
+    "load_multi_room_corpus",
+    "summarize_corpus",
+    "MIN_BODY_CHARS",
+    "SKIP_FILES",
+    "SKIP_DIRS",
+]