nexo-brain 7.31.13 → 7.33.0

package/src/db/_memory_v2.py

@@ -24,6 +24,103 @@ def _core():
     return module
 
 
+def _cognitive():
+    """Lazy import of the cognitive embedding core.
+
+    Imported lazily so the DB layer never hard-depends on the (heavy) cognitive
+    stack and so importing db._memory_v2 cannot fail when numpy/fastembed are
+    unavailable in a stripped environment.
+    """
+    module = sys.modules.get("cognitive._core")
+    if module is None:
+        module = importlib.import_module("cognitive._core")
+    return module
+
+
+# Identifier persisted alongside each precomputed observation vector so the
+# fusion path can refuse to compare vectors produced by an incompatible model.
+OBSERVATION_EMBEDDING_MODEL = "bge-base-embeddings"
+
+
+def _model_is_warm() -> bool:
+    """Return True only when embedding will NOT trigger a cold model load.
+
+    Two safe cases:
+      * the deterministic offline fallback is active
+        (NEXO_SKIP_COGNITIVE_MODEL_DOWNLOAD) — fast and dependency-free, so it is
+        always safe to embed; and
+      * the real fastembed model is already loaded in this process.
+
+    A cold real model returns False so the latency path degrades to FTS instead
+    of paying a download/load cost on a single query.
+    """
+    try:
+        cog = _cognitive()
+    except Exception:
+        return False
+    try:
+        if cog._model_download_disabled():
+            return True
+    except Exception:
+        return False
+    return getattr(cog, "_model", None) is not None
+
+
+def _embedding_columns_present(conn) -> bool:
+    try:
+        cols = {row[1] for row in conn.execute("PRAGMA table_info(memory_observations)").fetchall()}
+    except Exception:
+        return False
+    return "embedding" in cols and "embedding_model" in cols
+
+
+def _embed_text_blob(text: str):
+    """Embed text into (blob, model_name). Returns (None, '') on any failure.
+
+    Never raises; never blocks indefinitely on a cold model — callers gate on
+    _model_is_warm() before calling on the latency path.
+    """
+    clean = str(text or "").strip()
+    if not clean:
+        return None, ""
+    try:
+        cog = _cognitive()
+        vector = cog.embed(clean)
+        blob = cog._array_to_blob(vector)
+        if not blob:
+            return None, ""
+        return blob, OBSERVATION_EMBEDDING_MODEL
+    except Exception:
+        return None, ""
+
+
+def _write_observation_embedding(uid: str, *, summary: str = "", subject: str = "") -> bool:
+    """Precompute and store an observation embedding. Guarded; never raises.
+
+    Called AFTER the observation row is committed. A failure here must never
+    surface to the write path — the observation is already durable; the vector
+    is a shadow optimisation that the bounded backfill can fill in later.
+    """
+    try:
+        conn = _core().get_db()
+        if not _table_exists(conn, "memory_observations") or not _embedding_columns_present(conn):
+            return False
+        if not _model_is_warm():
+            return False
+        text = " ".join(part for part in [str(subject or "").strip(), str(summary or "").strip()] if part).strip()
+        blob, model_name = _embed_text_blob(text)
+        if blob is None:
+            return False
+        conn.execute(
+            "UPDATE memory_observations SET embedding = ?, embedding_model = ? WHERE observation_uid = ?",
+            (blob, model_name, uid),
+        )
+        conn.commit()
+        return True
+    except Exception:
+        return False
+
+
 _REDACT_PATTERNS = (
     re.compile(r"sk-[a-zA-Z0-9_\-]{20,}"),
     re.compile(r"ghp_[a-zA-Z0-9]{20,}"),
@@ -254,6 +351,12 @@ def _row_to_observation(row) -> dict:
     item["evidence_refs"] = _parse_json(item.pop("evidence_refs_json", "[]"), [])
     item["entities"] = _parse_json(item.pop("entities_json", "[]"), [])
     item["metadata"] = _parse_json(item.pop("metadata_json", "{}"), {})
+    # The shadow embedding BLOB is an internal optimisation, not user-facing
+    # payload. Drop the raw bytes (they are not JSON-serialisable) but expose a
+    # cheap boolean so callers/tests can assert it was precomputed.
+    embedding_blob = item.pop("embedding", None)
+    item.pop("embedding_model", None)
+    item["has_embedding"] = bool(embedding_blob)
     return item
 
 
@@ -671,6 +774,11 @@ def upsert_memory_observation(observation: dict) -> dict:
         ),
     )
     conn.commit()
+    # Precompute the semantic embedding AFTER the write is durable. This is a
+    # shadow optimisation: it is guarded, never blocks the write, and skips
+    # entirely when the model is cold/unavailable (the bounded backfill fills
+    # those rows later). The summary/subject already passed redaction above.
+    _write_observation_embedding(uid, summary=clean_summary, subject=clean_subject)
     row = conn.execute("SELECT * FROM memory_observations WHERE observation_uid = ?", (uid,)).fetchone()
     result = _row_to_observation(row) if row else {"observation_uid": uid}
     result["ok"] = True
@@ -898,6 +1006,174 @@ def search_memory_observations_fts(
     return [_row_to_observation(row) for row in rows]
 
 
+def backfill_observation_embeddings(*, limit: int = 200) -> dict:
+    """Bounded, idempotent backfill of precomputed observation embeddings.
+
+    Only touches rows whose ``embedding IS NULL`` (idempotent — re-running after
+    a full pass is a no-op). Bounded by ``limit`` so it never scans the whole
+    table in one call; callers loop until ``remaining == 0``. Skips entirely
+    when the model is cold so it never triggers a download on a hot path.
+    """
+    conn = _core().get_db()
+    if not _table_exists(conn, "memory_observations"):
+        return {"ok": True, "updated": 0, "skipped": True, "reason": "memory_observations table unavailable"}
+    if not _embedding_columns_present(conn):
+        return {"ok": True, "updated": 0, "skipped": True, "reason": "embedding columns unavailable"}
+    if not _model_is_warm():
+        return {"ok": True, "updated": 0, "skipped": True, "reason": "embedding model cold"}
+
+    max_rows = max(1, min(int(limit or 200), 1000))
+    try:
+        rows = conn.execute(
+            """
+            SELECT id, observation_uid, subject, summary
+              FROM memory_observations
+             WHERE embedding IS NULL
+             ORDER BY created_at DESC, id DESC
+             LIMIT ?
+            """,
+            (max_rows,),
+        ).fetchall()
+    except Exception as exc:
+        return {"ok": False, "updated": 0, "error": _truncate(str(exc), 300)}
+
+    updated = 0
+    failed = 0
+    for row in rows:
+        item = dict(row)
+        text = " ".join(
+            part for part in [str(item.get("subject") or "").strip(), str(item.get("summary") or "").strip()] if part
+        ).strip()
+        blob, model_name = _embed_text_blob(text)
+        if blob is None:
+            failed += 1
+            continue
+        try:
+            conn.execute(
+                "UPDATE memory_observations SET embedding = ?, embedding_model = ? WHERE id = ?",
+                (blob, model_name, item.get("id")),
+            )
+            updated += 1
+        except Exception:
+            failed += 1
+    if updated:
+        conn.commit()
+
+    try:
+        remaining = int(
+            conn.execute("SELECT COUNT(*) FROM memory_observations WHERE embedding IS NULL").fetchone()[0]
+        )
+    except Exception:
+        remaining = 0
+    return {
+        "ok": failed == 0,
+        "updated": updated,
+        "failed": failed,
+        "seen": len(rows),
+        "remaining": remaining,
+    }
+
+
+def vector_scan_observations(
+    query_vector,
+    *,
+    limit: int = 50,
+    scan_limit: int = 400,
+    start_ts: float | None = None,
+    end_ts: float | None = None,
+    project_key: str = "",
+    min_score: float = 0.0,
+) -> list[dict]:
+    """Bounded cosine scan over precomputed observation embeddings.
+
+    ``scan_limit`` caps how many embedded rows are deserialised/compared so a
+    single query can never walk an unbounded table. Returns the top ``limit``
+    matches as ``{observation_uid, vector_score}`` dicts. Never raises — any
+    failure yields an empty list so the caller degrades to FTS.
+    """
+    if query_vector is None:
+        return []
+    conn = _core().get_db()
+    if not _table_exists(conn, "memory_observations") or not _embedding_columns_present(conn):
+        return []
+    try:
+        cog = _cognitive()
+    except Exception:
+        return []
+
+    clauses = ["embedding IS NOT NULL"]
+    params: list[Any] = []
+    if start_ts is not None:
+        clauses.append("created_at >= ?")
+        params.append(float(start_ts))
+    if end_ts is not None:
+        clauses.append("created_at < ?")
+        params.append(float(end_ts))
+    if project_key.strip():
+        clauses.append("project_key = ?")
+        params.append(project_key.strip())
+
+    bounded_scan = max(1, min(int(scan_limit or 400), 2000))
+    bounded_limit = max(1, min(int(limit or 50), 200))
+    try:
+        rows = conn.execute(
+            f"""
+            SELECT observation_uid, embedding
+              FROM memory_observations
+             WHERE {' AND '.join(clauses)}
+             ORDER BY salience DESC, created_at DESC, id DESC
+             LIMIT ?
+            """,
+            params + [bounded_scan],
+        ).fetchall()
+    except Exception:
+        return []
+
+    scored: list[dict] = []
+    for row in rows:
+        blob = row["embedding"]
+        if not blob:
+            continue
+        try:
+            candidate_vector = cog._blob_to_array(blob)
+            score = cog.cosine_similarity(query_vector, candidate_vector)
+        except Exception:
+            continue
+        if score <= min_score:
+            continue
+        scored.append({"observation_uid": row["observation_uid"], "vector_score": float(score)})
+
+    scored.sort(key=lambda item: item["vector_score"], reverse=True)
+    return scored[:bounded_limit]
+
+
+def get_memory_observations_by_uids(uids: list[str]) -> dict[str, dict]:
+    """Fetch observation rows by uid (bounded). Returns {uid: observation}.
+
+    Used by the retrieval fusion path to materialise semantic-only matches that
+    the lexical/FTS scan did not surface. Bounded to a small batch; never raises.
+    """
+    conn = _core().get_db()
+    if not _table_exists(conn, "memory_observations"):
+        return {}
+    clean = [str(uid).strip() for uid in (uids or []) if str(uid).strip()][:200]
+    if not clean:
+        return {}
+    placeholders = ",".join("?" for _ in clean)
+    try:
+        rows = conn.execute(
+            f"SELECT * FROM memory_observations WHERE observation_uid IN ({placeholders})",
+            clean,
+        ).fetchall()
+    except Exception:
+        return {}
+    result: dict[str, dict] = {}
+    for row in rows:
+        item = _row_to_observation(row)
+        result[item.get("observation_uid")] = item
+    return result
+
+
 def memory_observation_stats(days: int = 7) -> dict:
     conn = _core().get_db()
     if not _table_exists(conn, "memory_observations"):

package/src/db/_schema.py

@@ -2119,6 +2119,81 @@ def _m64_local_context_live_dirs(conn):
     )
 
 
+def _m84_local_chunks_fts(conn):
+    """FTS5 keyword index over local_chunks for semantic-keyword local-file recall.
+
+    Additive + idempotent + reversible. Creates an FTS5 virtual table
+    ``local_chunks_fts`` keyed by ``local_chunks.rowid`` (local_chunks is NOT
+    WITHOUT ROWID, so its implicit rowid is stable and usable as the FTS key)
+    plus triggers that mirror local_chunks insert/delete/update. The FTS row
+    stores a denormalized snapshot of the owning asset's ``privacy_class`` /
+    ``status`` so privacy can be coarse-prefiltered INSIDE the FTS query without
+    a heavy join. The authoritative privacy check stays on the real
+    local_assets join + is_queryable_path in the retrieval path.
+
+    NOTE: no bulk INSERT...SELECT backfill here (that would scan/lock the live
+    19GB DB at schema time). The incremental, resumable backfill runs in the
+    cron tick (local_context/api.py:_backfill_fts_rows). No vector_blob column,
+    no VACUUM — explicitly out of scope.
+    """
+    try:
+        conn.execute(
+            """
+            CREATE VIRTUAL TABLE IF NOT EXISTS local_chunks_fts USING fts5(
+                text,
+                privacy_class UNINDEXED,
+                asset_status UNINDEXED,
+                tokenize='unicode61 remove_diacritics 2'
+            )
+            """
+        )
+    except Exception:
+        # Hosts without FTS5 support: fall back to a plain shadow table so the
+        # triggers below still succeed and the dual-read simply never flips on
+        # (FTS MATCH would raise OperationalError -> retrieval stays on LIKE).
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS local_chunks_fts (
+                rowid INTEGER PRIMARY KEY,
+                text TEXT DEFAULT '',
+                privacy_class TEXT DEFAULT '',
+                asset_status TEXT DEFAULT ''
+            )
+            """
+        )
+    conn.executescript(
+        """
+        CREATE TRIGGER IF NOT EXISTS local_chunks_fts_insert
+        AFTER INSERT ON local_chunks BEGIN
+            INSERT INTO local_chunks_fts(rowid, text, privacy_class, asset_status)
+            VALUES (
+                new.rowid,
+                new.text,
+                COALESCE((SELECT privacy_class FROM local_assets WHERE asset_id=new.asset_id), 'normal'),
+                COALESCE((SELECT status FROM local_assets WHERE asset_id=new.asset_id), 'active')
+            );
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS local_chunks_fts_delete
+        AFTER DELETE ON local_chunks BEGIN
+            DELETE FROM local_chunks_fts WHERE rowid = old.rowid;
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS local_chunks_fts_update
+        AFTER UPDATE OF text, asset_id ON local_chunks BEGIN
+            DELETE FROM local_chunks_fts WHERE rowid = old.rowid;
+            INSERT INTO local_chunks_fts(rowid, text, privacy_class, asset_status)
+            VALUES (
+                new.rowid,
+                new.text,
+                COALESCE((SELECT privacy_class FROM local_assets WHERE asset_id=new.asset_id), 'normal'),
+                COALESCE((SELECT status FROM local_assets WHERE asset_id=new.asset_id), 'active')
+            );
+        END;
+        """
+    )
+
+
 def _backfill_diary_quality(conn):
     for table in ("session_diary", "diary_archive"):
         conn.execute(f"""
@@ -3081,6 +3156,63 @@ def _m81_core_rules_product_metadata(conn):
     conn.execute("CREATE INDEX IF NOT EXISTS idx_core_rules_protected ON core_rules(protected, is_active)")
 
 
+def _m82_confidence_checks(conn):
+    """Persist nexo_confidence_check calls so the answer-contract gate works.
+
+    The G1 enforcer (hooks/g1_enforcer.py) treats a verify/ask/defer contract as
+    fulfilled when a confidence_checks row exists for the session created after
+    the task opened. No code ever created or wrote this table, so verify
+    contracts were structurally unfulfillable. handle_confidence_check now writes
+    a row per call; created_at uses datetime('now') to match opened_at format.
+    """
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS confidence_checks (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            session_id TEXT,
+            task_id TEXT,
+            goal_hash TEXT,
+            task_type TEXT,
+            area TEXT,
+            response_mode TEXT,
+            confidence INTEGER,
+            high_stakes INTEGER NOT NULL DEFAULT 0,
+            created_at TEXT
+        )
+        """
+    )
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_confidence_checks_session "
+        "ON confidence_checks(session_id, created_at)"
+    )
+
+
+def _m83_observation_embeddings(conn):
+    """Shadow embedding columns for semantic retrieval over observations.
+
+    Additive + reversible: two nullable columns on memory_observations. The
+    embedding is precomputed at write time (after the row is committed) and by a
+    bounded backfill. memory_search fuses the precomputed vector with the
+    existing FTS/token score so paraphrases retrieve the right observation.
+
+    The query/latency path never triggers a cold model load — it embeds the
+    query once only when a model is already warm (or the deterministic offline
+    fallback is active) and otherwise degrades to today's FTS-only behaviour.
+    """
+    # Ensure the base table exists before ALTER: partial-DB upgrade paths (which
+    # mark earlier migrations applied without materializing every table) would
+    # otherwise hit "no such table: memory_observations". m60 is idempotent.
+    _m60_memory_observations(conn)
+    _migrate_add_column(conn, "memory_observations", "embedding", "BLOB")
+    _migrate_add_column(conn, "memory_observations", "embedding_model", "TEXT DEFAULT ''")
+    # Partial index so the bounded backfill can find un-embedded rows cheaply.
+    conn.execute(
+        "CREATE INDEX IF NOT EXISTS idx_memory_obs_embedding_pending "
+        "ON memory_observations(id) WHERE embedding IS NULL"
+    )
+    conn.commit()
+
+
 MIGRATIONS = [
     (1, "learnings_columns", _m1_learnings_columns),
     (2, "followups_reasoning", _m2_followups_reasoning),
@@ -3163,6 +3295,8 @@ MIGRATIONS = [
     (79, "operational_closure_links_readiness", _m79_operational_closure_links_readiness),
     (80, "opportunity_orchestrator", _m80_opportunity_orchestrator),
     (81, "core_rules_product_metadata", _m81_core_rules_product_metadata),
+    (82, "confidence_checks", _m82_confidence_checks),
+    (83, "observation_embeddings", _m83_observation_embeddings),
 ]
 
 

package/src/hooks/auto_capture.py

@@ -288,31 +288,69 @@ def _content_hash(fact_type: str, content: str) -> str:
 # ---------------------------------------------------------------------------
 
 
-def _auto_learning_add(title: str, content: str) -> bool:
-    """Best-effort call to tools_learnings.add_learning.
+def _get_auto_capture_logger():
+    """Lazy, guarded logger so capture failures are observable, never silent.
 
-    Returns True when the learning was stored, False otherwise. Failures
-    are silent so the hook itself never breaks the user's prompt flow.
+    Import must never raise (tmp/read-only homes in tests) — falls back to a
+    NullHandler. Mirrors the enforcement_engine logger pattern.
     """
+    import logging
+    log = logging.getLogger("nexo.auto_capture")
+    if log.handlers:
+        return log
     try:
-        import tools_learnings  # type: ignore
+        import paths
+        d = paths.logs_dir()
+        d.mkdir(parents=True, exist_ok=True)
+        handler = logging.FileHandler(d / "auto_capture.log")
+        handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
+        log.addHandler(handler)
+        log.setLevel(logging.INFO)
     except Exception:
-        return False
+        log.addHandler(logging.NullHandler())
+    return log
 
+
+def _auto_learning_add(title: str, content: str) -> bool:
+    """Persist a correction-derived learning via tools_learnings.handle_learning_add.
+
+    Failures are LOGGED (never silently swallowed) and retried once with a tiny
+    backoff so a transient DB-lock contention on the shared Brain self-heals
+    within the same prompt. Never raises and never breaks the prompt flow
+    (returns bool); all logging/retry stays inside the hook's exit-0 contract.
+    """
+    import time
+    log = _get_auto_capture_logger()
     try:
-        result = tools_learnings.add_learning(
-            category="auto",
-            title=title,
-            content=content,
-            priority="medium",
-            reasoning="auto-captured from correction pattern in UserPromptSubmit/PostToolUse hook",
-        )
-        if isinstance(result, dict):
-            return bool(result.get("ok") or result.get("id") or result.get("learning_id"))
-        return bool(result)
-    except Exception:
+        import tools_learnings  # type: ignore
+    except Exception as exc:
+        log.info("auto_capture: tools_learnings unavailable (%s)", exc.__class__.__name__)
         return False
 
+    for attempt in range(2):
+        try:
+            result = tools_learnings.handle_learning_add(
+                category="auto",
+                title=title,
+                content=content,
+                priority="medium",
+                reasoning="auto-captured from correction pattern in UserPromptSubmit/PostToolUse hook",
+            )
+            if isinstance(result, str):
+                ok = not result.strip().upper().startswith("ERROR")
+            elif isinstance(result, dict):
+                ok = bool(result.get("ok") or result.get("id") or result.get("learning_id"))
+            else:
+                ok = bool(result)
+            if ok:
+                return True
+            log.warning("auto_capture learning_add failed (attempt %d/2): %r", attempt + 1, result)
+        except Exception as exc:
+            log.warning("auto_capture learning_add error (attempt %d/2): %s", attempt + 1, exc)
+        if attempt == 0:
+            time.sleep(0.05)
+    return False
+
 
 # ---------------------------------------------------------------------------
 # Core processing
@@ -402,7 +440,11 @@ def process_conversation(messages: list[str]) -> dict:
                 learning_added = True
                 learnings_added += 1
 
-        _dedup_record(dedup_conn, content_hash, fact_type)
+        # A FAILED correction-learning must stay retryable on the next identical
+        # prompt, so only arm the 1h dedup gate for a correction once its learning
+        # actually persisted. Non-correction facts dedup as before.
+        if not (fact_type == "correction" and not learning_added):
+            _dedup_record(dedup_conn, content_hash, fact_type)
 
         extracted_details.append({
             "type": fact_type,

package/src/learning_resolver.py

@@ -173,6 +173,46 @@ def applies_overlap(left: str, right: str) -> bool:
     return False
 
 
+def normalized_key(title: str, applies_to: str = "") -> str:
+    """Stable dedup key for a learning: normalized title + sorted applies scope.
+
+    Public thin wrapper so callers (e.g. the nightly consolidation brief builder)
+    depend only on the resolver's public surface for dedup math. Two learnings
+    that differ only in casing/whitespace of the title and ordering of applies_to
+    tokens collapse to the same key.
+    """
+    title_key = _normalize_text(title)
+    scope_tokens = sorted(
+        {_normalize_applies_token(item) for item in _split_applies_to(applies_to)} - {""}
+    )
+    if scope_tokens:
+        return f"{title_key}|{','.join(scope_tokens)}"
+    return title_key
+
+
+def candidate_similarity(text_a: str, text_b: str) -> float:
+    """Similarity between two free-text snippets using the resolver's own math.
+
+    Wraps hybrid_similarity_score with the SAME thresholds used in _similarity
+    (the per-candidate resolution path) so consolidation_prep and the resolver
+    stay in lockstep. Returns 0.0 when either side is empty.
+    """
+    left = str(text_a or "").strip()
+    right = str(text_b or "").strip()
+    if not left or not right:
+        return 0.0
+    return float(
+        hybrid_similarity_score(
+            left,
+            right,
+            keyword_extractor=extract_keywords,
+            strong_semantic_threshold=0.82,
+            moderate_semantic_threshold=0.74,
+            moderate_keyword_floor=0.08,
+        )
+    )
+
+
 def _table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
     try:
         return {str(row["name"]) for row in conn.execute(f"PRAGMA table_info({table})").fetchall()}
@@ -413,7 +453,9 @@ __all__ = [
     "CANONICAL_ACTIONS",
     "applies_overlap",
     "authority_rank",
+    "candidate_similarity",
     "looks_contradictory",
     "normalize_authority",
+    "normalized_key",
     "resolve_learning_candidate",
 ]