nexo-brain 5.3.19 → 5.3.21

package/src/maintenance 2.py

@@ -0,0 +1,53 @@
+"""Placeholder module for historical `maintenance` runner.
+
+HISTORICAL CONTEXT (NEXO-AUDIT-2026-04-11 finding, Learning #194)
+==================================================================
+
+This module previously exposed `check_and_run_overdue()` and a private
+`_run_task()` dispatcher that walked the `maintenance_schedule` table and
+executed cognitive decay, synthesis, self audit, weight learning, somatic
+decay, somatic projection, drive decay and graph maintenance as needed.
+
+None of that code was ever called from anywhere. A repository-wide grep
+for `check_and_run_overdue`, `from maintenance`, `import maintenance` and
+`maintenance.check` produced zero hits during the 2026-04-11 audit. Each
+of those tasks actually runs from its own LaunchAgent via its own script
+in `src/scripts/` (e.g. `nexo-cognitive-decay.py`, `nexo-daily-self-audit
+.py`, `nexo-evolution-run.py`), not through this dispatcher. The
+`maintenance_schedule` table in SQLite is still populated by migrations
+(see `db/_schema.py::_m9_maintenance_schedule` and the drive_decay
+registration) but is effectively dead data: nothing reads it.
+
+Why the dispatcher was removed
+------------------------------
+The dead dispatcher was actively misleading: developers reading the code
+could reasonably conclude that adding a row to `maintenance_schedule`
+would cause the named task to run. It would not. That false contract
+was the root of a near-miss during the Item 9 fix of the 2026-04-11
+audit, where the first plan was to register a new `read_token_cleanup`
+task via this mechanism before discovering that the mechanism is never
+invoked. See `src/db/_reminders.py::_purge_expired_read_tokens_if_due`
+for the opportunistic-cleanup pattern that replaced that initial plan.
+
+What to do if you need scheduled maintenance
+--------------------------------------------
+Do NOT reintroduce a dispatcher in this module. Pick one of:
+
+  * Add your work to an existing LaunchAgent script under
+    `src/scripts/` that already runs on the cadence you need.
+  * Register a new personal script via `nexo_personal_script_create` and
+    let the schedule/LaunchAgent system handle it.
+  * Run the cleanup opportunistically inside the hot path, throttled
+    by wall-clock (see `_purge_expired_read_tokens_if_due`).
+
+What happens to the `maintenance_schedule` table
+-------------------------------------------------
+The table is intentionally left in place. Removing it would require a
+destructive migration for every installed user with no benefit — the
+rows do no harm, cost a few KB each, and their removal is deferred to a
+future cleanup pass when migration numbering is renegotiated.
+"""
+
+from __future__ import annotations
+
+__all__: list[str] = []

package/src/memory_backends 2.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+"""Explicit backend registry for memory expansion layers.
+
+NEXO's historical memory system is still heavily SQLite-shaped, but newer layers
+should not keep backend assumptions implicit forever. This module introduces a
+small registry/contract that expansion surfaces can use today while SQLite
+remains the default backend.
+"""
+
+from dataclasses import dataclass
+import os
+
+
+@dataclass(frozen=True)
+class MemoryBackendInfo:
+    key: str
+    label: str
+    description: str
+    supports: tuple[str, ...]
+    maturity: str = "stable"
+
+
+_REGISTRY: dict[str, MemoryBackendInfo] = {}
+
+
+def register_backend(info: MemoryBackendInfo) -> None:
+    _REGISTRY[info.key] = info
+
+
+def active_backend_key() -> str:
+    return (os.environ.get("NEXO_MEMORY_BACKEND", "sqlite") or "sqlite").strip().lower()
+
+
+def get_backend(key: str = "") -> MemoryBackendInfo:
+    selected = (key or active_backend_key()).strip().lower()
+    return _REGISTRY.get(selected, _REGISTRY["sqlite"])
+
+
+def list_backends() -> list[dict]:
+    active = active_backend_key()
+    results = []
+    for key in sorted(_REGISTRY):
+        info = _REGISTRY[key]
+        item = {
+            "key": info.key,
+            "label": info.label,
+            "description": info.description,
+            "supports": list(info.supports),
+            "maturity": info.maturity,
+            "active": info.key == active,
+        }
+        results.append(item)
+    return results
+
+
+register_backend(
+    MemoryBackendInfo(
+        key="sqlite",
+        label="SQLite + FTS5",
+        description="Local-first default backend used by NEXO runtime surfaces.",
+        supports=(
+            "cognitive_core",
+            "claims",
+            "media_memory",
+            "user_state",
+            "memory_export",
+            "auto_flush",
+        ),
+    )
+)

package/src/migrate_embeddings 2.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+Migrate cognitive.db embeddings between models.
+
+Usage:
+  python migrate_embeddings.py upgrade   # 384 → 768 (bge-small → bge-base)
+  python migrate_embeddings.py rollback  # Restore from backup
+  python migrate_embeddings.py verify    # Check current embedding dims
+"""
+
+import os
+import shutil
+import sqlite3
+import sys
+import time
+import numpy as np
+
+NEXO_HOME = os.environ.get("NEXO_HOME", os.path.expanduser("~/.nexo"))
+_data_dir = os.path.join(NEXO_HOME, "data")
+os.makedirs(_data_dir, exist_ok=True)
+DB_PATH = os.path.join(_data_dir, "cognitive.db")
+BACKUP_PATH = DB_PATH + ".bak-384dims-pre-upgrade"
+
+MODELS = {
+    "small": ("BAAI/bge-small-en-v1.5", 384),
+    "base": ("BAAI/bge-base-en-v1.5", 768),
+}
+
+
+def verify():
+    """Check current embedding dimensions in the database."""
+    conn = sqlite3.connect(DB_PATH)
+    try:
+        for table in ["stm_memories", "ltm_memories"]:
+            count = conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
+            if count == 0:
+                print(f"  {table}: {count} rows (empty)")
+                continue
+            row = conn.execute(f"SELECT embedding FROM {table} LIMIT 1").fetchone()
+            vec = np.frombuffer(row[0], dtype=np.float32)
+            print(f"  {table}: {count} rows, embedding dim = {len(vec)}")
+    finally:
+        conn.close()
+
+
+def upgrade():
+    """Re-embed all memories from bge-small (384) to bge-base (768)."""
+    from fastembed import TextEmbedding
+
+    # Verify current state
+    print("Current state:")
+    verify()
+
+    # Verify backup exists
+    if not os.path.exists(BACKUP_PATH):
+        print(f"\nCreating backup at {BACKUP_PATH}")
+        shutil.copy2(DB_PATH, BACKUP_PATH)
+    else:
+        print(f"\nBackup already exists at {BACKUP_PATH}")
+
+    # Load new model
+    model_name, expected_dim = MODELS["base"]
+    print(f"\nLoading {model_name}...")
+    model = TextEmbedding(model_name)
+
+    conn = sqlite3.connect(DB_PATH)
+    try:
+        for table in ["stm_memories", "ltm_memories"]:
+            rows = conn.execute(f"SELECT id, content FROM {table}").fetchall()
+            if not rows:
+                print(f"\n{table}: empty, skipping")
+                continue
+
+            print(f"\n{table}: re-embedding {len(rows)} memories...")
+            t0 = time.time()
+
+            # Batch embed for speed
+            contents = [r[1] for r in rows]
+            ids = [r[0] for r in rows]
+
+            embeddings = list(model.embed(contents))
+
+            for mem_id, emb in zip(ids, embeddings):
+                blob = np.array(emb, dtype=np.float32).tobytes()
+                conn.execute(f"UPDATE {table} SET embedding = ? WHERE id = ?", (blob, mem_id))
+
+            conn.commit()
+            elapsed = time.time() - t0
+            print(f"  Done: {len(rows)} memories in {elapsed:.1f}s ({elapsed/len(rows)*1000:.0f}ms/memory)")
+    finally:
+        conn.close()
+
+    print("\nAfter upgrade:")
+    verify()
+    print("\nUpgrade complete. Run 'verify' to confirm.")
+
+
+def rollback():
+    """Restore database from pre-upgrade backup."""
+    if not os.path.exists(BACKUP_PATH):
+        print(f"ERROR: Backup not found at {BACKUP_PATH}")
+        sys.exit(1)
+
+    print(f"Restoring from {BACKUP_PATH}...")
+    shutil.copy2(BACKUP_PATH, DB_PATH)
+    print("Restored. Current state:")
+    verify()
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python migrate_embeddings.py [upgrade|rollback|verify]")
+        sys.exit(1)
+
+    cmd = sys.argv[1]
+    if cmd == "upgrade":
+        upgrade()
+    elif cmd == "rollback":
+        rollback()
+    elif cmd == "verify":
+        verify()
+    else:
+        print(f"Unknown command: {cmd}")
+        sys.exit(1)

package/src/nexo_sdk 2.py

@@ -0,0 +1,103 @@
+"""Minimal Python SDK for the public NEXO mental model."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from dataclasses import dataclass
+
+
+@dataclass
+class NEXOClient:
+    """Tiny Python wrapper around `nexo call` for common public operations."""
+
+    nexo_bin: str = "nexo"
+
+    def call(self, tool: str, payload: dict | None = None) -> dict | list | str:
+        result = subprocess.run(
+            [
+                self.nexo_bin,
+                "call",
+                tool,
+                "--input",
+                json.dumps(payload or {}, ensure_ascii=False),
+                "--json-output",
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+        if result.returncode != 0:
+            raise RuntimeError((result.stderr or result.stdout or f"{tool} failed").strip())
+        text = (result.stdout or "").strip()
+        if not text:
+            return {}
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError:
+            return {"result": text}
+
+    def remember(
+        self,
+        content: str,
+        *,
+        title: str = "",
+        domain: str = "",
+        source_type: str = "note",
+        tags: str = "",
+        bypass_gate: bool = True,
+    ) -> dict | list | str:
+        return self.call(
+            "nexo_remember",
+            {
+                "content": content,
+                "title": title,
+                "domain": domain,
+                "source_type": source_type,
+                "tags": tags,
+                "bypass_gate": bypass_gate,
+            },
+        )
+
+    def recall(self, query: str, *, days: int = 30) -> dict | list | str:
+        return self.call("nexo_memory_recall", {"query": query, "days": days})
+
+    def consolidate(
+        self,
+        *,
+        max_insights: int = 12,
+        threshold: float = 0.9,
+        dry_run: bool = False,
+    ) -> dict | list | str:
+        return self.call(
+            "nexo_consolidate",
+            {
+                "max_insights": max_insights,
+                "threshold": threshold,
+                "dry_run": dry_run,
+            },
+        )
+
+    def run_workflow(
+        self,
+        sid: str,
+        goal: str,
+        *,
+        steps: list[dict] | str,
+        goal_id: str = "",
+        shared_state: dict | str | None = None,
+        owner: str = "",
+        idempotency_key: str = "",
+    ) -> dict | list | str:
+        return self.call(
+            "nexo_run_workflow",
+            {
+                "sid": sid,
+                "goal": goal,
+                "steps": steps if isinstance(steps, str) else json.dumps(steps, ensure_ascii=False),
+                "goal_id": goal_id,
+                "shared_state": shared_state if isinstance(shared_state, str) else json.dumps(shared_state or {}, ensure_ascii=False),
+                "owner": owner,
+                "idempotency_key": idempotency_key,
+            },
+        )

package/src/observability 2.py

@@ -0,0 +1,199 @@
+"""OpenTelemetry observability for NEXO Brain — Fase 5 item 2.
+
+Closes Fase 5 item 2 of NEXO-AUDIT-2026-04-11. The audit asked for
+observability with OTEL / Langfuse / Phoenix integration. This module
+is the soft-import host: it adds tracing primitives that NEXO can call
+unconditionally, but only emit real spans when:
+
+  1. The opentelemetry-api package is installed in the user's environment
+     (`pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp`).
+  2. The OTEL_EXPORTER_OTLP_ENDPOINT environment variable is set
+     (or OTEL_SERVICE_NAME is set, indicating the user already
+     bootstrapped a tracer provider externally).
+
+Otherwise every primitive is a no-op so the runtime cost on installs
+without OTEL is exactly zero (no try/except per call site, no extra
+import time on the hot path).
+
+Why this design:
+  - NEXO core does NOT require opentelemetry as a hard dependency.
+    The 10k+ active users would have an extra 30 MB in their venv with
+    no benefit unless they opted in.
+  - Users who DO want telemetry get a single env var to flip on.
+  - The shape (span name, attributes, status) follows the OpenTelemetry
+    semantic conventions for "ai.tool" so dashboards in Langfuse,
+    Arize Phoenix, Honeycomb, Jaeger, and Grafana Tempo all render
+    NEXO traces with their built-in views.
+
+Usage:
+
+    from observability import tool_span
+
+    with tool_span("nexo_heartbeat", attributes={"sid": sid}) as span:
+        result = handle_heartbeat(sid, task)
+        if span is not None:
+            span.set_attribute("nexo.heartbeat.task", task[:200])
+        return result
+
+The `with` block always works whether or not OTEL is installed; the
+span is None when telemetry is disabled.
+"""
+
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from typing import Any, Iterator
+
+
+# ── OTEL availability detection ──────────────────────────────────────────
+
+
+_otel_available: bool | None = None
+_tracer = None  # cached after first use
+
+
+def _truthy_env(var: str) -> bool:
+    return bool((os.environ.get(var) or "").strip())
+
+
+def is_otel_enabled() -> bool:
+    """Return True if OpenTelemetry is installed AND a configuration is set.
+
+    The two activation conditions are:
+      - opentelemetry-api importable
+      - One of OTEL_EXPORTER_OTLP_ENDPOINT or OTEL_SERVICE_NAME is set
+        (the latter signals an externally-bootstrapped tracer provider).
+
+    Cached after first call so the hot path is a single bool lookup.
+    """
+    global _otel_available
+    if _otel_available is not None:
+        return _otel_available
+
+    if not (_truthy_env("OTEL_EXPORTER_OTLP_ENDPOINT") or _truthy_env("OTEL_SERVICE_NAME")):
+        _otel_available = False
+        return False
+
+    try:
+        import opentelemetry  # noqa: F401
+        from opentelemetry import trace  # noqa: F401
+    except ImportError:
+        _otel_available = False
+        return False
+
+    _otel_available = True
+    return True
+
+
+def get_tracer():
+    """Return a cached opentelemetry.trace.Tracer or None when OTEL is off.
+
+    Lazy: only constructs the tracer the first time it is needed so
+    installs without OTEL never pay any import cost.
+    """
+    global _tracer
+    if _tracer is not None:
+        return _tracer
+    if not is_otel_enabled():
+        return None
+    try:
+        from opentelemetry import trace
+        _tracer = trace.get_tracer("nexo-brain")
+        return _tracer
+    except Exception:
+        return None
+
+
+# ── span context manager ─────────────────────────────────────────────────
+
+
+@contextmanager
+def tool_span(
+    name: str,
+    *,
+    attributes: dict[str, Any] | None = None,
+) -> Iterator[Any]:
+    """Context manager that emits an OTEL span when telemetry is enabled.
+
+    The span name follows the OTEL semantic convention prefix "ai.tool."
+    so dashboards that already group by ai.tool.* automatically pick
+    NEXO traces up.
+
+    On success: status = OK.
+    On exception: status = ERROR with the exception message recorded as
+    an attribute, and the exception is re-raised so callers see it.
+
+    When telemetry is disabled, the context manager yields None and
+    does nothing else — the cost is one is_otel_enabled() bool lookup
+    plus the empty `with` block, which the Python compiler optimizes.
+
+    Args:
+        name: short tool name. The full span name becomes "ai.tool.<name>".
+        attributes: optional dict of OTEL attributes to set on the span.
+    """
+    if not is_otel_enabled():
+        yield None
+        return
+
+    tracer = get_tracer()
+    if tracer is None:
+        yield None
+        return
+
+    try:
+        from opentelemetry import trace as _trace
+        span_name = f"ai.tool.{name}" if not name.startswith("ai.tool.") else name
+        with tracer.start_as_current_span(span_name) as span:
+            try:
+                if attributes:
+                    for key, value in attributes.items():
+                        try:
+                            span.set_attribute(key, value)
+                        except Exception:
+                            # Some values (e.g. dicts) are not OTEL-compatible.
+                            try:
+                                span.set_attribute(key, str(value)[:1000])
+                            except Exception:
+                                pass
+                yield span
+                span.set_status(_trace.Status(_trace.StatusCode.OK))
+            except Exception as exc:
+                try:
+                    span.record_exception(exc)
+                    span.set_status(
+                        _trace.Status(_trace.StatusCode.ERROR, str(exc)[:300])
+                    )
+                except Exception:
+                    pass
+                raise
+    except Exception:
+        # If anything goes wrong with the OTEL machinery itself, fall
+        # back to a no-op so the caller never sees a telemetry-induced
+        # exception. The original work still runs.
+        yield None
+
+
+def record_tool_attributes(span: Any, attributes: dict[str, Any]) -> None:
+    """Set OTEL attributes on a span if it is non-None and OTEL is enabled.
+
+    Convenience helper so callers do not need to write the `if span is
+    not None` guard at every call site.
+    """
+    if span is None:
+        return
+    for key, value in (attributes or {}).items():
+        try:
+            span.set_attribute(key, value)
+        except Exception:
+            try:
+                span.set_attribute(key, str(value)[:1000])
+            except Exception:
+                pass
+
+
+def _reset_for_tests() -> None:
+    """Test-only helper to reset the cached availability + tracer."""
+    global _otel_available, _tracer
+    _otel_available = None
+    _tracer = None