nexo-brain 2.2.0 → 2.3.1

package/src/db/_tasks 2.py

@@ -1,91 +0,0 @@
-"""NEXO DB — Tasks module."""
-from db._core import get_db, now_epoch
-
-# ── Task History & Frequencies ─────────────────────────────────────
-
-def log_task(task_num: str, task_name: str, notes: str = '', reasoning: str = '') -> dict:
-    """Log a task execution with optional reasoning."""
-    conn = get_db()
-    now = now_epoch()
-    cursor = conn.execute(
-        "INSERT INTO task_history (task_num, task_name, executed_at, notes, reasoning) "
-        "VALUES (?, ?, ?, ?, ?)",
-        (task_num, task_name, now, notes, reasoning)
-    )
-    conn.commit()
-    row = conn.execute(
-        "SELECT * FROM task_history WHERE id = ?", (cursor.lastrowid,)
-    ).fetchone()
-    return dict(row)
-
-
-def list_task_history(task_num: str = None, days: int = 30) -> list[dict]:
-    """List task execution history, optionally filtered by task_num."""
-    conn = get_db()
-    cutoff = now_epoch() - (days * 86400)
-    if task_num:
-        rows = conn.execute(
-            "SELECT * FROM task_history WHERE task_num = ? AND executed_at >= ? "
-            "ORDER BY executed_at DESC",
-            (task_num, cutoff)
-        ).fetchall()
-    else:
-        rows = conn.execute(
-            "SELECT * FROM task_history WHERE executed_at >= ? "
-            "ORDER BY executed_at DESC",
-            (cutoff,)
-        ).fetchall()
-    return [dict(r) for r in rows]
-
-
-def set_task_frequency(task_num: str, task_name: str,
-                       frequency_days: int, description: str = '') -> dict:
-    """Set or update the expected frequency for a task."""
-    conn = get_db()
-    conn.execute(
-        "INSERT OR REPLACE INTO task_frequencies (task_num, task_name, frequency_days, description) "
-        "VALUES (?, ?, ?, ?)",
-        (task_num, task_name, frequency_days, description)
-    )
-    conn.commit()
-    row = conn.execute(
-        "SELECT * FROM task_frequencies WHERE task_num = ?", (task_num,)
-    ).fetchone()
-    return dict(row)
-
-
-def get_overdue_tasks() -> list[dict]:
-    """Get tasks where last execution exceeds the configured frequency."""
-    conn = get_db()
-    freqs = conn.execute("SELECT * FROM task_frequencies").fetchall()
-    now = now_epoch()
-    overdue = []
-    for f in freqs:
-        last = conn.execute(
-            "SELECT MAX(executed_at) as last_exec FROM task_history WHERE task_num = ?",
-            (f["task_num"],)
-        ).fetchone()
-        last_exec = last["last_exec"] if last and last["last_exec"] else None
-        threshold = f["frequency_days"] * 86400
-        if last_exec is None or (now - last_exec) > threshold:
-            days_ago = round((now - last_exec) / 86400, 1) if last_exec else None
-            overdue.append({
-                "task_num": f["task_num"],
-                "task_name": f["task_name"],
-                "frequency_days": f["frequency_days"],
-                "last_executed": last_exec,
-                "days_since_last": days_ago,
-                "description": f["description"]
-            })
-    return overdue
-
-
-def get_task_frequencies() -> list[dict]:
-    """Get all configured task frequencies."""
-    conn = get_db()
-    rows = conn.execute(
-        "SELECT * FROM task_frequencies ORDER BY task_num ASC"
-    ).fetchall()
-    return [dict(r) for r in rows]
-
-

package/src/evolution_cycle 2.py

@@ -1,266 +0,0 @@
-"""NEXO Evolution Cycle — Self-improvement via Opus API.
-
-Runs weekly after DMN. Analyzes patterns, proposes improvements.
-v1: observe-only (all proposals logged as 'proposed' for the user to review).
-v1.1 (future): sandbox execution of auto-approved changes.
-"""
-
-import json
-import os
-import shutil
-import subprocess
-import sqlite3
-import time
-from datetime import datetime, date, timedelta
-from pathlib import Path
-
-NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
-NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(NEXO_HOME)))
-NEXO_DB = NEXO_HOME / "data" / "nexo.db"
-SANDBOX_DIR = NEXO_HOME / "sandbox" / "workspace"
-SNAPSHOTS_DIR = NEXO_HOME / "snapshots"
-RESTORE_LOG = NEXO_HOME / "logs" / "snapshot-restores.log"
-
-# Evolution config: brain/ (canonical) > cortex/ (legacy) > NEXO_CODE (dev)
-def _resolve_evolution_file(name: str) -> Path:
-    for candidate in [NEXO_HOME / "brain" / name, NEXO_HOME / "cortex" / name, NEXO_CODE / name]:
-        if candidate.exists():
-            return candidate
-    return NEXO_HOME / "brain" / name  # default canonical path
-
-OBJECTIVE_FILE = _resolve_evolution_file("evolution-objective.json")
-PROMPT_FILE = _resolve_evolution_file("evolution-prompt.md")
-
-MAX_SNAPSHOTS = 8
-
-
-def load_objective() -> dict:
-    if OBJECTIVE_FILE.exists():
-        return json.loads(OBJECTIVE_FILE.read_text())
-    return {}
-
-
-def save_objective(obj: dict):
-    OBJECTIVE_FILE.write_text(json.dumps(obj, indent=2, ensure_ascii=False))
-
-
-def get_week_data(db_path: str) -> dict:
-    """Gather last 7 days of learnings, decisions, changes, diaries."""
-    conn = sqlite3.connect(db_path, timeout=10)
-    conn.row_factory = sqlite3.Row
-    cutoff_epoch = time.time() - 7 * 86400
-    cutoff_date = (date.today() - timedelta(days=7)).isoformat()
-
-    data = {}
-
-    rows = conn.execute(
-        "SELECT category, title, content FROM learnings WHERE created_at > ? ORDER BY created_at DESC LIMIT 50",
-        (cutoff_epoch,)
-    ).fetchall()
-    data["learnings"] = [dict(r) for r in rows]
-
-    rows = conn.execute(
-        "SELECT domain, decision, alternatives, based_on, confidence, outcome FROM decisions "
-        "WHERE created_at > ? ORDER BY created_at DESC LIMIT 20",
-        (cutoff_date,)
-    ).fetchall()
-    data["decisions"] = [dict(r) for r in rows]
-
-    rows = conn.execute(
-        "SELECT files, what_changed, why, affects, risks FROM change_log "
-        "WHERE created_at > ? ORDER BY created_at DESC LIMIT 30",
-        (cutoff_date,)
-    ).fetchall()
-    data["changes"] = [dict(r) for r in rows]
-
-    rows = conn.execute(
-        "SELECT summary, decisions as diary_decisions, pending, mental_state, domain, user_signals "
-        "FROM session_diary WHERE created_at > ? ORDER BY created_at DESC LIMIT 20",
-        (cutoff_date,)
-    ).fetchall()
-    data["diaries"] = [dict(r) for r in rows]
-
-    rows = conn.execute(
-        "SELECT * FROM evolution_log ORDER BY id DESC LIMIT 20"
-    ).fetchall()
-    data["evolution_history"] = [dict(r) for r in rows]
-
-    rows = conn.execute(
-        "SELECT dimension, score, delta, measured_at FROM evolution_metrics "
-        "WHERE id IN (SELECT MAX(id) FROM evolution_metrics GROUP BY dimension)"
-    ).fetchall()
-    data["current_metrics"] = {r["dimension"]: dict(r) for r in rows}
-
-    conn.close()
-    return data
-
-
-def create_snapshot(files_to_backup: list) -> str:
-    """Create a snapshot of specific files before modification."""
-    ts = datetime.now().strftime("%Y-%m-%dT%H:%M")
-    snap_dir = SNAPSHOTS_DIR / ts
-    files_dir = snap_dir / "files"
-
-    manifest = {
-        "created_at": datetime.now().isoformat(),
-        "files": [],
-        "reason": "evolution_cycle"
-    }
-
-    for filepath in files_to_backup:
-        fp = Path(filepath).expanduser()
-        if fp.exists():
-            rel = str(fp).replace(str(Path.home()) + "/", "")
-            dest = files_dir / rel
-            dest.parent.mkdir(parents=True, exist_ok=True)
-            if os.path.abspath(str(fp)) == os.path.abspath(str(dest)):
-                continue  # Skip: source and destination are the same file
-            shutil.copy2(fp, dest)
-            manifest["files"].append(rel)
-
-    snap_dir.mkdir(parents=True, exist_ok=True)
-    (snap_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))
-
-    latest = SNAPSHOTS_DIR / "latest"
-    if latest.is_symlink():
-        latest.unlink()
-    latest.symlink_to(snap_dir)
-
-    _cleanup_snapshots()
-    return str(snap_dir)
-
-
-def _cleanup_snapshots():
-    """Remove old snapshots, keeping MAX_SNAPSHOTS most recent + golden."""
-    if not SNAPSHOTS_DIR.exists():
-        return
-    snaps = sorted(
-        [d for d in SNAPSHOTS_DIR.iterdir()
-         if d.is_dir() and d.name not in ("latest", "golden")],
-        key=lambda d: d.stat().st_mtime,
-        reverse=True
-    )
-    for old in snaps[MAX_SNAPSHOTS:]:
-        shutil.rmtree(old)
-
-
-def dry_run_restore_test() -> bool:
-    """Test that snapshot+restore works before making real changes."""
-    test_file = SANDBOX_DIR / "restore-test.txt"
-    test_file.parent.mkdir(parents=True, exist_ok=True)
-    test_file.write_text("original_content")
-
-    snap_dir = create_snapshot([str(test_file)])
-
-    test_file.write_text("modified_content")
-
-    # Find restore script: NEXO_CODE/scripts/ first, then NEXO_HOME/scripts/
-    _nexo_code = Path(os.environ.get("NEXO_CODE", ""))
-    restore_script = None
-    for candidate in [_nexo_code / "scripts" / "nexo-snapshot-restore.sh",
-                      NEXO_HOME / "scripts" / "nexo-snapshot-restore.sh"]:
-        if candidate.exists():
-            restore_script = candidate
-            break
-    if not restore_script:
-        test_file.unlink(missing_ok=True)
-        return False  # No restore script available
-
-    try:
-        subprocess.run(
-            [str(restore_script), snap_dir],
-            capture_output=True, timeout=10, check=True
-        )
-        content = test_file.read_text()
-        test_file.unlink(missing_ok=True)
-        # Clean up test snapshot
-        snap_path = Path(snap_dir)
-        if snap_path.exists():
-            shutil.rmtree(snap_path)
-        return content == "original_content"
-    except Exception:
-        test_file.unlink(missing_ok=True)
-        return False
-
-
-def build_evolution_prompt(week_data: dict, objective: dict) -> str:
-    """Build a SHORT prompt — CLI investigates on its own using tools."""
-
-    # Summary stats only — CLI will dig deeper with tools
-    stats = {
-        "learnings_this_week": len(week_data.get("learnings", [])),
-        "decisions_this_week": len(week_data.get("decisions", [])),
-        "changes_this_week": len(week_data.get("changes", [])),
-        "diaries_this_week": len(week_data.get("diaries", [])),
-        "evolution_history": len(week_data.get("evolution_history", [])),
-        "current_scores": {dim: m["score"] for dim, m in week_data.get("current_metrics", {}).items()},
-    }
-
-    mode = objective.get("evolution_mode", "auto")
-    total = objective.get("total_evolutions", 0)
-    max_auto = max_auto_changes(total)
-
-    prompt = f"""You are NEXO Evolution — the weekly self-improvement cycle.
-
-YOUR JOB: Analyze the past week and propose concrete improvements to NEXO's codebase.
-
-WEEK SUMMARY:
-- {stats['learnings_this_week']} new learnings
-- {stats['decisions_this_week']} decisions made
-- {stats['changes_this_week']} code changes deployed
-- {stats['diaries_this_week']} session diaries
-- {stats['evolution_history']} past evolution proposals
-- Current scores: {json.dumps(stats['current_scores'])}
-
-MODE: {mode} ({"proposals only, owner reviews" if mode == "review" else f"max {max_auto} auto-applied changes"})
-CYCLE: #{total + 1}
-
-INVESTIGATE using these tools:
-1. Bash: sqlite3 {NEXO_DB} "SELECT category, title FROM learnings WHERE created_at > {time.time() - 7*86400} ORDER BY created_at DESC LIMIT 30"
-2. Bash: sqlite3 {NEXO_DB} "SELECT area, COUNT(*) as cnt FROM error_repetitions GROUP BY area ORDER BY cnt DESC LIMIT 10"
-3. Read ~/.nexo/coordination/daily-synthesis.md — today's context
-4. Read ~/.nexo/coordination/postmortem-daily.md — self-critique patterns
-5. Read ~/.nexo/logs/self-audit-summary.json — system health
-6. Glob ~/.nexo/scripts/*.py — existing scripts
-7. Glob ~/.nexo/plugins/*.py — existing plugins
-
-LOOK FOR:
-- Repeated errors that guard isn't preventing
-- Scripts or processes that are failing or underperforming
-- Missing functionality that session diaries keep asking for
-- Redundant code or config that could be simplified
-- Patterns in self-critique that suggest systemic issues
-
-SAFETY:
-- Safe zones for auto changes: ~/.nexo/scripts/, ~/.nexo/plugins/, ~/.nexo/brain/
-- IMMUTABLE files (never touch): db.py, server.py, plugin_loader.py, cognitive.py, CLAUDE.md
-- Every change needs: what file, what to change, why, risk, how to verify
-
-OUTPUT FORMAT (JSON):
-{{
-  "analysis": "one paragraph summary of what you found",
-  "patterns": [{{"type": "...", "description": "...", "frequency": "..."}}],
-  "proposals": [
-    {{
-      "classification": "auto" or "propose",
-      "dimension": "reliability|proactivity|efficiency|safety|learning",
-      "action": "what to do",
-      "reasoning": "why",
-      "scope": "local",
-      "changes": [{{"file": "path", "operation": "create|replace|append", "search": "text to find", "content": "new text"}}]
-    }}
-  ]
-}}
-
-Max 3 proposals. Quality over quantity. If nothing needs improving, say so."""
-
-    return prompt
-
-
-def max_auto_changes(total_evolutions: int) -> int:
-    """Progressive trust: 1 for first 4 cycles, 2 for next 4, then 3."""
-    if total_evolutions < 4:
-        return 1
-    elif total_evolutions < 8:
-        return 2
-    return 3

package/src/hnsw_index 2.py

@@ -1,254 +0,0 @@
-"""NEXO HNSW Vector Index — Optional acceleration for cognitive search.
-
-When memory count exceeds THRESHOLD (default 10_000), this module builds and
-maintains an HNSW index for approximate nearest neighbor search. Falls back
-gracefully to brute-force when hnswlib is not available or index is cold.
-
-Usage in cognitive.search():
-    from hnsw_index import hnsw_search
-    candidates = hnsw_search(query_vec, store="stm", top_k=50)
-    # candidates is a list of (memory_id, distance) or None if not available
-"""
-
-import os
-import sqlite3
-import threading
-import numpy as np
-from pathlib import Path
-from typing import Optional
-
-try:
-    import hnswlib
-    HNSWLIB_AVAILABLE = True
-except ImportError:
-    HNSWLIB_AVAILABLE = False
-
-# When to activate HNSW (below this, brute force is fine)
-ACTIVATION_THRESHOLD = int(os.environ.get("NEXO_HNSW_THRESHOLD", "10000"))
-
-# Index params
-EMBEDDING_DIM = 768
-EF_CONSTRUCTION = 200  # Higher = better recall during build, slower
-M = 16                 # Connections per node (16 is good for 768-dim)
-EF_SEARCH = 50         # Higher = better recall during search
-
-# Index file paths
-_INDEX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "hnsw_indices")
-
-# In-memory indices (one per store)
-_indices: dict = {}  # {"stm": hnswlib.Index, "ltm": hnswlib.Index}
-_index_lock = threading.Lock()
-_id_maps: dict = {}  # {"stm": {internal_id: db_id}, "ltm": {internal_id: db_id}}
-
-
-def is_available() -> bool:
-    """Check if HNSW is available and should be used."""
-    return HNSWLIB_AVAILABLE
-
-
-def _index_path(store: str) -> str:
-    return os.path.join(_INDEX_DIR, f"{store}.bin")
-
-
-def _id_map_path(store: str) -> str:
-    return os.path.join(_INDEX_DIR, f"{store}_ids.npy")
-
-
-def should_activate(store: str = "both") -> bool:
-    """Check if memory count exceeds threshold, making HNSW worthwhile."""
-    if not HNSWLIB_AVAILABLE:
-        return False
-    try:
-        import cognitive
-        db = cognitive._get_db()
-        total = 0
-        if store in ("both", "stm"):
-            total += db.execute("SELECT COUNT(*) FROM stm_memories WHERE promoted_to_ltm = 0").fetchone()[0]
-        if store in ("both", "ltm"):
-            total += db.execute("SELECT COUNT(*) FROM ltm_memories WHERE is_dormant = 0").fetchone()[0]
-        return total >= ACTIVATION_THRESHOLD
-    except Exception:
-        return False
-
-
-def build_index(store: str) -> dict:
-    """Build HNSW index from all active memories in the given store.
-
-    Args:
-        store: "stm" or "ltm"
-
-    Returns:
-        {"count": N, "store": store, "status": "built"} or error dict
-    """
-    if not HNSWLIB_AVAILABLE:
-        return {"error": "hnswlib not installed"}
-
-    try:
-        import cognitive
-        db = cognitive._get_db()
-    except Exception as e:
-        return {"error": str(e)}
-
-    table = "stm_memories" if store == "stm" else "ltm_memories"
-    where = "promoted_to_ltm = 0" if store == "stm" else "is_dormant = 0"
-
-    rows = db.execute(f"SELECT id, embedding FROM {table} WHERE {where}").fetchall()
-    if not rows:
-        return {"count": 0, "store": store, "status": "empty"}
-
-    count = len(rows)
-    index = hnswlib.Index(space='cosine', dim=EMBEDDING_DIM)
-    index.init_index(max_elements=max(count * 2, 1000), ef_construction=EF_CONSTRUCTION, M=M)
-    index.set_ef(EF_SEARCH)
-
-    id_map = {}
-    vectors = []
-    internal_ids = []
-
-    for i, row in enumerate(rows):
-        vec = np.frombuffer(row["embedding"], dtype=np.float32)
-        if len(vec) != EMBEDDING_DIM:
-            continue
-        vectors.append(vec)
-        internal_ids.append(i)
-        id_map[i] = row["id"]
-
-    if not vectors:
-        return {"count": 0, "store": store, "status": "no_valid_vectors"}
-
-    data = np.array(vectors, dtype=np.float32)
-    ids = np.array(internal_ids, dtype=np.int64)
-    index.add_items(data, ids)
-
-    # Save to disk
-    Path(_INDEX_DIR).mkdir(exist_ok=True)
-    index.save_index(_index_path(store))
-    np.save(_id_map_path(store), id_map)
-
-    with _index_lock:
-        _indices[store] = index
-        _id_maps[store] = id_map
-
-    return {"count": count, "store": store, "status": "built"}
-
-
-def load_index(store: str) -> bool:
-    """Load a previously built index from disk."""
-    if not HNSWLIB_AVAILABLE:
-        return False
-
-    idx_path = _index_path(store)
-    map_path = _id_map_path(store) + ".npy" if not _id_map_path(store).endswith(".npy") else _id_map_path(store)
-
-    if not os.path.exists(idx_path):
-        return False
-
-    try:
-        index = hnswlib.Index(space='cosine', dim=EMBEDDING_DIM)
-        index.load_index(idx_path)
-        index.set_ef(EF_SEARCH)
-
-        id_map = np.load(map_path, allow_pickle=True).item()
-
-        with _index_lock:
-            _indices[store] = index
-            _id_maps[store] = id_map
-        return True
-    except Exception:
-        return False
-
-
-def search(query_vec: np.ndarray, store: str = "stm", top_k: int = 50) -> Optional[list[tuple[int, float]]]:
-    """Search the HNSW index for approximate nearest neighbors.
-
-    Args:
-        query_vec: Query embedding (768-dim float32)
-        store: "stm" or "ltm"
-        top_k: Number of results
-
-    Returns:
-        List of (db_memory_id, cosine_distance) or None if index not available.
-        Note: hnswlib with cosine space returns 1 - cosine_similarity as distance.
-    """
-    with _index_lock:
-        index = _indices.get(store)
-        id_map = _id_maps.get(store)
-
-    if index is None or id_map is None:
-        # Try loading from disk
-        if load_index(store):
-            with _index_lock:
-                index = _indices.get(store)
-                id_map = _id_maps.get(store)
-        if index is None:
-            return None
-
-    try:
-        query = query_vec.reshape(1, -1).astype(np.float32)
-        labels, distances = index.knn_query(query, k=min(top_k, index.get_current_count()))
-        results = []
-        for label, dist in zip(labels[0], distances[0]):
-            db_id = id_map.get(int(label))
-            if db_id is not None:
-                # Convert cosine distance to similarity: sim = 1 - dist
-                results.append((db_id, float(1.0 - dist)))
-        return results
-    except Exception:
-        return None
-
-
-def add_item(store: str, db_id: int, embedding: np.ndarray) -> bool:
-    """Incrementally add a single item to the index (for new ingestions)."""
-    with _index_lock:
-        index = _indices.get(store)
-        id_map = _id_maps.get(store)
-
-    if index is None or id_map is None:
-        return False
-
-    try:
-        internal_id = max(id_map.keys()) + 1 if id_map else 0
-        # Resize if needed
-        if index.get_current_count() >= index.get_max_elements() - 1:
-            index.resize_index(index.get_max_elements() * 2)
-
-        vec = embedding.reshape(1, -1).astype(np.float32)
-        index.add_items(vec, np.array([internal_id], dtype=np.int64))
-
-        with _index_lock:
-            id_map[internal_id] = db_id
-        return True
-    except Exception:
-        return False
-
-
-def invalidate(store: str = "both"):
-    """Remove indices from memory (forces rebuild on next use)."""
-    with _index_lock:
-        if store in ("both", "stm"):
-            _indices.pop("stm", None)
-            _id_maps.pop("stm", None)
-        if store in ("both", "ltm"):
-            _indices.pop("ltm", None)
-            _id_maps.pop("ltm", None)
-
-
-def stats() -> dict:
-    """Return HNSW index statistics."""
-    result = {
-        "hnswlib_available": HNSWLIB_AVAILABLE,
-        "activation_threshold": ACTIVATION_THRESHOLD,
-        "indices": {},
-    }
-    with _index_lock:
-        for store in ("stm", "ltm"):
-            idx = _indices.get(store)
-            if idx:
-                result["indices"][store] = {
-                    "count": idx.get_current_count(),
-                    "max_elements": idx.get_max_elements(),
-                    "ef_search": EF_SEARCH,
-                }
-            else:
-                result["indices"][store] = {"status": "not_loaded"}
-    return result