nexo-brain 2.3.0 → 2.3.2

package/src/scripts/nexo-sleep 2.py

@@ -1,592 +0,0 @@
-#!/usr/bin/env python3
-"""
-NEXO Sleep System v2 — The brain dreams.
-
-Before: 834 lines with word-overlap "intelligence" for learning consolidation.
-Now: Stage A (mechanical cleanup) stays pure Python. Stage B (dreaming) uses
-Claude CLI (opus) to understand, deduplicate, and prune with real intelligence.
-
-Triggered hourly via LaunchAgent. Runs ONCE per day, first time Mac is awake.
-If interrupted (power loss, crash), resumes on next trigger.
-
-Stage A — Housekeeping (Python pure):
-  Delete old logs, rotate files, trim JSON. No intelligence needed.
-
-Stage B — Dreaming (Claude CLI opus):
-  Review learnings for duplicates and contradictions with UNDERSTANDING.
-  Prune MEMORY.md if over limit. Clean preferences. Compress old observations.
-  One CLI call that does what 500 lines of word-overlap couldn't.
-"""
-
-import fcntl
-import json
-import os
-import re
-import shutil
-import sqlite3
-import subprocess
-import sys
-from datetime import datetime, date, timedelta
-from pathlib import Path
-
-NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
-
-# ─── Paths ────────────────────────────────────────────────────────────────────
-CLAUDE_DIR = NEXO_HOME
-BRAIN_DIR = CLAUDE_DIR / "brain"
-COORD_DIR = CLAUDE_DIR / "coordination"
-MEMORY_DIR = CLAUDE_DIR / "memory"
-DAEMON_LOGS_DIR = CLAUDE_DIR / "daemon" / "logs"
-
-DAILY_SUMMARIES_DIR = BRAIN_DIR / "daily_summaries"
-SESSION_ARCHIVE_DIR = BRAIN_DIR / "session_archive"
-COMPRESSED_MEMORIES_DIR = BRAIN_DIR / "compressed_memories"
-
-HEARTBEAT_LOG = COORD_DIR / "heartbeat-log.json"
-REFLECTION_LOG = COORD_DIR / "reflection-log.json"
-SLEEP_LOG = COORD_DIR / "sleep-log.json"
-
-MEMORY_MD = NEXO_HOME / "memory" / "MEMORY.md"
-NEXO_DB = NEXO_HOME / "data" / "nexo.db"
-CLAUDE_MEM_DB = Path.home() / ".claude-mem" / "claude-mem.db"
-CLAUDE_CLI = Path.home() / ".local" / "bin" / "claude"
-
-LAST_RUN_FILE = COORD_DIR / "sleep-last-run"
-LOCK_FILE = COORD_DIR / "sleep.lock"
-PROCESS_LOCK = COORD_DIR / "sleep-process.lock"
-
-TODAY = date.today()
-NOW = datetime.now()
-TIMESTAMP = NOW.strftime("%Y-%m-%d %H:%M")
-
-
-# ─── Run-once & resume logic (unchanged from v1) ──────────────────────────────
-
-def already_ran_today() -> bool:
-    if not LAST_RUN_FILE.exists():
-        return False
-    try:
-        return LAST_RUN_FILE.read_text().strip() == str(TODAY)
-    except Exception:
-        return False
-
-
-def was_interrupted() -> bool:
-    if not LOCK_FILE.exists():
-        return False
-    try:
-        lock_data = json.loads(LOCK_FILE.read_text())
-        if lock_data.get("date") != str(TODAY):
-            LOCK_FILE.unlink()
-            return False
-        lock_pid = lock_data.get("pid")
-        if lock_pid:
-            try:
-                os.kill(lock_pid, 0)
-                log(f"Another instance running (PID {lock_pid}). Exiting.")
-                return False
-            except ProcessLookupError:
-                log(f"Interrupted run (phase: {lock_data.get('phase', '?')}). Resuming.")
-                return True
-            except PermissionError:
-                return False
-        LOCK_FILE.unlink()
-        return False
-    except Exception:
-        LOCK_FILE.unlink(missing_ok=True)
-        return False
-
-
-def get_interrupted_phase() -> str:
-    try:
-        return json.loads(LOCK_FILE.read_text()).get("phase", "stage_a")
-    except Exception:
-        return "stage_a"
-
-
-def set_lock(phase: str):
-    save_json(LOCK_FILE, {"date": str(TODAY), "phase": phase, "started": TIMESTAMP, "pid": os.getpid()})
-
-
-def mark_complete():
-    LAST_RUN_FILE.write_text(str(TODAY))
-    LOCK_FILE.unlink(missing_ok=True)
-
-
-# ─── Helpers ──────────────────────────────────────────────────────────────────
-
-def log(msg: str):
-    ts = datetime.now().strftime("%Y-%m-%d %H:%M")
-    print(f"[{ts}] {msg}")
-
-
-def load_json(path: Path, default=None):
-    if not path.exists():
-        return default if default is not None else {}
-    try:
-        return json.loads(path.read_text())
-    except Exception:
-        return default if default is not None else {}
-
-
-def save_json(path: Path, data):
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
-
-
-def parse_date_from_stem(stem: str):
-    m = re.search(r'(\d{4}-\d{2}-\d{2})', stem)
-    if m:
-        try:
-            return date.fromisoformat(m.group(1))
-        except ValueError:
-            return None
-    return None
-
-
-def append_sleep_log(entry: dict):
-    entries = load_json(SLEEP_LOG, [])
-    if not isinstance(entries, list):
-        entries = []
-    entries.append(entry)
-    if len(entries) > 90:
-        entries = entries[-90:]
-    save_json(SLEEP_LOG, entries)
-
-
-# ─── Stage A: Mechanical cleanup (UNCHANGED from v1) ─────────────────────────
-
-def stage_a_cleanup() -> dict:
-    """Pure Python cleanup. No LLM calls."""
-    stats = {
-        "a1_daily_summaries_deleted": 0,
-        "a2_session_archives_deleted": 0,
-        "a3_logs_rotated": 0,
-        "a4_compressed_memories_deleted": 0,
-        "a5_heartbeat_trimmed": False,
-        "a6_reflection_trimmed": False,
-        "a7_daemon_logs_deleted": 0,
-    }
-
-    # A1: Delete daily_summaries/*.md >90 days
-    cutoff_90 = TODAY - timedelta(days=90)
-    if DAILY_SUMMARIES_DIR.exists():
-        for f in DAILY_SUMMARIES_DIR.glob("*.md"):
-            d = parse_date_from_stem(f.stem)
-            if d and d < cutoff_90:
-                try:
-                    f.unlink()
-                    stats["a1_daily_summaries_deleted"] += 1
-                except Exception:
-                    pass
-
-    # A2: Delete session_archive/*.jsonl >30 days
-    cutoff_30 = TODAY - timedelta(days=30)
-    if SESSION_ARCHIVE_DIR.exists():
-        for f in SESSION_ARCHIVE_DIR.glob("*.jsonl"):
-            d = parse_date_from_stem(f.stem)
-            if d and d < cutoff_30:
-                try:
-                    f.unlink()
-                    stats["a2_session_archives_deleted"] += 1
-                except Exception:
-                    pass
-
-    # A3: Rotate coordination/*-stdout.log if >5MB
-    if COORD_DIR.exists():
-        for f in COORD_DIR.glob("*-stdout.log"):
-            try:
-                if f.stat().st_size > 5 * 1024 * 1024:
-                    lines = f.read_text().splitlines()
-                    keep = lines[-500:]
-                    f.write_text("\n".join(keep) + "\n")
-                    stats["a3_logs_rotated"] += 1
-            except Exception:
-                pass
-
-    # A4: Delete compressed_memories/week_*.md >180 days
-    cutoff_180 = TODAY - timedelta(days=180)
-    if COMPRESSED_MEMORIES_DIR.exists():
-        for f in COMPRESSED_MEMORIES_DIR.glob("week_*.md"):
-            d = parse_date_from_stem(f.stem)
-            if d and d < cutoff_180:
-                try:
-                    f.unlink()
-                    stats["a4_compressed_memories_deleted"] += 1
-                except Exception:
-                    pass
-
-    # A5: Trim heartbeat-log.json to 200 entries
-    if HEARTBEAT_LOG.exists():
-        try:
-            data = load_json(HEARTBEAT_LOG, [])
-            if isinstance(data, list) and len(data) > 200:
-                save_json(HEARTBEAT_LOG, data[-200:])
-                stats["a5_heartbeat_trimmed"] = True
-        except Exception:
-            pass
-
-    # A6: Trim reflection-log.json to 60 entries
-    if REFLECTION_LOG.exists():
-        try:
-            data = load_json(REFLECTION_LOG, [])
-            if isinstance(data, list) and len(data) > 60:
-                save_json(REFLECTION_LOG, data[-60:])
-                stats["a6_reflection_trimmed"] = True
-        except Exception:
-            pass
-
-    # A7: Delete daemon/logs/ dirs >14 days
-    cutoff_14 = TODAY - timedelta(days=14)
-    if DAEMON_LOGS_DIR.exists():
-        for d_path in sorted(DAEMON_LOGS_DIR.iterdir()):
-            if not d_path.is_dir():
-                continue
-            d = parse_date_from_stem(d_path.name)
-            if d and d < cutoff_14:
-                try:
-                    shutil.rmtree(d_path)
-                    stats["a7_daemon_logs_deleted"] += 1
-                except Exception:
-                    pass
-
-    # A8: Delete cortex/logs/*.log >7 days, truncate launchd >5MB
-    cutoff_7 = TODAY - timedelta(days=7)
-    cortex_logs = NEXO_HOME / "cortex" / "logs"
-    if cortex_logs.exists():
-        for f in cortex_logs.glob("*.log"):
-            if f.name.startswith("launchd-"):
-                try:
-                    if f.stat().st_size > 5 * 1024 * 1024:
-                        lines = f.read_text().splitlines()
-                        f.write_text("\n".join(lines[-500:]) + "\n")
-                        stats["a3_logs_rotated"] += 1
-                except Exception:
-                    pass
-                continue
-            d = parse_date_from_stem(f.stem)
-            if d and d < cutoff_7:
-                try:
-                    f.unlink()
-                except Exception:
-                    pass
-
-    return stats
-
-
-# ─── Stage B: Dreaming (Claude CLI) ─────────────────────────────────────────
-
-def collect_brain_state() -> dict:
-    """Collect all data the CLI needs to dream."""
-    state = {"learnings": [], "preferences": [], "memory_md_lines": 0,
-             "claude_mem_old": 0, "feedback_count": 0}
-
-    if NEXO_DB.exists():
-        try:
-            conn = sqlite3.connect(str(NEXO_DB))
-            conn.row_factory = sqlite3.Row
-
-            # Learnings
-            rows = conn.execute(
-                "SELECT id, title, content, category, created_at FROM learnings "
-                "WHERE status='active' ORDER BY id"
-            ).fetchall()
-            state["learnings"] = [dict(r) for r in rows]
-
-            # Preferences
-            rows = conn.execute("SELECT key, value, category, updated_at FROM preferences").fetchall()
-            state["preferences"] = [dict(r) for r in rows]
-
-            conn.close()
-        except Exception as e:
-            log(f"DB error: {e}")
-
-    # MEMORY.md
-    if MEMORY_MD.exists():
-        state["memory_md_lines"] = len(MEMORY_MD.read_text().splitlines())
-
-    # claude-mem.db old observations
-    if CLAUDE_MEM_DB.exists():
-        try:
-            cutoff = int((datetime.now() - timedelta(days=60)).timestamp() * 1000)
-            conn = sqlite3.connect(str(CLAUDE_MEM_DB))
-            state["claude_mem_old"] = conn.execute(
-                "SELECT COUNT(*) FROM observations WHERE created_at_epoch < ?", (cutoff,)
-            ).fetchone()[0]
-            conn.close()
-        except Exception:
-            pass
-
-    # Feedback count
-    state["feedback_count"] = len(list(MEMORY_MD.parent.glob("feedback_*.md")))
-
-    return state
-
-
-def should_dream(state: dict) -> bool:
-    """Check if there's enough to justify a CLI call."""
-    return (
-        len(state["learnings"]) > 10
-        or state["memory_md_lines"] > 170
-        or len(state["preferences"]) > 5
-        or state["claude_mem_old"] > 500
-    )
-
-
-def dream(state: dict) -> dict:
-    """The brain dreams — CLI does the intelligent work."""
-
-    # Truncate learnings JSON if too large
-    learnings_json = json.dumps(state["learnings"], ensure_ascii=False, indent=1)
-    if len(learnings_json) > 15000:
-        learnings_json = learnings_json[:15000] + "\n... (truncated)"
-
-    tasks = []
-
-    tasks.append(f"""TASK 1: LEARNING CONSOLIDATION ({len(state['learnings'])} active)
-Review these learnings and identify:
-a) DUPLICATES: learnings that say the same thing differently.
-b) CONTRADICTIONS: learnings that contradict each other.
-c) STALE: learnings about bugs/issues fixed >60 days ago that are never referenced.
-
-Write your findings to {COORD_DIR}/sleep-report.md with sections:
-- "## Duplicates to archive" — list learning IDs to archive and why
-- "## Contradictions" — pairs of conflicting learnings
-- "## Stale candidates" — IDs of learnings that may be obsolete
-
-Also write a machine-readable file {COORD_DIR}/sleep-actions.json:
-{{"archive_ids": [1, 2, 3], "contradiction_pairs": [[4, 5]], "stale_ids": [6, 7]}}
-
-The wrapper will execute the actual DB operations based on this JSON.
-
-LEARNINGS:
-{learnings_json}""")
-
-    if state["memory_md_lines"] > 170:
-        tasks.append(f"""TASK 2: MEMORY.MD COMPRESSION ({state['memory_md_lines']} lines, limit 200)
-File: {MEMORY_MD}
-Read it, compress resolved incidents >21 days, merge duplicates.
-NEVER delete: credentials, legal entity info, CRITICAL rules, infrastructure.
-Target: <180 lines.""")
-
-    if len(state["preferences"]) > 5:
-        tasks.append(f"""TASK 3: PREFERENCES CLEANUP ({len(state['preferences'])} entries)
-Review the preferences and identify duplicate keys.
-Add to sleep-actions.json: "duplicate_preference_keys": ["key1", "key2", ...]
-The wrapper will handle the actual DB cleanup safely.""")
-
-    if state["claude_mem_old"] > 500:
-        tasks.append(f"""TASK 4: OLD OBSERVATIONS ({state['claude_mem_old']} entries >60d)
-Note in sleep-report.md that old observations should be cleaned.
-Add to sleep-actions.json: "clean_old_observations": true
-The wrapper will handle the actual DB cleanup safely.""")
-
-    tasks_str = "\n\n".join(tasks)
-
-    prompt = f"""FIRST: Call nexo_startup(task='deep-sleep nightly maintenance') to register this session.
-
-You are NEXO Sleep — the nightly brain maintenance process.
-Like a human brain during sleep: consolidate important memories, discard noise,
-detect conflicts, prepare state for tomorrow.
-Use nexo_learning_add, nexo_followup_create, nexo_session_diary_write and other MCP tools directly.
-
-BRAIN STATE:
-- {len(state['learnings'])} active learnings
-- {state['memory_md_lines']} lines in MEMORY.md (limit: 200)
-- {len(state['preferences'])} preferences
-- {state['feedback_count']} feedback files
-- {state['claude_mem_old']} old observations (>60d)
-
-{tasks_str}
-
-ABSOLUTE RULES:
-- NEVER delete legal entity info (LLC, SLU, EIN, NIF, project)
-- NEVER delete credentials, tokens, API keys, secrets
-- NEVER delete rules marked CRITICAL or MAX PRIORITY
-- NEVER delete infrastructure info (servers, repos, deploys)
-- When in doubt, DON'T delete
-
-Write a summary to {COORD_DIR}/sleep-report.md when done.
-Execute without asking."""
-
-    log("Stage B: Invoking Claude CLI (opus) — dreaming...")
-    env = os.environ.copy()
-    env["NEXO_HEADLESS"] = "1"  # Skip stop hook post-mortem
-    env.pop("CLAUDECODE", None)
-    env.pop("CLAUDE_CODE", None)
-
-    try:
-        result = subprocess.run(
-            [str(CLAUDE_CLI), "-p", prompt, "--model", "opus",
-             "--output-format", "text",
-             "--allowedTools", "Read,Write,Edit,Glob,Grep,Bash,mcp__nexo__*"],
-            capture_output=True, text=True, timeout=21600, env=env
-        )
-
-        if result.returncode != 0:
-            log(f"Stage B: CLI error ({result.returncode}): {(result.stderr or '')[:300]}")
-            return {"error": result.returncode}
-
-        log(f"Stage B: Dreaming complete. Output: {len(result.stdout or '')} chars")
-        return {"ok": True, "output_len": len(result.stdout or "")}
-
-    except subprocess.TimeoutExpired:
-        log("Stage B: CLI timed out (600s)")
-        return {"error": "timeout"}
-    except Exception as e:
-        log(f"Stage B: Exception: {e}")
-        return {"error": str(e)}
-
-
-def execute_dream_actions(actions: dict, state: dict):
-    """Execute the DB actions decided by CLI, safely in Python."""
-    log("Stage B2: Executing dream actions...")
-
-    # Archive duplicate/stale learnings
-    archive_ids = actions.get("archive_ids", []) + actions.get("stale_ids", [])
-    if archive_ids and NEXO_DB.exists():
-        try:
-            conn = sqlite3.connect(str(NEXO_DB))
-            for lid in archive_ids:
-                if isinstance(lid, int):
-                    conn.execute(
-                        "UPDATE learnings SET status='archived' WHERE id=? AND status='active'",
-                        (lid,)
-                    )
-            conn.commit()
-            conn.close()
-            log(f"  Archived {len(archive_ids)} learnings: {archive_ids}")
-        except Exception as e:
-            log(f"  Error archiving learnings: {e}")
-
-    # Clean duplicate preferences
-    dup_keys = actions.get("duplicate_preference_keys", [])
-    if dup_keys and NEXO_DB.exists():
-        try:
-            conn = sqlite3.connect(str(NEXO_DB))
-            for key in dup_keys:
-                if isinstance(key, str):
-                    # Keep newest, delete older duplicates
-                    conn.execute(
-                        "DELETE FROM preferences WHERE key = ? AND rowid NOT IN "
-                        "(SELECT rowid FROM preferences WHERE key = ? ORDER BY updated_at DESC LIMIT 1)",
-                        (key, key)
-                    )
-            conn.commit()
-            conn.close()
-            log(f"  Cleaned {len(dup_keys)} duplicate preference keys")
-        except Exception as e:
-            log(f"  Error cleaning preferences: {e}")
-
-    # Clean old observations
-    if actions.get("clean_old_observations") and CLAUDE_MEM_DB.exists():
-        try:
-            cutoff_ms = int((datetime.now() - timedelta(days=60)).timestamp() * 1000)
-            conn = sqlite3.connect(str(CLAUDE_MEM_DB))
-            deleted = conn.execute(
-                "DELETE FROM observations WHERE created_at_epoch < ? "
-                "AND discovery_tokens < 300 "
-                "AND id NOT IN (SELECT id FROM observations WHERE "
-                "title LIKE '%CRITICO%' OR title LIKE '%credential%' "
-                "OR title LIKE '%token%' OR title LIKE '%API%' "
-                "OR title LIKE '%LLC%' OR title LIKE '%SLU%') "
-                "LIMIT 200",
-                (cutoff_ms,)
-            ).rowcount
-            conn.execute(
-                "DELETE FROM observations_fts WHERE rowid NOT IN "
-                "(SELECT id FROM observations)"
-            )
-            conn.execute("VACUUM")
-            conn.commit()
-            conn.close()
-            log(f"  Cleaned {deleted} old observations")
-        except Exception as e:
-            log(f"  Error cleaning observations: {e}")
-
-    log("Stage B2: Actions complete.")
-
-
-# ─── Main ────────────────────────────────────────────────────────────────────
-
-def main():
-    log("=" * 60)
-    log("NEXO Sleep System v2 starting")
-
-    # Process lock
-    try:
-        lock_fd = open(PROCESS_LOCK, "w")
-        fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-        lock_fd.write(str(os.getpid()))
-        lock_fd.flush()
-    except (IOError, OSError):
-        log("Another sleep instance running. Exiting.")
-        sys.exit(0)
-
-    try:
-        if already_ran_today():
-            log("Already ran today. Exiting.")
-            sys.exit(0)
-
-        start_phase = "stage_a"
-        if was_interrupted():
-            start_phase = get_interrupted_phase()
-
-        run_log = {"date": str(TODAY), "started": TIMESTAMP,
-                   "stage_a": None, "stage_b": None, "completed": None}
-
-        # Stage A: Housekeeping (mechanical)
-        if start_phase == "stage_a":
-            set_lock("stage_a")
-            log("─── Stage A: Housekeeping ───")
-            run_log["stage_a"] = stage_a_cleanup()
-
-        # Stage B: Dreaming (intelligent)
-        set_lock("stage_b")
-        log("─── Stage B: Dreaming ───")
-        state = collect_brain_state()
-
-        if should_dream(state):
-            log(f"Brain state: {len(state['learnings'])} learnings, "
-                f"{state['memory_md_lines']} MEMORY lines, "
-                f"{state['claude_mem_old']} old observations")
-            run_log["stage_b"] = dream(state)
-
-            # Stage B2: Execute actions from CLI output
-            actions_file = COORD_DIR / "sleep-actions.json"
-            if actions_file.exists():
-                try:
-                    actions = json.loads(actions_file.read_text())
-                    execute_dream_actions(actions, state)
-                except Exception as e:
-                    log(f"Stage B2: Error executing actions: {e}")
-        else:
-            log("Brain is clean — no dreaming needed.")
-            run_log["stage_b"] = {"skipped": True}
-
-        # Done
-        run_log["completed"] = datetime.now().strftime("%Y-%m-%d %H:%M")
-        mark_complete()
-        append_sleep_log(run_log)
-        log(f"NEXO Sleep v2 complete at {run_log['completed']}")
-
-        # Register for catch-up
-        try:
-            state_file = NEXO_HOME / "operations" / ".catchup-state.json"
-            st = json.loads(state_file.read_text()) if state_file.exists() else {}
-            st["sleep"] = datetime.now().isoformat()
-            state_file.write_text(json.dumps(st, indent=2))
-        except Exception:
-            pass
-
-    finally:
-        try:
-            fcntl.flock(lock_fd, fcntl.LOCK_UN)
-            lock_fd.close()
-            PROCESS_LOCK.unlink(missing_ok=True)
-        except Exception:
-            pass
-
-
-if __name__ == "__main__":
-    main()

package/src/scripts/nexo-snapshot-restore 2.sh

@@ -1,35 +0,0 @@
-#!/bin/bash
-# NEXO Snapshot Restore — restores files from a snapshot directory.
-# Usage: nexo-snapshot-restore.sh <snapshot-dir>
-set -euo pipefail
-
-SNAP_DIR="${1:?Usage: nexo-snapshot-restore.sh <snapshot-dir>}"
-MANIFEST="$SNAP_DIR/manifest.json"
-NEXO_HOME="${NEXO_HOME:-$HOME/.nexo}"
-RESTORE_LOG="$NEXO_HOME/logs/snapshot-restores.log"
-
-if [ ! -f "$MANIFEST" ]; then
-    echo "ERROR: No manifest.json in $SNAP_DIR" >&2
-    exit 1
-fi
-
-TS=$(date "+%Y-%m-%d %H:%M:%S")
-mkdir -p "$(dirname "$RESTORE_LOG")"
-echo "[$TS] Restoring from $SNAP_DIR" >> "$RESTORE_LOG"
-
-python3 -c "
-import json, shutil, os
-manifest = json.load(open('$MANIFEST'))
-for rel_path in manifest.get('files', []):
-    src = os.path.join('$SNAP_DIR', 'files', rel_path)
-    dst = os.path.expanduser('~/' + rel_path)
-    if os.path.exists(src):
-        os.makedirs(os.path.dirname(dst), exist_ok=True)
-        shutil.copy2(src, dst)
-        print(f'  Restored: {rel_path}')
-    else:
-        print(f'  SKIP (not in snapshot): {rel_path}')
-print('Restore complete.')
-"
-
-echo "[$TS] Restore complete from $SNAP_DIR" >> "$RESTORE_LOG"