nexo-brain 2.1.0 → 2.3.0

package/src/scripts/nexo-learning-validator 2.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""
+NEXO Learning Validator — Cross-checks findings against existing learnings.
+
+Wrapper collects the finding + all learnings from SQLite, then passes
+to Claude CLI (opus) to make an intelligent determination of whether
+the finding is known, related, or genuinely new.
+
+Usage as CLI:
+    python3 nexo-learning-validator.py "finding text to validate"
+    python3 nexo-learning-validator.py --category project "finding text"
+
+Usage as library:
+    from nexo_learning_validator import validate_finding
+    result = validate_finding("CRITICAL: message_id column is NULL")
+    if result["known"]:
+        print(f"Already known: {result['matching_learnings']}")
+
+Exit codes:
+    0 = Finding is NEW (not known)
+    1 = Finding is KNOWN (matches existing learning)
+"""
+
+import json
+import os
+import sqlite3
+import subprocess
+import sys
+from pathlib import Path
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+
+NEXO_DB = NEXO_HOME / "data" / "nexo.db"
+CLAUDE_CLI = Path.home() / ".local" / "bin" / "claude"
+
+
+def get_all_learnings(category: str = None) -> list[dict]:
+    """Fetch all learnings from nexo.db."""
+    conn = sqlite3.connect(str(NEXO_DB), timeout=10)
+    conn.row_factory = sqlite3.Row
+    if category:
+        rows = conn.execute(
+            "SELECT id, category, title, content FROM learnings WHERE category = ?",
+            (category,)
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            "SELECT id, category, title, content FROM learnings"
+        ).fetchall()
+    conn.close()
+    return [dict(r) for r in rows]
+
+
+def validate_finding(finding: str, category: str = None) -> dict:
+    """
+    Validate a finding against existing learnings using Claude CLI.
+
+    Returns:
+        {
+            "known": bool,
+            "confidence": float (0-1),
+            "matching_learnings": [{"id": int, "title": str, "similarity": float}],
+            "recommendation": str
+        }
+    """
+    learnings = get_all_learnings(category)
+
+    if not learnings:
+        return {
+            "known": False,
+            "confidence": 0,
+            "matching_learnings": [],
+            "recommendation": "No learnings in DB — finding is new by default"
+        }
+
+    # Build compact learnings reference for CLI
+    learnings_ref = []
+    for l in learnings:
+        learnings_ref.append({
+            "id": l["id"],
+            "cat": l["category"],
+            "title": l["title"],
+            "content": (l["content"] or "")[:300],
+        })
+
+    prompt = f"""You are a finding deduplication engine. Compare a new finding against existing learnings and determine if it's already known.
+
+NEW FINDING:
+{finding}
+
+EXISTING LEARNINGS ({len(learnings_ref)} total):
+{json.dumps(learnings_ref, indent=1)}
+
+Respond with ONLY valid JSON (no markdown, no code fences):
+{{
+  "known": true/false,
+  "confidence": 0.0-1.0,
+  "matching_learnings": [
+    {{"id": <learning_id>, "title": "<title>", "similarity": 0.0-1.0}}
+  ],
+  "recommendation": "<one line: KNOWN/LIKELY KNOWN/POSSIBLY RELATED/NEW>"
+}}
+
+Rules:
+- confidence >= 0.7 and same root cause = known: true
+- confidence 0.55-0.7 and related topic = known: true, say LIKELY KNOWN
+- confidence < 0.55 = known: false
+- Max 5 matching_learnings, sorted by similarity descending
+- If the finding describes the SAME bug/issue/pattern as a learning, it's known even if worded differently
+- Be strict: different symptoms of different bugs are NOT the same even if they mention the same file"""
+
+    # Try CLI first, fall back to mechanical similarity
+    if CLAUDE_CLI.exists():
+    # Fallback: mechanical SequenceMatcher (original logic)
+    return _mechanical_validate(finding, learnings)
+
+
+def _mechanical_validate(finding: str, learnings: list[dict]) -> dict:
+    """Fallback validation using SequenceMatcher when CLI is unavailable."""
+    from difflib import SequenceMatcher
+
+    threshold = 0.45
+    finding_kw = _extract_keywords(finding)
+    matches = []
+
+    for learning in learnings:
+        title_sim = SequenceMatcher(None, finding.lower(), learning["title"].lower()).ratio()
+        content_sim = SequenceMatcher(None, finding.lower(), (learning["content"] or "").lower()).ratio()
+
+        learning_text = f"{learning['title']} {learning['content'] or ''}"
+        learning_kw = _extract_keywords(learning_text)
+        kw_overlap = len(finding_kw & learning_kw) / len(finding_kw) if finding_kw and learning_kw else 0
+
+        combined = max(title_sim, content_sim) * 0.6 + kw_overlap * 0.4
+
+        if combined >= threshold:
+            matches.append({
+                "id": learning["id"],
+                "category": learning["category"],
+                "title": learning["title"],
+                "similarity": round(combined, 3),
+            })
+
+    matches.sort(key=lambda x: x["similarity"], reverse=True)
+    top = matches[:5]
+
+    if not top:
+        return {"known": False, "confidence": 0, "matching_learnings": [], "recommendation": "NEW finding"}
+
+    best = top[0]["similarity"]
+    if best >= 0.7:
+        return {"known": True, "confidence": best, "matching_learnings": top,
+                "recommendation": f"KNOWN issue (learning #{top[0]['id']})"}
+    elif best >= 0.55:
+        return {"known": True, "confidence": best, "matching_learnings": top,
+                "recommendation": f"LIKELY KNOWN (learning #{top[0]['id']})"}
+    else:
+        return {"known": False, "confidence": best, "matching_learnings": top,
+                "recommendation": "POSSIBLY RELATED but different enough to report"}
+
+
+def _extract_keywords(text: str) -> set:
+    """Extract meaningful keywords from text."""
+    stop_words = {
+        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+        'should', 'may', 'might', 'must', 'shall', 'can', 'need', 'dare',
+        'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as',
+        'and', 'but', 'or', 'nor', 'not', 'so', 'yet', 'both', 'either',
+        'error', 'critical', 'warning', 'bug', 'issue', 'problem', 'fix',
+        'el', 'la', 'los', 'las', 'un', 'una', 'de', 'en', 'que', 'por',
+    }
+    words = set()
+    for word in text.lower().split():
+        clean = ''.join(c for c in word if c.isalnum() or c == '_')
+        if clean and len(clean) > 2 and clean not in stop_words:
+            words.add(clean)
+    return words
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Validate findings against existing NEXO learnings")
+    parser.add_argument("finding", help="The finding text to validate")
+    parser.add_argument("--category", "-c", help="Filter learnings by category")
+    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
+    args = parser.parse_args()
+
+    result = validate_finding(args.finding, args.category)
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        status = "KNOWN" if result["known"] else "NEW"
+        print(f"Status: {status} (confidence: {result['confidence']:.0%})")
+        print(f"Recommendation: {result['recommendation']}")
+        if result["matching_learnings"]:
+            print(f"Related learnings:")
+            for m in result["matching_learnings"]:
+                cat = m.get('category', '?')
+                print(f"  #{m['id']} [{cat}] {m['title']} ({m['similarity']:.0%})")
+
+    sys.exit(1 if result["known"] else 0)
+
+
+if __name__ == "__main__":
+    main()

package/src/scripts/nexo-learning-validator.py

@@ -111,6 +111,25 @@ Rules:
 
     # Try CLI first, fall back to mechanical similarity
     if CLAUDE_CLI.exists():
+        try:
+            env = os.environ.copy()
+            env["NEXO_HEADLESS"] = "1"
+            env.pop("CLAUDECODE", None)
+            env.pop("CLAUDE_CODE", None)
+            learnings_text = "\n".join(
+                f"[#{l.get('id','')}] {l.get('title','')}: {l.get('content','')[:200]}"
+                for l in learnings[:20]
+            )
+            prompt = f"{VALIDATE_PROMPT}\n\nFinding:\n{finding}\n\nExisting learnings:\n{learnings_text}"
+            result = subprocess.run(
+                [str(CLAUDE_CLI), "-p", prompt, "--model", "sonnet", "--output-format", "text"],
+                capture_output=True, text=True, timeout=60, env=env
+            )
+            if result.returncode == 0 and result.stdout.strip():
+                parsed = json.loads(result.stdout.strip())
+                return parsed
+        except Exception:
+            pass
     # Fallback: mechanical SequenceMatcher (original logic)
     return _mechanical_validate(finding, learnings)
 

package/src/scripts/nexo-migrate 2.py

@@ -0,0 +1,232 @@
+#!/usr/bin/env python3
+"""NEXO Migration Tool — automatic, idempotent upgrades between versions.
+
+Usage:
+    python3 nexo-migrate.py              # auto-detect current → target
+    python3 nexo-migrate.py --dry-run    # show what would happen
+    python3 nexo-migrate.py --from 1.6.0 # override detected current version
+
+Reads current version from $NEXO_HOME/version.json.
+Reads target version from the repo's package.json.
+Backs up NEXO_HOME/db/ before any migration.
+Runs DB schema migrations via the existing _schema.py system.
+"""
+
+import argparse
+import json
+import os
+import shutil
+import sqlite3
+import sys
+from datetime import datetime
+from pathlib import Path
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", Path.home() / ".nexo"))
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent  # nexo/src/scripts -> nexo/
+
+
+# ── Version helpers ──────────────────────────────────────────────
+
+def parse_version(v: str) -> tuple:
+    """Parse '1.7.0-beta.1' → (1, 7, 0, 'beta.1'). Pre-release is optional."""
+    parts = v.strip().lstrip("v").split("-", 1)
+    nums = tuple(int(x) for x in parts[0].split("."))
+    pre = parts[1] if len(parts) > 1 else ""
+    return (*nums, pre)
+
+
+def version_key(v: str) -> tuple:
+    """Sortable key: releases sort after pre-releases of same version."""
+    nums = parse_version(v)
+    # Empty pre-release string sorts AFTER any pre-release tag
+    pre = nums[3] if len(nums) > 3 else ""
+    return (nums[0], nums[1], nums[2], 0 if pre else 1, pre)
+
+
+def get_current_version() -> str:
+    """Read installed version from NEXO_HOME/version.json."""
+    vfile = NEXO_HOME / "version.json"
+    if not vfile.exists():
+        return "0.0.0"
+    try:
+        data = json.loads(vfile.read_text())
+        return data.get("version", "0.0.0")
+    except Exception:
+        return "0.0.0"
+
+
+def get_target_version() -> str:
+    """Read target version from repo package.json."""
+    pkg = REPO_ROOT / "package.json"
+    if not pkg.exists():
+        print(f"ERROR: package.json not found at {pkg}", file=sys.stderr)
+        sys.exit(1)
+    data = json.loads(pkg.read_text())
+    return data["version"]
+
+
+# ── Backup ───────────────────────────────────────────────────────
+
+def backup_databases() -> str:
+    """Backup all .db files before migration. Returns backup dir path."""
+    ts = datetime.now().strftime("%Y%m%d-%H%M%S")
+    backup_dir = NEXO_HOME / "backups" / f"pre-migrate-{ts}"
+    backup_dir.mkdir(parents=True, exist_ok=True)
+
+    data_dir = NEXO_HOME / "data"
+    if data_dir.exists():
+        for db_file in data_dir.glob("*.db*"):
+            shutil.copy2(db_file, backup_dir / db_file.name)
+    # Also check legacy db/ location
+    legacy_db_dir = NEXO_HOME / "db"
+    if legacy_db_dir.exists():
+        for db_file in legacy_db_dir.glob("*.db*"):
+            if not (backup_dir / db_file.name).exists():
+                shutil.copy2(db_file, backup_dir / db_file.name)
+
+    # Also backup version.json
+    vfile = NEXO_HOME / "version.json"
+    if vfile.exists():
+        shutil.copy2(vfile, backup_dir / "version.json")
+
+    return str(backup_dir)
+
+
+# ── Migration steps ──────────────────────────────────────────────
+
+def ensure_nexo_home_dirs():
+    """Create all required NEXO_HOME subdirectories."""
+    dirs = [
+        "db", "brain", "logs", "operations", "coordination",
+        "scripts", "hooks", "plugins", "backups", "memory",
+        "docs", "projects", "learnings", "agents", "skills",
+    ]
+    for d in dirs:
+        (NEXO_HOME / d).mkdir(parents=True, exist_ok=True)
+
+
+def run_db_schema_migrations():
+    """Run the formal DB schema migration system from _schema.py."""
+    # Add src/ to path so we can import the db module
+    src_dir = REPO_ROOT / "src"
+    if str(src_dir) not in sys.path:
+        sys.path.insert(0, str(src_dir))
+
+    # Set NEXO_HOME env for the db module
+    os.environ["NEXO_HOME"] = str(NEXO_HOME)
+    os.environ["NEXO_SKIP_FS_INDEX"] = "1"  # Don't rebuild FTS during migration
+
+    try:
+        from db import init_db
+        init_db()
+        print("  DB schema migrations applied.")
+    except Exception as e:
+        print(f"  WARNING: DB schema migration error: {e}", file=sys.stderr)
+
+
+def write_version_json(version: str):
+    """Write version.json with the installed version."""
+    vfile = NEXO_HOME / "version.json"
+    data = {
+        "version": version,
+        "installed_at": datetime.now().isoformat(timespec="seconds"),
+        "nexo_home": str(NEXO_HOME),
+    }
+    vfile.write_text(json.dumps(data, indent=2) + "\n")
+
+
+# ── Migration registry ───────────────────────────────────────────
+# Each entry: version → list of (description, callable)
+# Migrations run for all versions > current AND <= target.
+
+def _migrate_1_7_0():
+    """1.7.0: Ensure NEXO_HOME paths, create directories, update version."""
+    ensure_nexo_home_dirs()
+    run_db_schema_migrations()
+    print("  Created/verified all NEXO_HOME directories.")
+
+
+MIGRATION_REGISTRY: dict[str, list[tuple[str, callable]]] = {
+    "1.7.0": [
+        ("Ensure NEXO_HOME dirs + DB schema", _migrate_1_7_0),
+    ],
+}
+
+
+# ── Main ─────────────────────────────────────────────────────────
+
+def get_applicable_migrations(current: str, target: str) -> list[tuple[str, str, callable]]:
+    """Return list of (version, description, fn) for migrations between current and target."""
+    current_key = version_key(current)
+    target_key = version_key(target)
+
+    applicable = []
+    for ver, steps in sorted(MIGRATION_REGISTRY.items(), key=lambda x: version_key(x[0])):
+        ver_key = version_key(ver)
+        # Run if version > current and <= target (base version comparison)
+        base_ver = ver.split("-")[0]  # strip pre-release for comparison
+        base_ver_key = version_key(base_ver)
+        if base_ver_key > (current_key[0], current_key[1], current_key[2], current_key[3], current_key[4] if len(current_key) > 4 else ""):
+            if base_ver_key <= (target_key[0], target_key[1], target_key[2], 1, ""):
+                for desc, fn in steps:
+                    applicable.append((ver, desc, fn))
+
+    return applicable
+
+
+def main():
+    parser = argparse.ArgumentParser(description="NEXO Migration Tool")
+    parser.add_argument("--dry-run", action="store_true", help="Show what would happen without executing")
+    parser.add_argument("--from", dest="from_ver", help="Override detected current version")
+    args = parser.parse_args()
+
+    current = args.from_ver or get_current_version()
+    target = get_target_version()
+
+    print(f"NEXO Migration: {current} → {target}")
+    print(f"NEXO_HOME: {NEXO_HOME}")
+    print()
+
+    if version_key(current) >= version_key(target):
+        print("Already up to date. Nothing to migrate.")
+        return
+
+    migrations = get_applicable_migrations(current, target)
+    if not migrations:
+        print("No migration steps needed (only version bump).")
+    else:
+        print(f"Migrations to run ({len(migrations)}):")
+        for ver, desc, _ in migrations:
+            print(f"  [{ver}] {desc}")
+        print()
+
+    if args.dry_run:
+        print("DRY RUN — no changes made.")
+        return
+
+    # Backup before anything
+    backup_path = backup_databases()
+    print(f"Backup created: {backup_path}")
+    print()
+
+    # Ensure base directories exist
+    ensure_nexo_home_dirs()
+
+    # Run migrations
+    for ver, desc, fn in migrations:
+        print(f"Running [{ver}] {desc}...")
+        try:
+            fn()
+            print(f"  Done.")
+        except Exception as e:
+            print(f"  ERROR: {e}", file=sys.stderr)
+            print(f"  Backup at: {backup_path}", file=sys.stderr)
+            sys.exit(1)
+
+    # Write final version
+    write_version_json(target)
+    print(f"\nMigration complete: {current} → {target}")
+
+
+if __name__ == "__main__":
+    main()