nexo-brain 5.3.13 → 5.3.15

package/src/scripts/nexo-learning-validator 2.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""
+NEXO Learning Validator — Cross-check findings against existing learnings.
+
+The wrapper collects the finding + current learnings from SQLite, then asks the
+configured automation backend whether the finding is already known, related, or
+genuinely new. If the backend is unavailable, it falls back to mechanical
+similarity matching.
+
+Usage as CLI:
+    python3 nexo-learning-validator.py "finding text to validate"
+    python3 nexo-learning-validator.py --category project "finding text"
+
+Usage as library:
+    from nexo_learning_validator import validate_finding
+    result = validate_finding("CRITICAL: message_id column is NULL")
+    if result["known"]:
+        print(f"Already known: {result['matching_learnings']}")
+
+Exit codes:
+    0 = Finding is NEW (not known)
+    1 = Finding is KNOWN (matches existing learning)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+import sys
+from pathlib import Path
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parents[1])))
+if str(NEXO_CODE) not in sys.path:
+    sys.path.insert(0, str(NEXO_CODE))
+
+from agent_runner import AutomationBackendUnavailableError, run_automation_prompt
+
+try:
+    from client_preferences import resolve_user_model as _resolve_user_model
+    _USER_MODEL = _resolve_user_model()
+except Exception:
+    _USER_MODEL = ""
+
+
+
+NEXO_DB = NEXO_HOME / "data" / "nexo.db"
+JSON_ONLY_SYSTEM_PROMPT = (
+    "Return exactly one valid JSON object. No markdown fences. No prose outside JSON."
+)
+
+
+def get_all_learnings(category: str | None = None) -> list[dict]:
+    """Fetch all learnings from nexo.db."""
+    conn = sqlite3.connect(str(NEXO_DB), timeout=10)
+    conn.row_factory = sqlite3.Row
+    if category:
+        rows = conn.execute(
+            "SELECT id, category, title, content FROM learnings WHERE category = ?",
+            (category,),
+        ).fetchall()
+    else:
+        rows = conn.execute(
+            "SELECT id, category, title, content FROM learnings"
+        ).fetchall()
+    conn.close()
+    return [dict(r) for r in rows]
+
+
+def _extract_json(text: str) -> dict | None:
+    text = (text or "").strip()
+    if not text:
+        return None
+    if text.startswith("```"):
+        lines = text.splitlines()
+        end = len(lines)
+        for idx in range(len(lines) - 1, 0, -1):
+            if lines[idx].strip() == "```":
+                end = idx
+                break
+        text = "\n".join(lines[1:end]).strip()
+    brace_start = text.find("{")
+    if brace_start < 0:
+        return None
+    depth = 0
+    for idx in range(brace_start, len(text)):
+        if text[idx] == "{":
+            depth += 1
+        elif text[idx] == "}":
+            depth -= 1
+            if depth == 0:
+                try:
+                    return json.loads(text[brace_start:idx + 1])
+                except json.JSONDecodeError:
+                    return None
+    return None
+
+
+def validate_finding(finding: str, category: str | None = None) -> dict:
+    """
+    Validate a finding against existing learnings.
+
+    Returns:
+        {
+            "known": bool,
+            "confidence": float (0-1),
+            "matching_learnings": [{"id": int, "title": str, "similarity": float}],
+            "recommendation": str
+        }
+    """
+    learnings = get_all_learnings(category)
+
+    if not learnings:
+        return {
+            "known": False,
+            "confidence": 0,
+            "matching_learnings": [],
+            "recommendation": "No learnings in DB — finding is new by default",
+        }
+
+    learnings_ref = [
+        {
+            "id": l["id"],
+            "cat": l["category"],
+            "title": l["title"],
+            "content": (l["content"] or "")[:300],
+        }
+        for l in learnings
+    ]
+
+    prompt = f"""You are a finding deduplication engine. Compare a new finding against existing learnings and determine if it's already known.
+
+NEW FINDING:
+{finding}
+
+EXISTING LEARNINGS ({len(learnings_ref)} total):
+{json.dumps(learnings_ref, indent=1, ensure_ascii=False)}
+
+Respond with ONLY valid JSON:
+{{
+  "known": true/false,
+  "confidence": 0.0-1.0,
+  "matching_learnings": [
+    {{"id": <learning_id>, "title": "<title>", "similarity": 0.0-1.0}}
+  ],
+  "recommendation": "<one line: KNOWN/LIKELY KNOWN/POSSIBLY RELATED/NEW>"
+}}
+
+Rules:
+- confidence >= 0.7 and same root cause = known: true
+- confidence 0.55-0.7 and related topic = known: true, say LIKELY KNOWN
+- confidence < 0.55 = known: false
+- Max 5 matching_learnings, sorted by similarity descending
+- If the finding describes the SAME bug/issue/pattern as a learning, it's known even if worded differently
+- Be strict: different symptoms of different bugs are NOT the same even if they mention the same file"""
+
+    try:
+        result = run_automation_prompt(
+            prompt,
+            model=_USER_MODEL or "sonnet",
+            timeout=60,
+            output_format="text",
+            append_system_prompt=JSON_ONLY_SYSTEM_PROMPT,
+        )
+        parsed = _extract_json(result.stdout)
+        if result.returncode == 0 and parsed:
+            return parsed
+    except AutomationBackendUnavailableError:
+        pass
+    except Exception:
+        pass
+
+    return _mechanical_validate(finding, learnings)
+
+
+def _mechanical_validate(finding: str, learnings: list[dict]) -> dict:
+    """Fallback validation using SequenceMatcher when backend is unavailable."""
+    from difflib import SequenceMatcher
+
+    threshold = 0.45
+    finding_kw = _extract_keywords(finding)
+    matches = []
+
+    for learning in learnings:
+        title_sim = SequenceMatcher(None, finding.lower(), learning["title"].lower()).ratio()
+        content_sim = SequenceMatcher(None, finding.lower(), (learning["content"] or "").lower()).ratio()
+
+        learning_text = f"{learning['title']} {learning['content'] or ''}"
+        learning_kw = _extract_keywords(learning_text)
+        kw_overlap = len(finding_kw & learning_kw) / len(finding_kw) if finding_kw and learning_kw else 0
+
+        combined = max(title_sim, content_sim) * 0.6 + kw_overlap * 0.4
+
+        if combined >= threshold:
+            matches.append({
+                "id": learning["id"],
+                "category": learning["category"],
+                "title": learning["title"],
+                "similarity": round(combined, 3),
+            })
+
+    matches.sort(key=lambda x: x["similarity"], reverse=True)
+    top = matches[:5]
+
+    if not top:
+        return {"known": False, "confidence": 0, "matching_learnings": [], "recommendation": "NEW finding"}
+
+    best = top[0]["similarity"]
+    if best >= 0.7:
+        return {"known": True, "confidence": best, "matching_learnings": top,
+                "recommendation": f"KNOWN issue (learning #{top[0]['id']})"}
+    if best >= 0.55:
+        return {"known": True, "confidence": best, "matching_learnings": top,
+                "recommendation": f"LIKELY KNOWN (learning #{top[0]['id']})"}
+    return {"known": False, "confidence": best, "matching_learnings": top,
+            "recommendation": "POSSIBLY RELATED but different enough to report"}
+
+
+def _extract_keywords(text: str) -> set:
+    """Extract meaningful keywords from text."""
+    stop_words = {
+        "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+        "have", "has", "had", "do", "does", "did", "will", "would", "could",
+        "should", "may", "might", "must", "shall", "can", "need", "dare",
+        "to", "of", "in", "for", "on", "with", "at", "by", "from", "as",
+        "and", "but", "or", "nor", "not", "so", "yet", "both", "either",
+        "error", "critical", "warning", "bug", "issue", "problem", "fix",
+        "el", "la", "los", "las", "un", "una", "de", "en", "que", "por",
+    }
+    words = set()
+    for word in text.lower().split():
+        clean = "".join(c for c in word if c.isalnum() or c == "_")
+        if clean and len(clean) > 2 and clean not in stop_words:
+            words.add(clean)
+    return words
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Validate findings against existing NEXO learnings")
+    parser.add_argument("finding", help="The finding text to validate")
+    parser.add_argument("--category", "-c", help="Filter learnings by category")
+    parser.add_argument("--json", "-j", action="store_true", help="Output as JSON")
+    args = parser.parse_args()
+
+    result = validate_finding(args.finding, args.category)
+
+    if args.json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+    else:
+        status = "KNOWN" if result["known"] else "NEW"
+        print(f"Status: {status} (confidence: {result['confidence']:.0%})")
+        print(f"Recommendation: {result['recommendation']}")
+        if result["matching_learnings"]:
+            print("Related learnings:")
+            for match in result["matching_learnings"]:
+                cat = match.get("category", "?")
+                print(f"  #{match['id']} [{cat}] {match['title']} ({match['similarity']:.0%})")
+
+    sys.exit(1 if result["known"] else 0)
+
+
+if __name__ == "__main__":
+    main()

package/src/scripts/nexo-migrate 2.py

@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""NEXO Migration Tool — automatic, idempotent upgrades between versions.
+
+Usage:
+    python3 nexo-migrate.py              # auto-detect current → target
+    python3 nexo-migrate.py --dry-run    # show what would happen
+    python3 nexo-migrate.py --from 1.6.0 # override detected current version
+
+Reads current version from $NEXO_HOME/version.json.
+Reads target version from the installed runtime package.json.
+Backs up NEXO_HOME/db/ before any migration.
+Runs DB schema migrations via the existing _schema.py system.
+"""
+
+import argparse
+import json
+import os
+import shutil
+import sys
+from datetime import datetime
+from pathlib import Path
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo"))).expanduser()
+SCRIPT_ROOT = Path(__file__).resolve().parent.parent
+
+
+def _resolve_runtime_root() -> Path:
+    candidates: list[Path] = []
+
+    raw_code = os.environ.get("NEXO_CODE", "").strip()
+    if raw_code:
+        env_code = Path(raw_code).expanduser()
+        candidates.append(env_code)
+        if env_code.name == "src":
+            candidates.append(env_code.parent)
+
+    candidates.extend([SCRIPT_ROOT, NEXO_HOME])
+
+    seen: set[str] = set()
+    for candidate in candidates:
+        candidate = candidate if candidate.is_dir() else candidate.parent
+        key = str(candidate)
+        if key in seen:
+            continue
+        seen.add(key)
+        if (candidate / "package.json").is_file():
+            return candidate
+        if (candidate / "src" / "server.py").is_file() and (candidate / "package.json").is_file():
+            return candidate
+
+    return SCRIPT_ROOT
+
+
+RUNTIME_ROOT = _resolve_runtime_root()
+SOURCE_ROOT = RUNTIME_ROOT / "src" if (RUNTIME_ROOT / "src" / "db").is_dir() else RUNTIME_ROOT
+
+
+# ── Version helpers ──────────────────────────────────────────────
+
+def parse_version(v: str) -> tuple:
+    """Parse '1.7.0-beta.1' → (1, 7, 0, 'beta.1'). Pre-release is optional."""
+    parts = v.strip().lstrip("v").split("-", 1)
+    nums = tuple(int(x) for x in parts[0].split("."))
+    pre = parts[1] if len(parts) > 1 else ""
+    return (*nums, pre)
+
+
+def version_key(v: str) -> tuple:
+    """Sortable key: releases sort after pre-releases of same version."""
+    nums = parse_version(v)
+    # Empty pre-release string sorts AFTER any pre-release tag
+    pre = nums[3] if len(nums) > 3 else ""
+    return (nums[0], nums[1], nums[2], 0 if pre else 1, pre)
+
+
+def get_current_version() -> str:
+    """Read installed version from NEXO_HOME/version.json."""
+    vfile = NEXO_HOME / "version.json"
+    if not vfile.exists():
+        return "0.0.0"
+    try:
+        data = json.loads(vfile.read_text())
+        return data.get("version", "0.0.0")
+    except Exception:
+        return "0.0.0"
+
+
+def get_target_version() -> str:
+    """Read target version from installed runtime package.json."""
+    pkg = RUNTIME_ROOT / "package.json"
+    if not pkg.exists():
+        print(f"ERROR: package.json not found at {pkg}", file=sys.stderr)
+        sys.exit(1)
+    data = json.loads(pkg.read_text())
+    return data["version"]
+
+
+# ── Backup ───────────────────────────────────────────────────────
+
+def backup_databases() -> str:
+    """Backup all .db files before migration. Returns backup dir path."""
+    ts = datetime.now().strftime("%Y%m%d-%H%M%S")
+    backup_dir = NEXO_HOME / "backups" / f"pre-migrate-{ts}"
+    backup_dir.mkdir(parents=True, exist_ok=True)
+
+    data_dir = NEXO_HOME / "data"
+    if data_dir.exists():
+        for db_file in data_dir.glob("*.db*"):
+            shutil.copy2(db_file, backup_dir / db_file.name)
+    # Also check legacy db/ location
+    legacy_db_dir = NEXO_HOME / "db"
+    if legacy_db_dir.exists():
+        for db_file in legacy_db_dir.glob("*.db*"):
+            if not (backup_dir / db_file.name).exists():
+                shutil.copy2(db_file, backup_dir / db_file.name)
+
+    # Also backup version.json
+    vfile = NEXO_HOME / "version.json"
+    if vfile.exists():
+        shutil.copy2(vfile, backup_dir / "version.json")
+
+    return str(backup_dir)
+
+
+# ── Migration steps ──────────────────────────────────────────────
+
+def ensure_nexo_home_dirs():
+    """Create all required NEXO_HOME subdirectories."""
+    dirs = [
+        "db", "brain", "logs", "operations", "coordination",
+        "scripts", "hooks", "plugins", "backups", "memory",
+        "docs", "projects", "learnings", "agents", "skills",
+    ]
+    for d in dirs:
+        (NEXO_HOME / d).mkdir(parents=True, exist_ok=True)
+
+
+def run_db_schema_migrations():
+    """Run the formal DB schema migration system from _schema.py."""
+    if str(SOURCE_ROOT) not in sys.path:
+        sys.path.insert(0, str(SOURCE_ROOT))
+
+    # Set NEXO_HOME env for the db module
+    os.environ["NEXO_HOME"] = str(NEXO_HOME)
+    os.environ["NEXO_SKIP_FS_INDEX"] = "1"  # Don't rebuild FTS during migration
+
+    try:
+        from db import init_db
+        init_db()
+        print("  DB schema migrations applied.")
+    except Exception as e:
+        print(f"  WARNING: DB schema migration error: {e}", file=sys.stderr)
+
+
+def write_version_json(version: str):
+    """Write version.json with the installed version."""
+    vfile = NEXO_HOME / "version.json"
+    data = {
+        "version": version,
+        "installed_at": datetime.now().isoformat(timespec="seconds"),
+        "nexo_home": str(NEXO_HOME),
+    }
+    vfile.write_text(json.dumps(data, indent=2) + "\n")
+
+
+# ── Migration registry ───────────────────────────────────────────
+# Each entry: version → list of (description, callable)
+# Migrations run for all versions > current AND <= target.
+
+def _migrate_1_7_0():
+    """1.7.0: Ensure NEXO_HOME paths, create directories, update version."""
+    ensure_nexo_home_dirs()
+    run_db_schema_migrations()
+    print("  Created/verified all NEXO_HOME directories.")
+
+
+MIGRATION_REGISTRY: dict[str, list[tuple[str, callable]]] = {
+    "1.7.0": [
+        ("Ensure NEXO_HOME dirs + DB schema", _migrate_1_7_0),
+    ],
+}
+
+
+# ── Main ─────────────────────────────────────────────────────────
+
+def get_applicable_migrations(current: str, target: str) -> list[tuple[str, str, callable]]:
+    """Return list of (version, description, fn) for migrations between current and target."""
+    current_key = version_key(current)
+    target_key = version_key(target)
+
+    applicable = []
+    for ver, steps in sorted(MIGRATION_REGISTRY.items(), key=lambda x: version_key(x[0])):
+        ver_key = version_key(ver)
+        # Run if version > current and <= target (base version comparison)
+        base_ver = ver.split("-")[0]  # strip pre-release for comparison
+        base_ver_key = version_key(base_ver)
+        if base_ver_key > (current_key[0], current_key[1], current_key[2], current_key[3], current_key[4] if len(current_key) > 4 else ""):
+            if base_ver_key <= (target_key[0], target_key[1], target_key[2], 1, ""):
+                for desc, fn in steps:
+                    applicable.append((ver, desc, fn))
+
+    return applicable
+
+
+def main():
+    parser = argparse.ArgumentParser(description="NEXO Migration Tool")
+    parser.add_argument("--dry-run", action="store_true", help="Show what would happen without executing")
+    parser.add_argument("--from", dest="from_ver", help="Override detected current version")
+    args = parser.parse_args()
+
+    current = args.from_ver or get_current_version()
+    target = get_target_version()
+
+    print(f"NEXO Migration: {current} → {target}")
+    print(f"NEXO_HOME: {NEXO_HOME}")
+    print()
+
+    if version_key(current) >= version_key(target):
+        print("Already up to date. Nothing to migrate.")
+        return
+
+    migrations = get_applicable_migrations(current, target)
+    if not migrations:
+        print("No migration steps needed (only version bump).")
+    else:
+        print(f"Migrations to run ({len(migrations)}):")
+        for ver, desc, _ in migrations:
+            print(f"  [{ver}] {desc}")
+        print()
+
+    if args.dry_run:
+        print("DRY RUN — no changes made.")
+        return
+
+    # Backup before anything
+    backup_path = backup_databases()
+    print(f"Backup created: {backup_path}")
+    print()
+
+    # Ensure base directories exist
+    ensure_nexo_home_dirs()
+
+    # Run migrations
+    for ver, desc, fn in migrations:
+        print(f"Running [{ver}] {desc}...")
+        try:
+            fn()
+            print(f"  Done.")
+        except Exception as e:
+            print(f"  ERROR: {e}", file=sys.stderr)
+            print(f"  Backup at: {backup_path}", file=sys.stderr)
+            sys.exit(1)
+
+    # Write final version
+    write_version_json(target)
+    print(f"\nMigration complete: {current} → {target}")
+
+
+if __name__ == "__main__":
+    main()

package/src/scripts/nexo-outcome-checker 2.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+"""NEXO Outcome Checker — daily verification of pending tracked outcomes."""
+
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+_script_dir = Path(__file__).resolve().parent
+_repo_src = _script_dir.parent
+NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(_repo_src) if (_repo_src / "server.py").exists() else str(NEXO_HOME)))
+if str(NEXO_CODE) not in sys.path:
+    sys.path.insert(0, str(NEXO_CODE))
+
+import db as nexo_db
+
+LOG_FILE = NEXO_HOME / "logs" / "outcome-checker.log"
+SUMMARY_FILE = NEXO_HOME / "coordination" / "outcome-checker-summary.json"
+
+
+def log(message: str):
+    ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    line = f"[{ts}] {message}"
+    print(line, flush=True)
+    LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with open(LOG_FILE, "a") as handle:
+        handle.write(line + "\n")
+
+
+def main() -> int:
+    log("=== Outcome Checker starting ===")
+    nexo_db.init_db()
+
+    due_rows = nexo_db.pending_outcomes_due(limit=500)
+    if not due_rows:
+        summary = {
+            "checked_at": datetime.now().isoformat(timespec="seconds"),
+            "checked": 0,
+            "met": 0,
+            "missed": 0,
+            "pending": 0,
+            "errors": 0,
+            "ids": [],
+        }
+        SUMMARY_FILE.parent.mkdir(parents=True, exist_ok=True)
+        SUMMARY_FILE.write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+        log("No pending due outcomes.")
+        log("=== Outcome Checker complete ===")
+        return 0
+
+    checked = met = missed = pending = errors = 0
+    checked_ids: list[int] = []
+    for row in due_rows:
+        checked += 1
+        checked_ids.append(int(row["id"]))
+        result = nexo_db.evaluate_outcome(int(row["id"]), create_learning_on_miss=True)
+        if "error" in result:
+            errors += 1
+            log(f"ERROR outcome #{row['id']}: {result['error']}")
+            continue
+        status = result.get("status", "pending")
+        if status == "met":
+            met += 1
+        elif status == "missed":
+            missed += 1
+        else:
+            pending += 1
+        log(
+            f"Outcome #{row['id']} -> {status} "
+            f"(action={result.get('action_type')}:{result.get('action_id') or '—'}, "
+            f"deadline={result.get('deadline')})"
+        )
+
+    # Phase 2 item 2: after closing outcomes, attempt to promote any
+    # outcome pattern that just crossed the suggested-skill threshold to a
+    # real draft skill. The helper is idempotent and capped at
+    # max_promotions per run, so this is safe to call on every cycle.
+    promotion_summary: dict = {"promoted": [], "skipped": [], "errors": [], "scanned": 0}
+    try:
+        from skills_runtime import auto_promote_outcome_patterns_to_skills
+        promotion_summary = auto_promote_outcome_patterns_to_skills(
+            min_success_rate=0.8,
+            max_promotions=3,
+        )
+        if promotion_summary.get("promoted"):
+            log(
+                f"Auto-promoted {len(promotion_summary['promoted'])} outcome pattern(s) "
+                f"to skill draft(s) (scanned={promotion_summary.get('scanned', 0)})"
+            )
+            for entry in promotion_summary["promoted"]:
+                log(
+                    f"  -> {entry.get('pattern_key')} -> skill {entry.get('skill_id')} "
+                    f"(success_rate={entry.get('success_rate')}, created={entry.get('created')})"
+                )
+        elif promotion_summary.get("scanned"):
+            log(
+                f"Outcome pattern auto-promote: scanned {promotion_summary['scanned']}, "
+                f"none qualified (skipped={len(promotion_summary['skipped'])})"
+            )
+    except Exception as e:
+        log(f"WARN: outcome pattern auto-promote raised: {e}")
+
+    summary = {
+        "checked_at": datetime.now().isoformat(timespec="seconds"),
+        "checked": checked,
+        "met": met,
+        "missed": missed,
+        "pending": pending,
+        "errors": errors,
+        "ids": checked_ids,
+        "auto_promoted_patterns": promotion_summary,
+    }
+    SUMMARY_FILE.parent.mkdir(parents=True, exist_ok=True)
+    SUMMARY_FILE.write_text(json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+    log(
+        f"Summary: checked={checked} met={met} missed={missed} "
+        f"pending={pending} errors={errors}"
+    )
+    log("=== Outcome Checker complete ===")
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())