nexo-brain 5.3.13 → 5.3.15

package/src/public_evolution_queue 2.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+"""Durable queue for public-core ports discovered outside the public runner.
+
+Managed flows such as self-audit may apply a local/core fix inline. When that
+fix belongs in the public repository as well, we persist a normalized queue
+entry in ``evolution_log`` so the weekly public contribution cycle can port it
+later instead of losing the improvement inside one machine.
+"""
+
+import json
+import os
+import sqlite3
+from pathlib import Path
+
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo"))).expanduser()
+QUEUE_CLASSIFICATION = "public_port_queue"
+QUEUE_STATUS_PENDING = "pending_public_port"
+PUBLIC_ALLOWED_PREFIXES = (
+    "src/",
+    "bin/",
+    "tests/",
+    "templates/",
+    "hooks/",
+    "migrations/",
+    ".claude-plugin/",
+)
+
+
+def resolve_repo_root(nexo_code: str | os.PathLike[str] | None = None) -> Path | None:
+    raw = Path(
+        nexo_code
+        or os.environ.get("NEXO_CODE")
+        or str(NEXO_HOME)
+    ).expanduser()
+    candidates = []
+    if raw.name == "src":
+        candidates.append(raw.parent)
+    candidates.append(raw)
+    for candidate in candidates:
+        if (candidate / "package.json").exists():
+            return candidate.resolve()
+    return None
+
+
+def normalize_public_path(
+    filepath: str,
+    *,
+    repo_root: Path | None = None,
+) -> str:
+    text = str(filepath or "").strip()
+    if not text:
+        return ""
+
+    normalized_raw = text.replace("\\", "/").lstrip("./")
+    if any(
+        normalized_raw == prefix.rstrip("/")
+        or normalized_raw.startswith(prefix)
+        for prefix in PUBLIC_ALLOWED_PREFIXES
+    ):
+        return normalized_raw
+
+    repo_root = repo_root or resolve_repo_root()
+    if not repo_root:
+        return ""
+
+    candidate = Path(text).expanduser()
+    if not candidate.is_absolute():
+        candidate = repo_root / candidate
+    try:
+        rel = candidate.resolve().relative_to(repo_root.resolve()).as_posix()
+    except Exception:
+        for prefix in PUBLIC_ALLOWED_PREFIXES:
+            marker = normalized_raw.find(prefix)
+            if marker >= 0:
+                return normalized_raw[marker:]
+        return ""
+    if any(rel == prefix.rstrip("/") or rel.startswith(prefix) for prefix in PUBLIC_ALLOWED_PREFIXES):
+        return rel
+    return ""
+
+
+def is_public_core_path(filepath: str, *, repo_root: Path | None = None) -> bool:
+    return bool(normalize_public_path(filepath, repo_root=repo_root))
+
+
+def queue_public_port_candidate(
+    conn: sqlite3.Connection,
+    *,
+    title: str,
+    reasoning: str,
+    files_changed: list[str],
+    source: str,
+    metadata: dict | None = None,
+) -> dict:
+    row = conn.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='evolution_log'"
+    ).fetchone()
+    if not row:
+        return {"ok": False, "reason": "evolution_log_missing"}
+
+    repo_root = resolve_repo_root()
+    normalized_files: list[str] = []
+    seen: set[str] = set()
+    for filepath in files_changed:
+        rel = normalize_public_path(filepath, repo_root=repo_root)
+        if rel and rel not in seen:
+            normalized_files.append(rel)
+            seen.add(rel)
+    if not normalized_files:
+        return {"ok": False, "reason": "no_public_files"}
+
+    proposal = str(title or "").strip()[:300] or "Managed core autofix queued for public port"
+    clean_reasoning = str(reasoning or "").strip()[:4000] or "Queued for public-core port."
+    payload = dict(metadata or {})
+    payload.setdefault("source", source)
+    payload["files"] = normalized_files
+
+    existing = conn.execute(
+        """SELECT id, status
+           FROM evolution_log
+           WHERE classification = ?
+             AND proposal = ?
+             AND files_changed = ?
+             AND status IN (?, 'draft_pr_created', 'skipped_duplicate_existing_pr')
+           ORDER BY id DESC
+           LIMIT 1""",
+        (
+            QUEUE_CLASSIFICATION,
+            proposal,
+            json.dumps(normalized_files),
+            QUEUE_STATUS_PENDING,
+        ),
+    ).fetchone()
+    if existing:
+        return {
+            "ok": True,
+            "queued": False,
+            "log_id": int(existing["id"]),
+            "status": str(existing["status"] or ""),
+            "files_changed": normalized_files,
+        }
+
+    cur = conn.execute(
+        """INSERT INTO evolution_log (
+               cycle_number, dimension, proposal, classification, reasoning, status, files_changed, test_result
+           ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+        (
+            0,
+            "public_core",
+            proposal,
+            QUEUE_CLASSIFICATION,
+            clean_reasoning,
+            QUEUE_STATUS_PENDING,
+            json.dumps(normalized_files),
+            json.dumps(payload, ensure_ascii=False),
+        ),
+    )
+    return {
+        "ok": True,
+        "queued": True,
+        "log_id": int(cur.lastrowid),
+        "status": QUEUE_STATUS_PENDING,
+        "files_changed": normalized_files,
+    }
+
+
+def list_pending_public_port_candidates(
+    conn: sqlite3.Connection,
+    *,
+    limit: int = 3,
+) -> list[dict]:
+    rows = conn.execute(
+        """SELECT id, created_at, proposal, reasoning, status, files_changed, test_result
+           FROM evolution_log
+           WHERE classification = ?
+             AND status = ?
+           ORDER BY created_at ASC, id ASC
+           LIMIT ?""",
+        (QUEUE_CLASSIFICATION, QUEUE_STATUS_PENDING, max(1, int(limit))),
+    ).fetchall()
+    results: list[dict] = []
+    for row in rows:
+        metadata = {}
+        raw_payload = str(row["test_result"] or "").strip()
+        if raw_payload:
+            try:
+                parsed = json.loads(raw_payload)
+                if isinstance(parsed, dict):
+                    metadata = parsed
+            except Exception:
+                metadata = {"raw": raw_payload}
+        files_changed = []
+        raw_files = str(row["files_changed"] or "").strip()
+        if raw_files:
+            try:
+                parsed_files = json.loads(raw_files)
+                if isinstance(parsed_files, list):
+                    files_changed = [str(item).strip() for item in parsed_files if str(item).strip()]
+            except Exception:
+                pass
+        results.append(
+            {
+                "id": int(row["id"]),
+                "created_at": str(row["created_at"] or ""),
+                "title": str(row["proposal"] or ""),
+                "reasoning": str(row["reasoning"] or ""),
+                "status": str(row["status"] or ""),
+                "files_changed": files_changed,
+                "metadata": metadata,
+            }
+        )
+    return results
+
+
+def update_public_port_candidate(
+    conn: sqlite3.Connection,
+    log_id: int,
+    *,
+    status: str,
+    metadata_patch: dict | None = None,
+) -> None:
+    row = conn.execute(
+        "SELECT test_result FROM evolution_log WHERE id = ? LIMIT 1",
+        (int(log_id),),
+    ).fetchone()
+    payload: dict = {}
+    if row and str(row["test_result"] or "").strip():
+        try:
+            parsed = json.loads(str(row["test_result"]))
+            if isinstance(parsed, dict):
+                payload = parsed
+        except Exception:
+            payload = {"raw": str(row["test_result"])}
+    if metadata_patch:
+        payload.update(metadata_patch)
+    conn.execute(
+        "UPDATE evolution_log SET status = ?, test_result = ? WHERE id = ?",
+        (status, json.dumps(payload, ensure_ascii=False), int(log_id)),
+    )

package/src/requirements 2.txt

@@ -0,0 +1,14 @@
+# NEXO Brain — runtime dependencies
+# Core (required)
+fastmcp>=2.9.0
+numpy
+tomli; python_version < "3.11"
+
+# Embedding model (optional but recommended for cognitive features)
+fastembed
+
+# Dashboard (optional, only needed for `python -m dashboard.app`)
+fastapi
+uvicorn
+pydantic
+jinja2

package/src/requirements.txt

@@ -2,6 +2,7 @@
 # Core (required)
 fastmcp>=2.9.0
 numpy
+tomli; python_version < "3.11"
 
 # Embedding model (optional but recommended for cognitive features)
 fastembed
@@ -11,4 +12,3 @@ fastapi
 uvicorn
 pydantic
 jinja2
-tomli; python_version < "3.11"

package/src/retroactive_learnings 2.py

@@ -0,0 +1,373 @@
+"""Retroactive application of new learnings to past decisions.
+
+Closes Fase 2 item 3 of NEXO-AUDIT-2026-04-11. The other six items in the
+phase wired existing infrastructure together, but this one was a true
+green-field gap: grep for "retroactive" in src/ returned zero matches
+before this module existed.
+
+The idea is simple. Whenever a new learning lands with a `prevention`
+rule, scan recent decisions and find the ones that would have been
+decided differently under the new rule. Surface each match as a
+deterministic `NF-RETRO-L<learning_id>-D<decision_id>` followup so the
+operator can revisit the call without the system silently mutating any
+historical record.
+
+Why no new schema:
+    Followups already have idempotent INSERT OR REPLACE semantics on the
+    primary id. Using a deterministic id per (learning, decision) pair
+    means re-running the helper is a no-op. There is no need for a
+    `retroactive_learning_matches` table; the followups table is the
+    single source of truth and the existing dashboards already render it.
+
+Matching strategy:
+    Two cheap signals combined into a single score in [0.0, 1.0]:
+      1. applies_to overlap: if the learning lists files / areas / domains
+         in `applies_to`, and the decision's `domain` (or words from it)
+         matches any of those tokens, applies_to_score = 1.0 else 0.0.
+      2. keyword overlap: significant tokens (>= 4 chars, not stopwords)
+         from the learning's title + content + prevention, intersected
+         with significant tokens from the decision's
+         decision + based_on + alternatives + context_ref. Score is
+         intersection_size / max(1, learning_token_count) clipped to 1.0.
+    Guardrail: if a learning defines `applies_to` and that anchor scores
+        below 0.3, auto-dismiss the match even if keyword overlap is high.
+        This blocks keyword-only false positives outside the learning's
+        actual blast radius.
+    Default match threshold: 0.4. Default cap: 5 matches per learning.
+
+Anti-spam guards:
+    - Skip if the learning has no `prevention` (just narrative learnings
+      do not generate retroactive followups — they are not enforceable).
+    - Skip if the learning's status is not 'active'.
+    - Hard cap of `max_matches` followups per call (default 5).
+    - Per (learning, decision) idempotency via deterministic id.
+    - Lookback window default 14 days; configurable.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from typing import Any
+
+# A small stopword list keeps the keyword matcher from picking up filler.
+_STOPWORDS = frozenset({
+    "para", "este", "esta", "esto", "como", "cuando", "donde", "porque",
+    "pero", "tambien", "siempre", "nunca", "antes", "despues", "sobre",
+    "entre", "hacia", "desde", "hasta", "with", "from", "this", "that",
+    "have", "been", "will", "would", "should", "could", "after", "before",
+    "while", "when", "where", "what", "which", "their", "there", "these",
+    "those", "into", "than", "then", "very", "more", "most", "less",
+    "make", "made", "take", "took", "give", "given", "para", "como",
+    "cosa", "todo", "toda", "todos", "todas", "muy",
+})
+
+_TOKEN_RE = re.compile(r"[a-zA-ZáéíóúñÁÉÍÓÚÑ_][a-zA-Z0-9áéíóúñÁÉÍÓÚÑ_]{3,}")
+
+
+def _significant_tokens(text: str) -> set[str]:
+    """Extract significant tokens (>=4 chars, not stopwords, lowercased)."""
+    if not text:
+        return set()
+    tokens = _TOKEN_RE.findall(text)
+    return {t.lower() for t in tokens if t.lower() not in _STOPWORDS and len(t) >= 4}
+
+
+def _split_applies_to(value: str) -> set[str]:
+    """Split a learning's applies_to into normalized tokens."""
+    if not value:
+        return set()
+    pieces: set[str] = set()
+    for chunk in re.split(r"[,;\s]+", value):
+        chunk = chunk.strip().lower()
+        if chunk:
+            pieces.add(chunk)
+            # Also keep the basename so 'src/db/_core.py' matches a domain like '_core'.
+            base = chunk.rsplit("/", 1)[-1]
+            if base and base != chunk:
+                pieces.add(base)
+    return pieces
+
+
+def _decision_text_blob(row: dict) -> str:
+    """Concatenate the searchable text fields of a decision row."""
+    parts = [
+        row.get("decision", "") or "",
+        row.get("based_on", "") or "",
+        row.get("alternatives", "") or "",
+        row.get("context_ref", "") or "",
+        row.get("domain", "") or "",
+    ]
+    return " ".join(parts)
+
+
+def _score_match(
+    *,
+    learning_keywords: set[str],
+    learning_applies_to: set[str],
+    decision_row: dict,
+) -> tuple[float, dict]:
+    """Score how strongly a learning applies retroactively to a decision.
+
+    Returns (score in [0.0, 1.0], breakdown dict for transparency).
+    """
+    decision_blob = _decision_text_blob(decision_row)
+    decision_tokens = _significant_tokens(decision_blob)
+
+    if learning_applies_to:
+        domain_tokens = _significant_tokens(decision_row.get("domain", "") or "")
+        applies_to_hits = learning_applies_to & (domain_tokens | decision_tokens)
+        applies_to_score = 1.0 if applies_to_hits else 0.0
+    else:
+        applies_to_hits = set()
+        applies_to_score = 0.0
+
+    if learning_keywords:
+        keyword_hits = learning_keywords & decision_tokens
+        # Three significant overlapping tokens is a strong signal on its
+        # own — that is the threshold a human reviewer needs to suspect a
+        # rule violation. We score linearly up to that and clip.
+        keyword_score = min(1.0, len(keyword_hits) / 3.0)
+    else:
+        keyword_hits = set()
+        keyword_score = 0.0
+
+    gated_by_applies_to = bool(learning_applies_to and applies_to_score < 0.3)
+    # When a learning explicitly scopes its blast radius via applies_to,
+    # keyword overlap alone is too noisy to justify a retroactive review.
+    score = applies_to_score if gated_by_applies_to else max(applies_to_score, keyword_score)
+    breakdown = {
+        "applies_to_score": round(applies_to_score, 3),
+        "applies_to_hits": sorted(applies_to_hits),
+        "keyword_score": round(keyword_score, 3),
+        "keyword_hits": sorted(keyword_hits),
+        "gated_by_applies_to": gated_by_applies_to,
+    }
+    return round(score, 3), breakdown
+
+
+def _format_followup_description(
+    *,
+    learning: dict,
+    decision: dict,
+    score: float,
+    breakdown: dict,
+) -> str:
+    learning_id = learning.get("id")
+    decision_id = decision.get("id")
+    title = (learning.get("title") or "").strip()
+    prevention = (learning.get("prevention") or "").strip()
+    domain = (decision.get("domain") or "").strip()
+    decision_text = (decision.get("decision") or "").strip()
+    created = (decision.get("created_at") or "").strip()
+
+    lines = [
+        f"Retroactive review: learning #{learning_id} may apply to decision #{decision_id}.",
+        f"Score: {score:.2f} (applies_to={breakdown.get('applies_to_score', 0)}, "
+        f"keyword={breakdown.get('keyword_score', 0)})",
+        "",
+        f"New learning: {title}",
+        f"Prevention rule: {prevention}",
+        "",
+        f"Past decision (#{decision_id}, {domain}, {created}):",
+        f"  {decision_text[:280]}",
+        "",
+        "Action: revisit this decision under the new rule. Update the "
+        "decision row, capture a corrective learning, or close this "
+        "followup as 'still valid' if the rule does not actually conflict.",
+    ]
+    return "\n".join(lines)
+
+
+def apply_learning_retroactively(
+    learning_id: int,
+    *,
+    lookback_days: int = 14,
+    max_matches: int = 5,
+    min_score: float = 0.4,
+    dry_run: bool = False,
+) -> dict:
+    """Scan recent decisions for ones a new learning would re-evaluate.
+
+    Args:
+        learning_id: The learning row id to apply.
+        lookback_days: How many days back to scan decisions (default 14).
+        max_matches: Hard cap of followups created per call (default 5).
+        min_score: Score threshold in [0.0, 1.0] for a match (default 0.4).
+        dry_run: If True, scores matches but does not create followups.
+
+    Returns:
+        {
+          "ok": bool,
+          "learning_id": int,
+          "scanned": int,            # decisions inspected
+          "matched": int,            # decisions scored at or above threshold
+          "followups_created": int,  # actual followup INSERT OR REPLACE rows
+          "skipped_reason": str|None,
+          "matches": [
+              {"decision_id", "score", "breakdown", "followup_id"|None}, ...
+          ],
+        }
+
+    Best-effort: never raises. A failing decision row is logged via the
+    breakdown but does not abort the loop. The full payload is returned so
+    callers (handle_learning_add, MCP tool, tests) can react.
+    """
+    from db import get_db
+
+    base_result: dict[str, Any] = {
+        "ok": True,
+        "learning_id": int(learning_id),
+        "scanned": 0,
+        "matched": 0,
+        "followups_created": 0,
+        "skipped_reason": None,
+        "matches": [],
+    }
+
+    try:
+        conn = get_db()
+    except Exception as e:
+        base_result["ok"] = False
+        base_result["skipped_reason"] = f"cannot open db: {e}"
+        return base_result
+
+    try:
+        learning_row = conn.execute(
+            "SELECT id, category, title, content, prevention, applies_to, status, priority "
+            "FROM learnings WHERE id = ?",
+            (int(learning_id),),
+        ).fetchone()
+    except Exception as e:
+        base_result["ok"] = False
+        base_result["skipped_reason"] = f"learnings query failed: {e}"
+        return base_result
+
+    if not learning_row:
+        base_result["skipped_reason"] = "learning not found"
+        return base_result
+
+    learning = dict(learning_row)
+    if learning.get("status") and learning["status"] != "active":
+        base_result["skipped_reason"] = f"learning status is {learning['status']}, not active"
+        return base_result
+    prevention = (learning.get("prevention") or "").strip()
+    if not prevention:
+        base_result["skipped_reason"] = "learning has no prevention rule — nothing enforceable to apply"
+        return base_result
+
+    learning_keywords = _significant_tokens(
+        " ".join([
+            learning.get("title") or "",
+            learning.get("content") or "",
+            prevention,
+        ])
+    )
+    learning_applies_to = _split_applies_to(learning.get("applies_to") or "")
+
+    if not learning_keywords and not learning_applies_to:
+        base_result["skipped_reason"] = "learning has no usable keywords or applies_to anchors"
+        return base_result
+
+    try:
+        rows = conn.execute(
+            "SELECT id, session_id, created_at, domain, decision, alternatives, "
+            "based_on, confidence, context_ref, outcome, status "
+            "FROM decisions "
+            "WHERE created_at >= datetime('now', ?) "
+            "ORDER BY created_at DESC LIMIT 200",
+            (f"-{max(1, int(lookback_days))} days",),
+        ).fetchall()
+    except Exception as e:
+        base_result["ok"] = False
+        base_result["skipped_reason"] = f"decisions query failed: {e}"
+        return base_result
+
+    matches: list[dict] = []
+    for row in rows:
+        try:
+            decision = dict(row)
+        except Exception:
+            continue
+        base_result["scanned"] += 1
+        score, breakdown = _score_match(
+            learning_keywords=learning_keywords,
+            learning_applies_to=learning_applies_to,
+            decision_row=decision,
+        )
+        if score >= float(min_score):
+            matches.append({
+                "decision": decision,
+                "score": score,
+                "breakdown": breakdown,
+            })
+
+    matches.sort(key=lambda m: m["score"], reverse=True)
+    capped = matches[: max(0, int(max_matches))]
+    base_result["matched"] = len(matches)
+
+    if dry_run:
+        base_result["matches"] = [
+            {
+                "decision_id": int(m["decision"]["id"]),
+                "score": m["score"],
+                "breakdown": m["breakdown"],
+                "followup_id": None,
+            }
+            for m in capped
+        ]
+        return base_result
+
+    created = 0
+    now_epoch = datetime.now().timestamp()
+    summary_matches = []
+    for m in capped:
+        decision = m["decision"]
+        followup_id = f"NF-RETRO-L{learning['id']}-D{decision['id']}"
+        description = _format_followup_description(
+            learning=learning,
+            decision=decision,
+            score=m["score"],
+            breakdown=m["breakdown"],
+        )
+        verification = (
+            f"SELECT id, domain, decision, based_on, status FROM decisions WHERE id = {int(decision['id'])}"
+        )
+        try:
+            conn.execute(
+                "INSERT OR REPLACE INTO followups (id, description, date, status, "
+                "verification, created_at, updated_at, priority) "
+                "VALUES (?, ?, NULL, 'PENDING', ?, ?, ?, ?)",
+                (
+                    followup_id,
+                    description,
+                    verification,
+                    now_epoch,
+                    now_epoch,
+                    learning.get("priority") or "medium",
+                ),
+            )
+            created += 1
+            summary_matches.append({
+                "decision_id": int(decision["id"]),
+                "score": m["score"],
+                "breakdown": m["breakdown"],
+                "followup_id": followup_id,
+            })
+        except Exception as e:
+            summary_matches.append({
+                "decision_id": int(decision.get("id", 0)),
+                "score": m["score"],
+                "breakdown": m["breakdown"],
+                "followup_id": None,
+                "error": str(e),
+            })
+
+    try:
+        conn.commit()
+    except Exception:
+        pass
+
+    base_result["followups_created"] = created
+    base_result["matches"] = summary_matches
+    return base_result