nexo-brain 5.0.4 → 5.1.0

package/src/db/_schema.py

@@ -886,6 +886,56 @@ def _m37_cortex_goal_profile_trace(conn):
     conn.execute("CREATE INDEX IF NOT EXISTS idx_cortex_evaluations_goal_profile ON cortex_evaluations(goal_profile_id)")
 
 
+def _m38_evolution_log_proposal_payload(conn):
+    """Persist the full proposal dict (with `changes` array) so user-approved
+    proposals can be applied by a later cycle.
+
+    Before m38, evolution_log only stored the proposal `action` string. When a
+    user marked a proposal as `accepted` via nexo_evolution_approve, the next
+    cycle had no way to re-execute it because the `changes` operations were
+    discarded after the original cycle. Adding `proposal_payload` (TEXT/JSON)
+    closes that loop and lets _apply_accepted_proposals() in the runner pick
+    up accepted rows and run them through execute_auto_proposal().
+
+    Idempotent and append-only: ALTER TABLE ADD COLUMN is non-destructive in
+    SQLite. Pre-m38 rows keep proposal_payload NULL and are skipped by the
+    apply step (which requires a non-null payload).
+    """
+    _migrate_add_column(conn, "evolution_log", "proposal_payload", "TEXT DEFAULT NULL")
+
+
+def _m39_hook_runs(conn):
+    """Persist hook lifecycle observability — closes Fase 3 item 7.
+
+    Before m39, NEXO had 12 hook scripts (session-start.sh, post-compact.sh,
+    pre-compact.sh, inbox-hook.sh, etc.) but no central record of when they
+    ran, how long they took, or whether they succeeded. The audit lifecycle
+    was a black box. This table is the storage layer that
+    src/hook_observability.py records into and that the new
+    nexo_hook_runs MCP tool reads from.
+
+    Idempotent: CREATE TABLE IF NOT EXISTS plus indexes by hook_name and
+    started_at so the daily query patterns are cheap.
+    """
+    conn.execute(
+        """CREATE TABLE IF NOT EXISTS hook_runs (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            hook_name TEXT NOT NULL,
+            started_at REAL NOT NULL,
+            duration_ms INTEGER NOT NULL DEFAULT 0,
+            exit_code INTEGER NOT NULL DEFAULT 0,
+            status TEXT NOT NULL DEFAULT 'ok',
+            session_id TEXT DEFAULT '',
+            summary TEXT DEFAULT '',
+            metadata TEXT DEFAULT '{}',
+            created_at REAL NOT NULL
+        )"""
+    )
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_hook_runs_hook_name ON hook_runs(hook_name)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_hook_runs_started_at ON hook_runs(started_at)")
+    conn.execute("CREATE INDEX IF NOT EXISTS idx_hook_runs_status ON hook_runs(status)")
+
+
 MIGRATIONS = [
     (1, "learnings_columns", _m1_learnings_columns),
     (2, "followups_reasoning", _m2_followups_reasoning),
@@ -924,6 +974,8 @@ MIGRATIONS = [
     (35, "cortex_evaluation_outcome_link", _m35_cortex_evaluation_outcome_link),
     (36, "goal_profiles", _m36_goal_profiles),
     (37, "cortex_goal_profile_trace", _m37_cortex_goal_profile_trace),
+    (38, "evolution_log_proposal_payload", _m38_evolution_log_proposal_payload),
+    (39, "hook_runs", _m39_hook_runs),
 ]
 
 

package/src/hook_observability.py

@@ -0,0 +1,293 @@
+"""Observability for the NEXO hook lifecycle pipeline.
+
+Closes Fase 3 item 7 of NEXO-AUDIT-2026-04-11. Before this module, NEXO
+had 12 hook scripts (session-start.sh, post-compact.sh, pre-compact.sh,
+inbox-hook.sh, etc.) but no central record of when they ran, how long
+they took, or whether they succeeded. The audit lifecycle was a black box
+— a hook could silently fail for weeks before anyone noticed.
+
+This module is the API layer on top of the m39 hook_runs table:
+
+  record_hook_run(hook_name, ...)  -> int (rowid)
+  list_recent_hook_runs(hours=24, hook_name='', status='', limit=200)
+  hook_health_summary(hours=24)    -> dict with success rate per hook
+
+It is consumed by:
+  - src/scripts/nexo-hook-record.py: a tiny shell-friendly CLI so any
+    bash hook can pipe its result back into the database with one line.
+  - src/server.py:nexo_hook_runs: an MCP tool so the agent can read the
+    hook lifecycle without needing the dashboard.
+
+Best-effort throughout: every helper wraps the DB call in try/except so
+the hook itself never fails because observability could not write.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+import time
+from typing import Any
+
+from db import get_db
+
+
+_VALID_STATUS = {"ok", "error", "skipped", "timeout", "blocked"}
+
+
+def _coerce_status(exit_code: int, status: str = "") -> str:
+    """Derive status from exit_code when not explicitly provided."""
+    s = (status or "").strip().lower()
+    if s in _VALID_STATUS:
+        return s
+    if exit_code == 0:
+        return "ok"
+    return "error"
+
+
+def record_hook_run(
+    hook_name: str,
+    *,
+    started_at: float | None = None,
+    duration_ms: int = 0,
+    exit_code: int = 0,
+    status: str = "",
+    session_id: str = "",
+    summary: str = "",
+    metadata: dict | None = None,
+) -> int:
+    """Insert a single row into hook_runs and return its id.
+
+    Args:
+        hook_name: The hook identifier (e.g. 'session-start', 'post-compact').
+        started_at: Unix epoch when the hook started. Defaults to now.
+        duration_ms: Wall-clock duration in milliseconds.
+        exit_code: Process exit code (0 = ok). When status is empty, it is
+            derived from this value.
+        status: One of {ok, error, skipped, timeout, blocked}. Optional.
+        session_id: Claude Code session id when known.
+        summary: Short human-readable summary (truncated to 500 chars).
+        metadata: Extra JSON-serializable payload (truncated to 4 KB serialized).
+
+    Returns:
+        The new hook_runs row id, or 0 if the insert failed.
+
+    This helper never raises. A failure here must never block the hook.
+    """
+    name = (hook_name or "").strip()
+    if not name:
+        return 0
+    if started_at is None:
+        started_at = time.time()
+    try:
+        duration_ms = max(0, int(duration_ms))
+    except (TypeError, ValueError):
+        duration_ms = 0
+    try:
+        exit_code = int(exit_code)
+    except (TypeError, ValueError):
+        exit_code = 0
+    final_status = _coerce_status(exit_code, status)
+    summary_clean = (summary or "")[:500]
+    try:
+        metadata_blob = json.dumps(metadata or {}, ensure_ascii=False)
+    except Exception:
+        metadata_blob = "{}"
+    if len(metadata_blob) > 4096:
+        metadata_blob = metadata_blob[:4096]
+    now_epoch = time.time()
+    try:
+        conn = get_db()
+        cur = conn.execute(
+            "INSERT INTO hook_runs (hook_name, started_at, duration_ms, exit_code, "
+            "status, session_id, summary, metadata, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (
+                name[:120],
+                float(started_at),
+                duration_ms,
+                exit_code,
+                final_status,
+                (session_id or "")[:80],
+                summary_clean,
+                metadata_blob,
+                now_epoch,
+            ),
+        )
+        try:
+            conn.commit()
+        except Exception:
+            pass
+        return int(cur.lastrowid or 0)
+    except Exception:
+        return 0
+
+
+def list_recent_hook_runs(
+    *,
+    hours: int = 24,
+    hook_name: str = "",
+    status: str = "",
+    limit: int = 200,
+) -> list[dict]:
+    """Return recent hook_runs filtered by time window, name, and status.
+
+    Args:
+        hours: How far back to look. Default 24h.
+        hook_name: Optional substring filter on hook_name (LIKE %name%).
+        status: Optional exact match on status field.
+        limit: Max rows. Default 200.
+
+    Returns ordered list (newest first) of dicts. Empty list on any error.
+    """
+    try:
+        cutoff = time.time() - max(60, int(hours)) * 3600
+    except (TypeError, ValueError):
+        cutoff = time.time() - 86400
+    clauses = ["started_at >= ?"]
+    params: list[Any] = [cutoff]
+    if hook_name:
+        clauses.append("hook_name LIKE ?")
+        params.append(f"%{hook_name.strip()}%")
+    if status:
+        clauses.append("status = ?")
+        params.append(status.strip().lower())
+    where = " AND ".join(clauses)
+    try:
+        conn = get_db()
+        rows = conn.execute(
+            f"SELECT id, hook_name, started_at, duration_ms, exit_code, status, "
+            f"session_id, summary, metadata, created_at FROM hook_runs "
+            f"WHERE {where} ORDER BY started_at DESC LIMIT ?",
+            params + [max(1, int(limit))],
+        ).fetchall()
+    except Exception:
+        return []
+    result = []
+    for row in rows:
+        d = dict(row)
+        try:
+            d["metadata"] = json.loads(d.get("metadata") or "{}")
+        except Exception:
+            d["metadata"] = {}
+        result.append(d)
+    return result
+
+
+def hook_health_summary(hours: int = 24) -> dict:
+    """Aggregate per-hook health stats over a time window.
+
+    Returns dict with shape:
+        {
+          "window_hours": N,
+          "total_runs": N,
+          "by_hook": [
+            {"hook_name": str, "runs": int, "ok": int, "errors": int,
+             "p50_duration_ms": int, "p95_duration_ms": int,
+             "success_rate": float (0..1), "last_run_at": float},
+            ...
+          ],
+          "unhealthy_hooks": [hook_name, ...]   # success rate < 0.8 with >= 3 runs
+        }
+
+    Used by:
+      - nexo_hook_runs MCP tool
+      - dashboard widgets
+      - the daily self-audit's hook health column
+    """
+    try:
+        cutoff = time.time() - max(60, int(hours)) * 3600
+    except (TypeError, ValueError):
+        cutoff = time.time() - 86400
+    try:
+        conn = get_db()
+        rows = conn.execute(
+            "SELECT hook_name, status, duration_ms, started_at "
+            "FROM hook_runs WHERE started_at >= ? ORDER BY hook_name, started_at",
+            (cutoff,),
+        ).fetchall()
+    except Exception:
+        return {"window_hours": hours, "total_runs": 0, "by_hook": [], "unhealthy_hooks": []}
+
+    by_hook: dict[str, dict] = {}
+    for row in rows:
+        name = row["hook_name"]
+        bucket = by_hook.setdefault(
+            name,
+            {"hook_name": name, "runs": 0, "ok": 0, "errors": 0, "_durations": [], "last_run_at": 0.0},
+        )
+        bucket["runs"] += 1
+        status = row["status"]
+        if status == "ok":
+            bucket["ok"] += 1
+        elif status in {"error", "timeout", "blocked"}:
+            bucket["errors"] += 1
+        bucket["_durations"].append(int(row["duration_ms"] or 0))
+        if row["started_at"] > bucket["last_run_at"]:
+            bucket["last_run_at"] = float(row["started_at"])
+
+    summary_rows = []
+    unhealthy = []
+    for name, bucket in by_hook.items():
+        durations = sorted(bucket.pop("_durations"))
+        n = len(durations)
+        if n:
+            p50 = durations[n // 2]
+            p95 = durations[min(n - 1, int(n * 0.95))]
+        else:
+            p50 = p95 = 0
+        success_rate = (bucket["ok"] / bucket["runs"]) if bucket["runs"] else 0.0
+        bucket["p50_duration_ms"] = p50
+        bucket["p95_duration_ms"] = p95
+        bucket["success_rate"] = round(success_rate, 3)
+        summary_rows.append(bucket)
+        if bucket["runs"] >= 3 and success_rate < 0.8:
+            unhealthy.append(name)
+
+    summary_rows.sort(key=lambda b: b["runs"], reverse=True)
+    return {
+        "window_hours": hours,
+        "total_runs": sum(b["runs"] for b in summary_rows),
+        "by_hook": summary_rows,
+        "unhealthy_hooks": unhealthy,
+    }
+
+
+def main_cli(argv: list[str]) -> int:
+    """Tiny CLI shim so bash hooks can call this module directly.
+
+    Usage from a hook:
+      python3 -m hook_observability record \
+          --hook session-start --duration-ms 142 --exit 0 --session abc
+
+    The first positional verb selects the action (`record` only for now).
+    Returns 0 always — the recorder must never break the hook itself.
+    """
+    if len(argv) < 1 or argv[0] != "record":
+        print("usage: hook_observability record --hook NAME [--duration-ms N] [--exit N] [--session SID] [--summary TEXT]")
+        return 0
+    args: dict[str, str] = {}
+    i = 1
+    while i < len(argv):
+        token = argv[i]
+        if token.startswith("--") and i + 1 < len(argv):
+            args[token[2:]] = argv[i + 1]
+            i += 2
+        else:
+            i += 1
+    try:
+        record_hook_run(
+            args.get("hook", ""),
+            duration_ms=int(args.get("duration-ms", "0") or 0),
+            exit_code=int(args.get("exit", "0") or 0),
+            status=args.get("status", ""),
+            session_id=args.get("session", ""),
+            summary=args.get("summary", ""),
+        )
+    except Exception:
+        pass
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main_cli(sys.argv[1:]))

package/src/hooks/session-start.sh

@@ -4,6 +4,33 @@
 # Caches output for 1 hour to avoid regenerating on rapid successive sessions.
 set -uo pipefail
 
+# Fase 3 item 7: hook lifecycle observability — record duration + exit code
+# in hook_runs on EXIT. Best-effort: a failure here must not break the hook.
+NEXO_HOOK_START_MS=$(python3 -c "import time; print(int(time.time()*1000))" 2>/dev/null || echo 0)
+NEXO_HOOK_NAME="session-start"
+_nexo_record_hook_run() {
+    local exit_code=$?
+    local duration_ms=0
+    if [ "$NEXO_HOOK_START_MS" != "0" ]; then
+        local now_ms
+        now_ms=$(python3 -c "import time; print(int(time.time()*1000))" 2>/dev/null || echo 0)
+        if [ "$now_ms" != "0" ]; then
+            duration_ms=$((now_ms - NEXO_HOOK_START_MS))
+        fi
+    fi
+    local recorder
+    recorder="${NEXO_CODE:-/Users/franciscoc/Documents/_PhpstormProjects/nexo/src}/scripts/nexo-hook-record.py"
+    if [ -f "$recorder" ]; then
+        python3 "$recorder" record \
+            --hook "$NEXO_HOOK_NAME" \
+            --duration-ms "$duration_ms" \
+            --exit "$exit_code" \
+            --session "${CLAUDE_SID:-}" \
+            >/dev/null 2>&1 || true
+    fi
+}
+trap _nexo_record_hook_run EXIT
+
 NEXO_HOME="${NEXO_HOME:-$HOME/.nexo}"
 BRIEFING_FILE="$NEXO_HOME/coordination/session-briefing.txt"
 MAX_AGE_SECONDS=3600  # 1 hour cache

package/src/knowledge_graph.py

@@ -255,3 +255,182 @@ def extract_subgraph(center_id: int, depth: int = 2) -> dict:
     d3_edges = [{"source": e["source_id"], "target": e["target_id"],
                  "relation": e["relation"], "weight": e["weight"]} for e in graph["edges"]]
     return {"nodes": d3_nodes, "edges": d3_edges}
+
+
+# ── Bitemporal export — Fase 5 item 1 ────────────────────────────────────
+#
+# The KG is bi-temporal by design: kg_edges has valid_from and valid_until
+# columns and the upsert_edge / delete_edge helpers maintain them
+# correctly. The audit's "exportable" requirement asked for emitting the
+# graph to standard interchange formats so external tools can ingest it
+# without speaking SQLite. The two helpers below cover the canonical
+# choices: JSON-LD (semantic web, human-readable) and GraphML (igraph,
+# Gephi, NetworkX, Cytoscape).
+#
+# Both helpers respect the bitemporal model: when as_of is None, only
+# active edges (valid_until IS NULL) are emitted. When as_of is a
+# timestamp string, the historical state at that instant is emitted.
+
+import json as _json
+
+
+def export_to_jsonld(*, as_of: str = "") -> dict:
+    """Export the active or historical KG to a JSON-LD document.
+
+    The vocabulary lives under https://nexo-brain.com/kg/v1# so external
+    tools can resolve types and relations consistently. Each node becomes
+    a top-level @graph entry with @id = nexo:node:<id> and @type =
+    nexo:<node_type>. Each edge becomes a relation property on its source
+    node, plus a parallel @reverse on the target so the JSON-LD remains
+    fully traversable.
+
+    Args:
+        as_of: ISO timestamp. If empty, exports active edges only
+               (valid_until IS NULL). If provided, exports the snapshot
+               that was valid at that instant via temporal range query.
+
+    Returns a JSON-LD-shaped dict ready for json.dumps().
+    """
+    db = _get_db()
+    # kg_nodes is NOT bitemporal — only kg_edges has valid_from/valid_until.
+    # The audit's "bitemporal" requirement is satisfied at the edge level
+    # because nodes are stable identities while edges encode the temporal
+    # facts (relationships valid during a time window).
+    nodes = [dict(row) for row in db.execute(
+        "SELECT id, node_type, node_ref, label, properties FROM kg_nodes"
+    ).fetchall()]
+
+    if as_of and as_of.strip():
+        edge_rows = db.execute(
+            "SELECT id, source_id, target_id, relation, weight, confidence, "
+            "valid_from, valid_until, properties FROM kg_edges "
+            "WHERE valid_from <= ? AND (valid_until IS NULL OR valid_until > ?)",
+            (as_of, as_of),
+        ).fetchall()
+    else:
+        edge_rows = db.execute(
+            "SELECT id, source_id, target_id, relation, weight, confidence, "
+            "valid_from, valid_until, properties FROM kg_edges WHERE valid_until IS NULL"
+        ).fetchall()
+    edges = [dict(row) for row in edge_rows]
+
+    nodes_by_id: dict[int, dict] = {}
+    for n in nodes:
+        try:
+            props = _json.loads(n.get("properties") or "{}")
+        except Exception:
+            props = {}
+        nodes_by_id[n["id"]] = {
+            "@id": f"nexo:node:{n['id']}",
+            "@type": f"nexo:{n['node_type']}",
+            "label": n.get("label") or "",
+            "node_ref": n.get("node_ref") or "",
+            "properties": props,
+        }
+
+    for e in edges:
+        src_id = e["source_id"]
+        tgt_id = e["target_id"]
+        if src_id not in nodes_by_id or tgt_id not in nodes_by_id:
+            continue  # orphan edge — skip
+        relation_key = f"nexo:{e['relation']}"
+        edge_payload = {
+            "@id": f"nexo:edge:{e['id']}",
+            "target": f"nexo:node:{tgt_id}",
+            "weight": float(e.get("weight") or 0.0),
+            "confidence": float(e.get("confidence") or 0.0),
+            "valid_from": e.get("valid_from"),
+            "valid_until": e.get("valid_until"),
+        }
+        nodes_by_id[src_id].setdefault(relation_key, []).append(edge_payload)
+
+    snapshot_label = as_of.strip() if as_of and as_of.strip() else "active"
+    return {
+        "@context": {
+            "nexo": "https://nexo-brain.com/kg/v1#",
+            "label": "https://nexo-brain.com/kg/v1#label",
+            "node_ref": "https://nexo-brain.com/kg/v1#node_ref",
+            "weight": "https://nexo-brain.com/kg/v1#weight",
+            "confidence": "https://nexo-brain.com/kg/v1#confidence",
+            "valid_from": "https://nexo-brain.com/kg/v1#valid_from",
+            "valid_until": "https://nexo-brain.com/kg/v1#valid_until",
+            "properties": "https://nexo-brain.com/kg/v1#properties",
+        },
+        "@type": "nexo:KnowledgeGraphSnapshot",
+        "snapshot": snapshot_label,
+        "node_count": len(nodes_by_id),
+        "edge_count": len(edges),
+        "@graph": list(nodes_by_id.values()),
+    }
+
+
+def export_to_graphml(*, as_of: str = "") -> str:
+    """Export the active or historical KG to a GraphML XML string.
+
+    GraphML is the canonical interchange for igraph, Gephi, NetworkX, and
+    Cytoscape. Bitemporal columns are emitted as edge data attributes so
+    importers that support them (Gephi temporal layouts, NetworkX
+    DiGraph) can render the historical view.
+
+    Args:
+        as_of: ISO timestamp. Same semantics as export_to_jsonld.
+
+    Returns a string with a valid GraphML 1.1 document.
+    """
+    db = _get_db()
+    nodes = [dict(row) for row in db.execute(
+        "SELECT id, node_type, node_ref, label FROM kg_nodes"
+    ).fetchall()]
+    if as_of and as_of.strip():
+        edge_rows = db.execute(
+            "SELECT id, source_id, target_id, relation, weight, valid_from, valid_until FROM kg_edges "
+            "WHERE valid_from <= ? AND (valid_until IS NULL OR valid_until > ?)",
+            (as_of, as_of),
+        ).fetchall()
+    else:
+        edge_rows = db.execute(
+            "SELECT id, source_id, target_id, relation, weight, valid_from, valid_until FROM kg_edges "
+            "WHERE valid_until IS NULL"
+        ).fetchall()
+
+    def _xml_escape(value: object) -> str:
+        text = "" if value is None else str(value)
+        return (
+            text.replace("&", "&amp;")
+            .replace("<", "&lt;")
+            .replace(">", "&gt;")
+            .replace('"', "&quot;")
+            .replace("'", "&apos;")
+        )
+
+    out: list[str] = []
+    out.append('<?xml version="1.0" encoding="UTF-8"?>')
+    out.append('<graphml xmlns="http://graphml.graphdrawing.org/xmlns">')
+    out.append('  <key id="label" for="node" attr.name="label" attr.type="string"/>')
+    out.append('  <key id="node_type" for="node" attr.name="node_type" attr.type="string"/>')
+    out.append('  <key id="node_ref" for="node" attr.name="node_ref" attr.type="string"/>')
+    out.append('  <key id="relation" for="edge" attr.name="relation" attr.type="string"/>')
+    out.append('  <key id="weight" for="edge" attr.name="weight" attr.type="double"/>')
+    out.append('  <key id="valid_from" for="edge" attr.name="valid_from" attr.type="string"/>')
+    out.append('  <key id="valid_until" for="edge" attr.name="valid_until" attr.type="string"/>')
+    snapshot_label = as_of.strip() if as_of and as_of.strip() else "active"
+    out.append(f'  <graph id="nexo_kg_{_xml_escape(snapshot_label)}" edgedefault="directed">')
+    for n in nodes:
+        out.append(f'    <node id="n{n["id"]}">')
+        out.append(f'      <data key="label">{_xml_escape(n.get("label"))}</data>')
+        out.append(f'      <data key="node_type">{_xml_escape(n.get("node_type"))}</data>')
+        out.append(f'      <data key="node_ref">{_xml_escape(n.get("node_ref"))}</data>')
+        out.append('    </node>')
+    for e in edge_rows:
+        out.append(
+            f'    <edge id="e{e["id"]}" source="n{e["source_id"]}" target="n{e["target_id"]}">'
+        )
+        out.append(f'      <data key="relation">{_xml_escape(e["relation"])}</data>')
+        out.append(f'      <data key="weight">{float(e["weight"] or 0.0)}</data>')
+        out.append(f'      <data key="valid_from">{_xml_escape(e["valid_from"])}</data>')
+        if e["valid_until"]:
+            out.append(f'      <data key="valid_until">{_xml_escape(e["valid_until"])}</data>')
+        out.append('    </edge>')
+    out.append('  </graph>')
+    out.append('</graphml>')
+    return "\n".join(out)