nexo-brain 2.2.0 → 2.3.1

package/src/scripts/nexo-deep-sleep.sh

@@ -1,13 +1,11 @@
 #!/bin/bash
-# NEXO Deep Sleep — Complete overnight session analysis
+# NEXO Deep Sleep — Overnight session analysis with watermark tracking
 # Runs at 4:30 AM via LaunchAgent
-# Reads ALL session transcripts from the day, analyzes with Claude CLI,
-# and applies findings (learnings, feedbacks, followups, trust adjustments)
 #
-# Features:
-# - Catch-up: if yesterday was missed (Mac off/asleep), runs it first
-# - Logs to $NEXO_HOME/logs/deep-sleep.log
-# - Marks completion in .last-run for watchdog monitoring
+# Watermark approach: tracks the last processed timestamp so nothing is missed.
+# Sessions from late-night/early-morning work are included in the next run.
+#
+# Logs to $NEXO_HOME/logs/deep-sleep.log
 
 set -euo pipefail
 
@@ -15,83 +13,74 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 NEXO_HOME="${NEXO_HOME:-$HOME/.nexo}"
 LOG_DIR="$NEXO_HOME/logs"
 DEEP_SLEEP_DIR="$NEXO_HOME/operations/deep-sleep"
-LAST_RUN_FILE="$DEEP_SLEEP_DIR/.last-run"
-TODAY=$(date +%Y-%m-%d)
+WATERMARK_FILE="$DEEP_SLEEP_DIR/.watermark"
+RUN_ID=$(date +%Y-%m-%d)
 
 mkdir -p "$LOG_DIR" "$DEEP_SLEEP_DIR"
 
 log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_DIR/deep-sleep.log"; }
 
-run_analysis() {
-    local DATE="$1"
-    log "=== Deep Sleep v2 starting for $DATE ==="
-
-    # Phase 1: Collect all context (Python, no LLM)
-    log "Phase 1: Collecting context for $DATE..."
-    python3 "$SCRIPT_DIR/deep-sleep/collect.py" "$DATE" 2>&1 | tee -a "$LOG_DIR/deep-sleep.log"
-
-    if [ ! -f "$DEEP_SLEEP_DIR/$DATE-context.txt" ]; then
-        log "No context file generated for $DATE. Skipping."
-        return 0
-    fi
-
-    # Check meta for session count
-    SESSIONS=0
-    if [ -f "$DEEP_SLEEP_DIR/$DATE-meta.json" ]; then
-        SESSIONS=$(python3 -c "import json; print(json.load(open('$DEEP_SLEEP_DIR/$DATE-meta.json'))['sessions_found'])")
-    elif [ -f "$DEEP_SLEEP_DIR/$DATE-index.json" ]; then
-        SESSIONS=$(python3 -c "import json; print(json.load(open('$DEEP_SLEEP_DIR/$DATE-index.json'))['sessions_found'])")
-    fi
-    if [ "$SESSIONS" -eq 0 ]; then
-        log "No sessions found for $DATE. Skipping."
-        return 0
-    fi
-
-    # Phase 2: Extract findings per session (Claude Opus)
-    log "Phase 2: Extracting findings from $SESSIONS sessions..."
-    python3 "$SCRIPT_DIR/deep-sleep/extract.py" "$DATE" 2>&1 | tee -a "$LOG_DIR/deep-sleep.log"
-
-    if [ ! -f "$DEEP_SLEEP_DIR/$DATE-extractions.json" ]; then
-        log "Extraction failed for $DATE. No output."
-        return 1
-    fi
-
-    # Phase 3: Cross-session synthesis (Claude Opus, one call)
-    log "Phase 3: Synthesizing cross-session findings..."
-    python3 "$SCRIPT_DIR/deep-sleep/synthesize.py" "$DATE" 2>&1 | tee -a "$LOG_DIR/deep-sleep.log"
-
-    if [ ! -f "$DEEP_SLEEP_DIR/$DATE-synthesis.json" ]; then
-        log "Synthesis failed for $DATE. Falling back to extractions only."
-        # Fall back: apply extractions directly
-        cp "$DEEP_SLEEP_DIR/$DATE-extractions.json" "$DEEP_SLEEP_DIR/$DATE-synthesis.json"
-    fi
-
-    # Phase 4: Apply findings
-    log "Phase 4: Applying findings..."
-    python3 "$SCRIPT_DIR/deep-sleep/apply_findings.py" "$DATE" 2>&1 | tee -a "$LOG_DIR/deep-sleep.log"
-
-    log "=== Deep Sleep v2 complete for $DATE ==="
-    return 0
-}
-
-# --- Catch-up: check if the day before yesterday was missed ---
-YESTERDAY=$(date -v-1d +%Y-%m-%d 2>/dev/null || date -d "yesterday" +%Y-%m-%d 2>/dev/null)
-DAY_BEFORE=$(date -v-2d +%Y-%m-%d 2>/dev/null || date -d "2 days ago" +%Y-%m-%d 2>/dev/null)
-LAST_RUN=""
-if [ -f "$LAST_RUN_FILE" ]; then
-    LAST_RUN=$(cat "$LAST_RUN_FILE")
+# Read watermark (last processed timestamp)
+SINCE=""
+if [ -f "$WATERMARK_FILE" ]; then
+    SINCE=$(cat "$WATERMARK_FILE")
+    log "Watermark: processing sessions since $SINCE"
+else
+    # First run ever: process last 48h
+    SINCE=$(date -v-2d '+%Y-%m-%dT%H:%M:%S' 2>/dev/null || date -d "2 days ago" '+%Y-%m-%dT%H:%M:%S' 2>/dev/null)
+    log "No watermark found. First run, collecting since $SINCE"
 fi
 
-if [ -n "$DAY_BEFORE" ] && [ "$LAST_RUN" != "$DAY_BEFORE" ] && [ "$LAST_RUN" != "$YESTERDAY" ]; then
-    # Day before yesterday wasn't analyzed — catch up
-    if [ ! -f "$DEEP_SLEEP_DIR/$DAY_BEFORE-analysis.json" ]; then
-        log "*** CATCH-UP: $DAY_BEFORE was missed. Running now. ***"
-        run_analysis "$DAY_BEFORE" || log "Catch-up for $DAY_BEFORE failed."
-    fi
+UNTIL=$(date '+%Y-%m-%dT%H:%M:%S')
+
+log "=== Deep Sleep v2 starting (run_id=$RUN_ID) ==="
+
+# Phase 1: Collect all context (Python, no LLM)
+log "Phase 1: Collecting context since $SINCE until $UNTIL..."
+python3 "$SCRIPT_DIR/deep-sleep/collect.py" "$RUN_ID" "$SINCE" "$UNTIL" >> "$LOG_DIR/deep-sleep.log" 2>&1
+
+if [ ! -f "$DEEP_SLEEP_DIR/$RUN_ID-context.txt" ]; then
+    log "No context file generated. Skipping."
+    echo "$UNTIL" > "$WATERMARK_FILE"
+    log "Watermark updated to $UNTIL (no sessions to process)"
+    exit 0
+fi
+
+# Check meta for session count
+SESSIONS=0
+if [ -f "$DEEP_SLEEP_DIR/$RUN_ID-meta.json" ]; then
+    SESSIONS=$(python3 -c "import json; print(json.load(open('$DEEP_SLEEP_DIR/$RUN_ID-meta.json'))['sessions_found'])")
+fi
+if [ "$SESSIONS" -eq 0 ]; then
+    log "No sessions found. Skipping."
+    echo "$UNTIL" > "$WATERMARK_FILE"
+    log "Watermark updated to $UNTIL (no sessions)"
+    exit 0
+fi
+
+# Phase 2: Extract findings per session (Claude Opus)
+log "Phase 2: Extracting findings from $SESSIONS sessions..."
+python3 "$SCRIPT_DIR/deep-sleep/extract.py" "$RUN_ID" >> "$LOG_DIR/deep-sleep.log" 2>&1
+
+if [ ! -f "$DEEP_SLEEP_DIR/$RUN_ID-extractions.json" ]; then
+    log "Extraction failed. Watermark NOT updated (will retry next run)."
+    exit 1
+fi
+
+# Phase 3: Cross-session synthesis (Claude Opus, one call)
+log "Phase 3: Synthesizing cross-session findings..."
+python3 "$SCRIPT_DIR/deep-sleep/synthesize.py" "$RUN_ID" >> "$LOG_DIR/deep-sleep.log" 2>&1
+
+if [ ! -f "$DEEP_SLEEP_DIR/$RUN_ID-synthesis.json" ]; then
+    log "Synthesis failed. Falling back to extractions only."
+    cp "$DEEP_SLEEP_DIR/$RUN_ID-extractions.json" "$DEEP_SLEEP_DIR/$RUN_ID-synthesis.json"
 fi
 
-# --- Run yesterday's analysis (main task — at 4:30 AM, today has no sessions yet) ---
-run_analysis "$YESTERDAY"
+# Phase 4: Apply findings
+log "Phase 4: Applying findings..."
+python3 "$SCRIPT_DIR/deep-sleep/apply_findings.py" "$RUN_ID" >> "$LOG_DIR/deep-sleep.log" 2>&1
 
-# Mark completion with yesterday's date (what we actually analyzed)
-echo "$YESTERDAY" > "$LAST_RUN_FILE"
+# Update watermark on success
+echo "$UNTIL" > "$WATERMARK_FILE"
+log "Watermark updated to $UNTIL"
+log "=== Deep Sleep v2 complete (run_id=$RUN_ID) ==="

package/src/scripts/nexo-evolution-run.py

@@ -104,6 +104,19 @@ CLI_TIMEOUT = 21600  # 3h safety net (prevents zombie processes)
 
 
 def verify_claude_cli() -> bool:
+    """Check Claude CLI is available and authenticated."""
+    if not CLAUDE_CLI.exists():
+        return False
+    try:
+        result = subprocess.run(
+            [str(CLAUDE_CLI), "-p", "reply OK", "--output-format", "text"],
+            capture_output=True, text=True, timeout=30
+        )
+        return result.returncode == 0
+    except Exception:
+        return False
+
+
 def call_claude_cli(prompt: str) -> str:
     """Call claude -p prompt --model opus via subprocess. Returns stdout text."""
     env = os.environ.copy()

package/src/scripts/nexo-learning-housekeep.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 """NEXO Learning Housekeeping — Nightly dedup, weight adjustment, and review.
 
 Runs daily. Adjusts learning weights based on usage (guard_hits),
@@ -31,6 +32,7 @@ MIN_WEIGHT = 0.05
 MAX_WEIGHT = 1.0
 DEDUP_THRESHOLD = 0.85        # cosine similarity for duplicate detection
 ARCHIVE_AFTER_DAYS = 90       # archive if weight < 0.1 and no hits in this many days
+REVIEW_EXTEND_DAYS = 30       # extend review_due by this many days when confirming
 
 
 def get_db():
@@ -195,6 +197,157 @@ def archive_stale(conn):
     return len(stale)
 
 
+def _reconcile_decision_outcome(conn, decision_id: int, decision_text: str) -> str | None:
+    """Try to find evidence of a decision's outcome in diaries, followups, and change_log.
+
+    Returns outcome text if found, None otherwise.
+    """
+    # Extract keywords from the decision for matching
+    keywords = [w for w in decision_text.lower().split() if len(w) > 4][:5]
+    if not keywords:
+        return None
+
+    like_clauses = " OR ".join(f"summary LIKE ?" for _ in keywords)
+    like_params = [f"%{kw}%" for kw in keywords]
+
+    # Check session diaries for evidence
+    diary_match = conn.execute(
+        f"SELECT summary FROM session_diary WHERE ({like_clauses}) "
+        "AND created_at > (SELECT created_at FROM decisions WHERE id = ?) "
+        "ORDER BY created_at DESC LIMIT 1",
+        like_params + [decision_id]
+    ).fetchone()
+    if diary_match:
+        return f"[auto-reconciled from diary] {diary_match['summary'][:200]}"
+
+    # Check completed followups
+    like_clauses_f = " OR ".join(f"description LIKE ?" for _ in keywords)
+    followup_match = conn.execute(
+        f"SELECT description, verification FROM followups WHERE status = 'COMPLETED' "
+        f"AND ({like_clauses_f}) ORDER BY date DESC LIMIT 1",
+        like_params
+    ).fetchone()
+    if followup_match:
+        result = followup_match['verification'] or followup_match['description']
+        return f"[auto-reconciled from followup] {result[:200]}"
+
+    # Check change_log (schema: what_changed, why, commit_ref, affects)
+    like_clauses_c = " OR ".join(f"what_changed LIKE ?" for _ in keywords)
+    change_match = conn.execute(
+        f"SELECT what_changed, why, commit_ref FROM change_log WHERE ({like_clauses_c}) "
+        "ORDER BY created_at DESC LIMIT 1",
+        like_params
+    ).fetchone()
+    if change_match:
+        ref = change_match['commit_ref'] or ''
+        desc = change_match['what_changed'] or change_match['why'] or ''
+        return f"[auto-reconciled from change_log] {desc[:150]} {ref}"
+
+    return None
+
+
+def process_overdue_reviews(conn):
+    """Process learnings and decisions whose review_due_at has passed.
+
+    Learnings:
+      - guard_hits > 5 since last review -> confirm (extend review_due by 30 days)
+      - guard_hits = 0 and weight < 0.3 -> archive
+      - otherwise -> extend review_due by 30 days (still useful, just not urgent)
+
+    Decisions:
+      - status = 'pending_review' and review_due_at < now -> archive if >30 days old
+    """
+    ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    now = time.time()
+    now_iso = datetime.now().isoformat(timespec="seconds")
+
+    # --- Overdue learnings ---
+    try:
+        overdue_learnings = conn.execute(
+            "SELECT id, title, weight, guard_hits, review_due_at, last_reviewed_at "
+            "FROM learnings "
+            "WHERE review_due_at IS NOT NULL AND review_due_at <= ? AND status = 'active'",
+            (now,)
+        ).fetchall()
+    except Exception as e:
+        print(f"[{ts}] Overdue reviews: error querying learnings: {e}")
+        return 0
+
+    confirmed = 0
+    archived = 0
+    for l in overdue_learnings:
+        lid = l["id"]
+        hits = l["guard_hits"] or 0
+        weight = l["weight"] or 0.5
+        last_reviewed = l["last_reviewed_at"] or 0
+
+        if hits > 5:
+            # Active and useful -- confirm: extend review date
+            new_due = now + (REVIEW_EXTEND_DAYS * 86400)
+            conn.execute(
+                "UPDATE learnings SET review_due_at = ?, last_reviewed_at = ? WHERE id = ?",
+                (new_due, now, lid)
+            )
+            confirmed += 1
+        elif hits == 0 and weight < 0.3:
+            # Unused and low weight -- archive
+            conn.execute(
+                "UPDATE learnings SET status = 'archived' WHERE id = ?",
+                (lid,)
+            )
+            archived += 1
+            print(f"[{ts}]   Archived overdue learning #{lid} '{l['title'][:50]}' (hits=0, weight={weight:.2f})")
+        else:
+            # Middle ground -- extend review date, keep active
+            new_due = now + (REVIEW_EXTEND_DAYS * 86400)
+            conn.execute(
+                "UPDATE learnings SET review_due_at = ?, last_reviewed_at = ? WHERE id = ?",
+                (new_due, now, lid)
+            )
+            confirmed += 1
+
+    # --- Overdue decisions ---
+    decision_archived = 0
+    try:
+        cutoff_30d = (datetime.now() - timedelta(days=30)).isoformat(timespec="seconds")
+        overdue_decisions = conn.execute(
+            "SELECT id, decision, created_at FROM decisions "
+            "WHERE status = 'pending_review' AND review_due_at IS NOT NULL AND review_due_at <= ?",
+            (now_iso,)
+        ).fetchall()
+
+        for d in overdue_decisions:
+            did = d["id"]
+            created = d["created_at"] or ""
+            decision_text = d["decision"] or ""
+
+            # Try to reconcile outcome from diaries, followups, change_log
+            outcome = _reconcile_decision_outcome(conn, did, decision_text)
+            if outcome:
+                conn.execute(
+                    "UPDATE decisions SET status = 'resolved', outcome = ? WHERE id = ?",
+                    (outcome, did)
+                )
+                decision_archived += 1
+                print(f"[{ts}]   Resolved decision #{did} '{decision_text[:50]}' — outcome found in logs")
+            elif created < cutoff_30d:
+                conn.execute(
+                    "UPDATE decisions SET status = 'archived' WHERE id = ?",
+                    (did,)
+                )
+                decision_archived += 1
+                print(f"[{ts}]   Archived decision #{did} '{decision_text[:50]}' (>30d, no outcome found)")
+    except Exception as e:
+        print(f"[{ts}] Overdue reviews: error processing decisions: {e}")
+
+    conn.commit()
+    total_learnings = len(overdue_learnings) if 'overdue_learnings' in dir() else 0
+    total_decisions = len(overdue_decisions) if 'overdue_decisions' in dir() else 0
+    print(f"[{ts}] Overdue reviews: {total_learnings} learnings ({confirmed} confirmed, {archived} archived), "
+          f"{total_decisions} decisions ({decision_archived} archived)")
+    return confirmed + archived + decision_archived
+
+
 def print_summary(conn):
     """Print summary stats."""
     ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -233,7 +386,10 @@ def main():
     # 4. Archive stale learnings
     archive_stale(conn)
 
-    # 5. Summary
+    # 5. Process overdue reviews (review_due_at < now)
+    process_overdue_reviews(conn)
+
+    # 6. Summary
     print_summary(conn)
 
     conn.close()

package/src/scripts/nexo-learning-validator.py

@@ -111,6 +111,25 @@ Rules:
 
     # Try CLI first, fall back to mechanical similarity
     if CLAUDE_CLI.exists():
+        try:
+            env = os.environ.copy()
+            env["NEXO_HEADLESS"] = "1"
+            env.pop("CLAUDECODE", None)
+            env.pop("CLAUDE_CODE", None)
+            learnings_text = "\n".join(
+                f"[#{l.get('id','')}] {l.get('title','')}: {l.get('content','')[:200]}"
+                for l in learnings[:20]
+            )
+            prompt = f"{VALIDATE_PROMPT}\n\nFinding:\n{finding}\n\nExisting learnings:\n{learnings_text}"
+            result = subprocess.run(
+                [str(CLAUDE_CLI), "-p", prompt, "--model", "sonnet", "--output-format", "text"],
+                capture_output=True, text=True, timeout=60, env=env
+            )
+            if result.returncode == 0 and result.stdout.strip():
+                parsed = json.loads(result.stdout.strip())
+                return parsed
+        except Exception:
+            pass
     # Fallback: mechanical SequenceMatcher (original logic)
     return _mechanical_validate(finding, learnings)
 

package/src/scripts/nexo-postmortem-consolidator.py

@@ -387,10 +387,11 @@ def main():
     if not data["diaries"]:
         log("No session diaries today. Nothing to consolidate.")
     else:
-        # Stage 2: CLI intelligence
+        # Stage 2: CLI intelligence (graceful fallback: Stage 3 still runs)
         success = consolidate_with_cli(data)
         if not success:
-            log("Stage 2 failed — falling back to skip (no v1 fallback)")
+            log("Stage 2 failed (CLI unavailable or error). "
+                "Skipping intelligent consolidation. Stage 3 (sensory + force) will still run.")
 
     # Stage 3: Sensory Register (mechanical, kept from v1)
     try:

package/src/scripts/nexo-sleep.py

@@ -550,18 +550,23 @@ def main():
             log(f"Brain state: {len(state['learnings'])} learnings, "
                 f"{state['memory_md_lines']} MEMORY lines, "
                 f"{state['claude_mem_old']} old observations")
-            run_log["stage_b"] = dream(state)
-
-            # Stage B2: Execute actions from CLI output
-            actions_file = COORD_DIR / "sleep-actions.json"
-            if actions_file.exists():
-                try:
-                    actions = json.loads(actions_file.read_text())
-                    execute_dream_actions(actions, state)
-                except Exception as e:
-                    log(f"Stage B2: Error executing actions: {e}")
+            dream_result = dream(state)
+            run_log["stage_b"] = dream_result
+
+            if "error" in dream_result:
+                log(f"Stage B: Dreaming failed ({dream_result['error']}). "
+                    "Stage A cleanup completed successfully. Marking done to avoid retry loop.")
+            else:
+                # Stage B2: Execute actions from CLI output
+                actions_file = COORD_DIR / "sleep-actions.json"
+                if actions_file.exists():
+                    try:
+                        actions = json.loads(actions_file.read_text())
+                        execute_dream_actions(actions, state)
+                    except Exception as e:
+                        log(f"Stage B2: Error executing actions: {e}")
         else:
-            log("Brain is clean — no dreaming needed.")
+            log("Brain is clean -- no dreaming needed.")
             run_log["stage_b"] = {"skipped": True}
 
         # Done

package/src/scripts/nexo-synthesis.py

@@ -6,7 +6,7 @@ Before: ~400 lines of Python concatenating SQL results into markdown sections.
 Now: Collects raw data, passes to Claude CLI (sonnet) which synthesizes
 with real understanding of what matters for tomorrow.
 
-Runs every 2 hours via LaunchAgent. Executes ONCE per day (internal gate).
+Runs daily at 06:00 via LaunchAgent.
 """
 
 import fcntl
@@ -213,6 +213,47 @@ Execute without asking."""
         return False
 
 
+def fallback_synthesis(data: dict):
+    """Write a basic synthesis from raw data when CLI is unavailable."""
+    log("Fallback: writing basic synthesis from raw data...")
+    lines = [f"# NEXO Daily Synthesis -- {TODAY_STR}", "",
+             "*(Generated by fallback -- CLI was unavailable)*", ""]
+
+    if data.get("learnings"):
+        lines.append("## Errors & Learnings")
+        for l in data["learnings"][:10]:
+            lines.append(f"- [{l.get('category', 'general')}] {l.get('title', 'untitled')}")
+        lines.append("")
+
+    if data.get("decisions"):
+        lines.append("## Decisions Made")
+        for d in data["decisions"][:10]:
+            lines.append(f"- [{d.get('domain', 'general')}] {d.get('decision', '')[:120]}")
+        lines.append("")
+
+    if data.get("changes"):
+        lines.append("## Changes Deployed")
+        for c in data["changes"][:10]:
+            lines.append(f"- {c.get('what_changed', '')[:120]}")
+        lines.append("")
+
+    if data.get("overdue_reminders"):
+        lines.append("## Overdue Reminders")
+        for r in data["overdue_reminders"][:10]:
+            lines.append(f"- #{r.get('id', '?')} {r.get('title', '')} (due {r.get('due_date', '?')})")
+        lines.append("")
+
+    if data.get("pending_followups"):
+        lines.append("## Pending Followups")
+        for f in data["pending_followups"][:10]:
+            lines.append(f"- #{f.get('id', '?')} {f.get('title', '')} (due {f.get('due_date', '?')})")
+        lines.append("")
+
+    OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True)
+    OUTPUT_FILE.write_text("\n".join(lines))
+    log(f"Fallback synthesis written to {OUTPUT_FILE}")
+
+
 def main():
     if not should_run():
         log(f"Already ran today ({TODAY_STR}). Skipping.")
@@ -220,7 +261,7 @@ def main():
 
     lock_fd = acquire_lock()
     try:
-        log(f"=== NEXO Synthesis v2 — {TODAY_STR} ===")
+        log(f"=== NEXO Synthesis v2 -- {TODAY_STR} ===")
 
         data = collect_data()
         log(f"Collected: {len(data.get('learnings', []))} learnings, "
@@ -234,7 +275,9 @@ def main():
             mark_done()
             log("Synthesis v2 complete.")
         else:
-            log("Synthesis failed — will retry next trigger.")
+            log("Synthesis CLI failed -- writing fallback synthesis.")
+            fallback_synthesis(data)
+            mark_done()
 
         # Register for catch-up
         try: