nexo-brain 2.2.0 → 2.3.1

package/src/scripts/deep-sleep/collect.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 """
 Deep Sleep v2 -- Phase 1: Collect all context for overnight analysis.
 
@@ -116,8 +117,15 @@ def extract_session(jsonl_path: Path) -> dict | None:
     }
 
 
-def collect_transcripts(target_date: str) -> list[dict]:
-    """Collect all sessions modified on the target date."""
+def collect_transcripts_since(since_iso: str, until_iso: str = "") -> list[dict]:
+    """Collect all sessions modified after `since_iso` (exclusive) up to `until_iso` (inclusive).
+
+    Uses a watermark approach: deep sleep tracks the last processed timestamp
+    so nothing is missed regardless of when sessions happen (day, night, etc.).
+    """
+    since_dt = datetime.fromisoformat(since_iso)
+    until_dt = datetime.fromisoformat(until_iso) if until_iso else datetime.now()
+
     sessions = []
     for sdir in find_session_dirs():
         for f in sdir.glob("*.jsonl"):
@@ -125,7 +133,7 @@ def collect_transcripts(target_date: str) -> list[dict]:
                 mtime = datetime.fromtimestamp(f.stat().st_mtime)
             except OSError:
                 continue
-            if mtime.strftime("%Y-%m-%d") == target_date:
+            if since_dt < mtime <= until_dt:
                 session = extract_session(f)
                 if session:
                     session["modified"] = mtime.isoformat()
@@ -339,25 +347,40 @@ def format_transcripts(sessions: list[dict]) -> str:
 
 
 def main():
-    target_date = sys.argv[1] if len(sys.argv) > 1 else datetime.now().strftime("%Y-%m-%d")
+    # Watermark-based collection: since_iso and until_iso passed by the wrapper script
+    # argv[1] = run_id (date label for output files)
+    # argv[2] = since_iso (exclusive lower bound, e.g. "2026-04-01T04:30:00")
+    # argv[3] = until_iso (inclusive upper bound, e.g. "2026-04-02T04:30:00") — optional, defaults to now
+    run_id = sys.argv[1] if len(sys.argv) > 1 else datetime.now().strftime("%Y-%m-%d")
+    since_iso = sys.argv[2] if len(sys.argv) > 2 else ""
+    until_iso = sys.argv[3] if len(sys.argv) > 3 else ""
+
     DEEP_SLEEP_DIR.mkdir(parents=True, exist_ok=True)
 
-    print(f"[collect] Phase 1: Collecting context for {target_date}")
+    print(f"[collect] Phase 1: Collecting context (run_id={run_id})")
 
-    # 1. Transcripts
-    print("[collect] Gathering transcripts...")
-    sessions = collect_transcripts(target_date)
+    # 1. Transcripts — watermark-based
+    if since_iso:
+        print(f"[collect] Gathering transcripts since {since_iso}" + (f" until {until_iso}" if until_iso else ""))
+        sessions = collect_transcripts_since(since_iso, until_iso)
+    else:
+        # Fallback: collect everything from last 48h (safe catch-all)
+        fallback_since = (datetime.now() - timedelta(hours=48)).isoformat()
+        print(f"[collect] No watermark — collecting last 48h since {fallback_since}")
+        sessions = collect_transcripts_since(fallback_since)
     print(f"  Found {len(sessions)} sessions")
 
     if not sessions:
-        print(f"[collect] No sessions found for {target_date}. Writing minimal context file.")
-        output_file = DEEP_SLEEP_DIR / f"{target_date}-context.txt"
+        print(f"[collect] No new sessions found. Writing minimal context file.")
+        output_file = DEEP_SLEEP_DIR / f"{run_id}-context.txt"
         output_file.write_text(
-            f"Deep Sleep Context for {target_date}\n\nNo sessions found for this date.\n"
+            f"Deep Sleep Context for {run_id}\n\nNo sessions found.\n"
         )
         print(f"[collect] Output: {output_file}")
         return
 
+    target_date = run_id  # Keep variable name for downstream compat
+
     # 2. Core DB data
     print("[collect] Querying databases...")
     followups = collect_followups()

package/src/scripts/deep-sleep/extract-prompt.md

@@ -58,6 +58,22 @@ Detect work that was started but not finished in this session:
 - Investigations started but conclusions never reached
 Only flag if the work was NOT captured in a followup or reminder.
 
+### 9. Skill Candidates (Reusable Procedures)
+Detect multi-step tasks that were completed successfully and could be reused:
+- Tasks that required 3+ distinct steps to complete
+- Tasks where the agent followed a clear sequence of actions
+- Procedures that are likely to be repeated in the future
+- Examples: deploying code, configuring a service, running an audit, setting up infrastructure
+
+For each candidate, extract:
+- The full step-by-step procedure (what was actually done, in order)
+- Tags describing the domain (e.g., "shopify", "chrome", "deploy")
+- Trigger phrases that would indicate this procedure is needed (e.g., "deploy extension", "push theme")
+- Any gotchas or warnings discovered during execution
+
+Only flag if the procedure was SUCCESSFUL (the task was completed without major failures).
+Do NOT flag trivial tasks (single-step actions, simple file edits, quick lookups).
+
 ### 8. Productivity Patterns
 Analyze how the session went in terms of efficiency:
 - How many times did the agent need correction before getting it right?
@@ -195,6 +211,28 @@ Return ONLY valid JSON. No markdown code fences. No explanation text before or a
     }
   ],
 
+  "skill_candidates": [
+    {
+      "name": "Short name for the procedure (e.g., Deploy Chrome Extension)",
+      "description": "What this procedure accomplishes (1-2 sentences)",
+      "steps": [
+        "Step 1: What was done first",
+        "Step 2: What was done next",
+        "Step 3: etc."
+      ],
+      "tags": ["domain1", "domain2"],
+      "trigger_phrases": ["phrase that would trigger this", "another trigger"],
+      "gotchas": ["Warning or caveat discovered during execution"],
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 10,
+        "quote": "Start of the multi-step task"
+      },
+      "confidence": 0.85
+    }
+  ],
+
   "productivity_score": {
     "corrections_needed": 0,
     "proactivity": "reactive|mixed|proactive",

package/src/scripts/deep-sleep/extract.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 """
 Deep Sleep v2 -- Phase 2: Extract findings from each session using Claude CLI.
 
@@ -113,6 +114,14 @@ def analyze_session(session_id: str, date_dir: Path, shared_context_file: Path |
     try:
         env = os.environ.copy()
         env["NEXO_HEADLESS"] = "1"  # Skip stop hook post-mortem
+        env.pop("CLAUDECODE", None)
+        env.pop("CLAUDE_CODE", None)
+
+        JSON_SYSTEM_PROMPT = (
+            "You are a JSON-only analyst. Your ENTIRE response must be a single valid JSON object. "
+            "No text before it. No text after it. No markdown fences. No explanations. "
+            "If you want to summarize, put it inside the JSON fields. Start with { and end with }."
+        )
 
         result = subprocess.run(
             [
@@ -120,8 +129,9 @@ def analyze_session(session_id: str, date_dir: Path, shared_context_file: Path |
                 "-p", prompt,
                 "--model", "opus",
                 "--output-format", "text",
+                "--append-system-prompt", JSON_SYSTEM_PROMPT,
                 "--allowedTools",
-                "Read,Grep,Bash,mcp__nexo__nexo_startup,mcp__nexo__nexo_learning_search,mcp__nexo__nexo_recall"
+                "Read,Grep,Bash"
             ],
             capture_output=True,
             text=True,
@@ -139,6 +149,28 @@ def analyze_session(session_id: str, date_dir: Path, shared_context_file: Path |
             if not line.strip().startswith("Post-mortem") and line.strip()
         )
         parsed = extract_json_from_response(output)
+
+        # Fallback: if Claude returned text instead of JSON, ask a short conversion call
+        if not parsed and len(output.strip()) > 50:
+            print(f"    Got text instead of JSON ({len(output)} chars). Converting...")
+            convert_prompt = (
+                f"Convert the following analysis into the exact JSON schema required. "
+                f"Return ONLY the JSON object, nothing else.\n\n"
+                f"Analysis:\n{output[:8000]}\n\n"
+                f"Required schema: session_id, findings[], emotional_timeline[], "
+                f"abandoned_projects[], skill_candidates[], productivity_score, protocol_summary"
+            )
+            convert_result = subprocess.run(
+                [claude_bin, "-p", convert_prompt, "--model", "sonnet",
+                 "--output-format", "text",
+                 "--append-system-prompt", JSON_SYSTEM_PROMPT],
+                capture_output=True, text=True, timeout=120, env=env
+            )
+            if convert_result.returncode == 0:
+                parsed = extract_json_from_response(convert_result.stdout)
+                if parsed:
+                    print(f"    Conversion succeeded")
+
         if not parsed:
             # Save raw output for debugging
             debug_file = DEEP_SLEEP_DIR / f"debug-extract-{session_id[:20]}.txt"
@@ -207,32 +239,70 @@ def main():
     print(f"[extract] Phase 2: Analyzing {len(session_files)} sessions for {target_date}")
     print(f"[extract] Claude CLI: {claude_bin}")
 
+    # Checkpoint directory: one JSON per session, survives crashes
+    checkpoint_dir = date_dir / "checkpoints"
+    checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
     all_extractions = []
     total_findings = 0
+    skipped = 0
+    MAX_RETRIES = 3
 
     for i, session_id in enumerate(session_files):
+        sid_safe = session_id.replace(".jsonl", "")[:30]
+        checkpoint_file = checkpoint_dir / f"{sid_safe}.json"
+
+        # Resume: skip already-processed sessions
+        if checkpoint_file.exists():
+            try:
+                with open(checkpoint_file) as f:
+                    cached = json.load(f)
+                findings_count = len(cached.get("findings", []))
+                total_findings += findings_count
+                all_extractions.append(cached)
+                skipped += 1
+                print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id} (cached, {findings_count} findings)")
+                continue
+            except (json.JSONDecodeError, KeyError):
+                pass  # Corrupted checkpoint, re-process
+
         print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id}")
 
-        result = analyze_session(session_id, date_dir, shared_context_file, claude_bin)
+        # Retry loop
+        result = None
+        for attempt in range(1, MAX_RETRIES + 1):
+            result = analyze_session(session_id, date_dir, shared_context_file, claude_bin)
+            if result:
+                break
+            if attempt < MAX_RETRIES:
+                print(f"    -> Attempt {attempt}/{MAX_RETRIES} failed, retrying...")
 
         if result:
             findings_count = len(result.get("findings", []))
             total_findings += findings_count
             all_extractions.append(result)
-            print(f"    -> {findings_count} findings extracted")
+            # Save checkpoint
+            with open(checkpoint_file, "w") as f:
+                json.dump(result, f, indent=2, ensure_ascii=False)
+            print(f"    -> {findings_count} findings extracted (checkpointed)")
         else:
-            print(f"    -> Extraction failed, continuing with next session")
-            all_extractions.append({
+            print(f"    -> Failed after {MAX_RETRIES} attempts, marking as failed")
+            failed_entry = {
                 "session_id": session_id,
                 "findings": [],
-                "error": "Extraction failed"
-            })
+                "error": f"Extraction failed after {MAX_RETRIES} attempts"
+            }
+            all_extractions.append(failed_entry)
+            # Save failed checkpoint too (so we don't retry forever)
+            with open(checkpoint_file, "w") as f:
+                json.dump(failed_entry, f, indent=2, ensure_ascii=False)
 
     # Merge into output
     output = {
         "date": target_date,
         "sessions_analyzed": len(session_files),
         "sessions_succeeded": len([e for e in all_extractions if "error" not in e]),
+        "sessions_cached": skipped,
         "total_findings": total_findings,
         "extractions": all_extractions
     }
@@ -241,7 +311,10 @@ def main():
     with open(output_file, "w") as f:
         json.dump(output, f, indent=2, ensure_ascii=False)
 
-    print(f"\n[extract] Done. {total_findings} total findings from {len(session_files)} sessions.")
+    if skipped:
+        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions ({skipped} cached, {len(session_files) - skipped} new).")
+    else:
+        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions.")
     print(f"[extract] Output: {output_file}")
 
 

package/src/scripts/deep-sleep/synthesize-prompt.md

@@ -73,6 +73,19 @@ Consider ALL of these:
 
 The score should feel fair. A day with 2 minor corrections and 10 tasks completed is still a good day (75+). A day with 1 catastrophic error might be a 40 even if everything else was fine.
 
+### 9. Skill Extraction
+Consolidate `skill_candidates` from all session extractions into publishable skills:
+- Merge similar procedures from different sessions into a single skill
+- Generalize: replace session-specific IDs, paths, or names with placeholders or descriptions
+- Only include skills with confidence >= 0.7
+- Check if a similar skill already exists (use `nexo_skill_match` if available) — if so, note it for merging instead of creating new
+
+For each skill, generate:
+- A unique ID starting with `SK-` (e.g., `SK-DEPLOY-CHROME-EXT`)
+- Name, description, tags, trigger_patterns
+- The full step-by-step procedure as the skill content
+- Source session IDs for traceability
+
 ### 8. Consolidated Actions
 Merge and deduplicate all findings into a final action list. Each action should have:
 - `action_type`: `learning_add`, `followup_create`, `morning_briefing_item`
@@ -122,9 +135,24 @@ Return ONLY valid JSON. No markdown code fences. No explanation text.
     }
   ],
 
+  "skills": [
+    {
+      "id": "SK-SHORT-ID",
+      "name": "Human readable name",
+      "description": "What this procedure does (1-2 sentences)",
+      "steps": ["Step 1", "Step 2", "Step 3"],
+      "tags": ["tag1", "tag2"],
+      "trigger_patterns": ["trigger phrase 1", "trigger phrase 2"],
+      "gotchas": ["Warning or caveat"],
+      "source_sessions": ["session1.jsonl"],
+      "confidence": 0.85,
+      "merge_with": null
+    }
+  ],
+
   "actions": [
     {
-      "action_type": "learning_add|followup_create|morning_briefing_item",
+      "action_type": "learning_add|followup_create|skill_create|morning_briefing_item",
       "action_class": "auto_apply|draft_for_morning",
       "confidence": 0.9,
       "impact": "low|medium|high",

package/src/scripts/deep-sleep/synthesize.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+from __future__ import annotations
 """
 Deep Sleep v2 -- Phase 3: Synthesize extractions into actionable findings.
 
@@ -115,6 +116,8 @@ def main():
     try:
         env = os.environ.copy()
         env["NEXO_HEADLESS"] = "1"  # Skip stop hook post-mortem
+        env.pop("CLAUDECODE", None)
+        env.pop("CLAUDE_CODE", None)
 
         result = subprocess.run(
             [
@@ -123,7 +126,7 @@ def main():
                 "--model", "opus",
                 "--output-format", "text",
                 "--allowedTools",
-                "Read,Write,Edit,Glob,Grep,Bash,mcp__nexo__nexo_startup,mcp__nexo__nexo_learning_search,mcp__nexo__nexo_recall,mcp__nexo__nexo_reminders"
+                "Read,Grep,Bash"
             ],
             capture_output=True,
             text=True,

package/src/scripts/nexo-catchup.py

@@ -1,20 +1,14 @@
 #!/usr/bin/env python3
 """
-NEXO Catch-Up — Runs at Mac boot to execute any missed scheduled tasks.
+NEXO Catch-Up — Runs at boot/wake to recover any missed scheduled tasks.
 
-When the Mac was asleep/off during scheduled times, launchd does NOT retry
-missed StartCalendarInterval jobs. This script detects what was missed and
-runs them in the correct order.
+Tasks are loaded dynamically from crons/manifest.json (single source of truth).
+Only scheduled crons (with hour/minute) are recovered — interval-based crons
+(immune, watchdog, auto-close) restart automatically via launchd/systemd.
 
-Scheduled tasks (ordered by intended run time):
-  03:00 — cognitive-decay (Ebbinghaus decay + STM→LTM promotion)
-  03:00 — evolution (weekly, Sundays only)
-  04:00 — sleep (session cleanup)
-  07:00 — self-audit (health checks + weekly cognitive GC on Sundays)
-  23:30 — postmortem (consolidation + sensory register)
-
-Logic: For each task, check if its last successful run was before the
-most recent scheduled time. If so, run it now.
+Logic: For each scheduled task, check if its last successful run was before
+the most recent scheduled time. If so, run it now. Only marks success on exit 0.
+Uses cron/launchd weekday convention (0=Sunday) converted to Python (0=Monday).
 """
 
 import json
@@ -52,6 +46,49 @@ def _resolve_python() -> str:
     return sys.executable
 
 NEXO_PYTHON = _resolve_python()
+NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parent.parent)))
+MANIFEST = NEXO_CODE / "crons" / "manifest.json"
+
+
+def _load_tasks_from_manifest() -> list[tuple]:
+    """Read scheduled tasks from manifest.json — single source of truth.
+
+    Only includes crons with a schedule (hour/minute). Excludes interval-based
+    crons (immune, watchdog, auto-close) and run_at_load (catchup itself).
+    Returns: list of (name, hour, minute, python_or_bash, script, weekday)
+    """
+    if not MANIFEST.exists():
+        log(f"WARNING: manifest not found at {MANIFEST}, using empty task list")
+        return []
+
+    with open(MANIFEST) as f:
+        data = json.load(f)
+
+    tasks = []
+    for cron in data.get("crons", []):
+        schedule = cron.get("schedule")
+        if not schedule or "hour" not in schedule:
+            continue  # Skip interval-based and run_at_load crons
+        if cron["id"] == "catchup":
+            continue  # Don't catch up ourselves
+
+        script = cron["script"]
+        script_type = cron.get("type", "python")
+        interpreter = NEXO_PYTHON if script_type == "python" else "/bin/bash"
+        weekday = schedule.get("weekday")
+
+        tasks.append((
+            cron["id"],
+            schedule["hour"],
+            schedule["minute"],
+            interpreter,
+            Path(script).name,
+            weekday,
+        ))
+
+    # Sort by hour, minute for correct execution order
+    tasks.sort(key=lambda t: (t[1], t[2]))
+    return tasks
 
 
 def log(msg: str):
@@ -83,7 +120,11 @@ def last_scheduled_time(hour: int, minute: int, weekday: int = None) -> datetime
 
     if weekday is not None:
         # Weekly task — find the most recent matching weekday
-        days_since = (now.weekday() - weekday) % 7
+        # Manifest uses cron/launchd convention: 0=Sunday, 6=Saturday
+        # Python datetime.weekday() uses: 0=Monday, 6=Sunday
+        # Convert: manifest 0 (Sun) -> python 6, manifest 1 (Mon) -> python 0, etc.
+        py_weekday = (weekday - 1) % 7
+        days_since = (now.weekday() - py_weekday) % 7
         target = now - timedelta(days=days_since)
         target = target.replace(hour=hour, minute=minute, second=0, microsecond=0)
         if target > now:
@@ -130,13 +171,14 @@ def run_task(name: str, python: str, script: str, state: dict) -> bool:
         )
         if result.returncode == 0:
             log(f"  OK {name} (exit 0)")
+            state[name] = datetime.now().isoformat()
+            save_state(state)
+            return True
         else:
-            log(f"  WARN {name} (exit {result.returncode})")
+            log(f"  FAIL {name} (exit {result.returncode})")
             if result.stderr:
                 log(f"    stderr: {result.stderr[:300]}")
-        state[name] = datetime.now().isoformat()
-        save_state(state)
-        return True
+            return False
     except subprocess.TimeoutExpired:
         log(f"  TIMEOUT {name} (300s)")
         return False
@@ -149,17 +191,8 @@ def main():
     log("=== NEXO Catch-Up starting (boot/wake) ===")
     state = load_state()
 
-    # Define tasks in execution order (matching their intended schedule order)
-    # Note: auto-update is handled by the MCP server on startup, not by catchup.
-    tasks = [
-        # (name, hour, minute, python, script, weekday)
-        ("cognitive-decay", 3, 0, NEXO_PYTHON, "nexo-cognitive-decay.py", None),
-        ("evolution", 3, 0, NEXO_PYTHON, "nexo-evolution-run.py", 6),  # Sunday = 6
-        ("sleep", 4, 0, NEXO_PYTHON, "nexo-sleep.py", None),
-        ("self-audit", 7, 0, NEXO_PYTHON, "nexo-daily-self-audit.py", None),
-        ("github-monitor", 8, 0, NEXO_PYTHON, "nexo-github-monitor.py", None),
-        ("postmortem", 23, 30, NEXO_PYTHON, "nexo-postmortem-consolidator.py", None),
-    ]
+    # Read tasks from manifest — single source of truth
+    tasks = _load_tasks_from_manifest()
 
     ran = 0
     skipped = 0
@@ -187,6 +220,9 @@ def _cli_post_catchup_assessment(ran: int, skipped: int, state: dict):
     if not CLAUDE_CLI.exists():
         log(f"Caught up {ran} tasks, {skipped} already current. (CLI unavailable for assessment)")
         return
+    auth_check = subprocess.run(
+        [str(CLAUDE_CLI), "-p", "reply OK", "--output-format", "text"],
+        capture_output=True, text=True, timeout=30
     )
     if auth_check.returncode != 0:
         # CLI not authenticated, skip gracefully

package/src/scripts/nexo-cron-wrapper.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# NEXO Cron Wrapper — Records execution in cron_runs table.
+# Usage: nexo-cron-wrapper.sh <cron_id> <command...>
+# Example: nexo-cron-wrapper.sh deep-sleep bash nexo-deep-sleep.sh
+#
+# Wraps any cron command to automatically record start/end/exit_code/summary.
+# Used by sync.py when generating LaunchAgents from manifest.json.
+
+set -uo pipefail
+
+CRON_ID="${1:?Usage: nexo-cron-wrapper.sh <cron_id> <command...>}"
+shift
+
+NEXO_HOME="${NEXO_HOME:-$HOME/.nexo}"
+DB="$NEXO_HOME/data/nexo.db"
+
+# Record start
+RUN_ID=$(sqlite3 "$DB" "INSERT INTO cron_runs (cron_id) VALUES ('$CRON_ID'); SELECT last_insert_rowid();" 2>/dev/null)
+
+if [ -z "$RUN_ID" ]; then
+    # DB not ready — run without tracking
+    exec "$@"
+fi
+
+# Run the actual command, capture output
+OUTPUT_FILE=$(mktemp)
+"$@" > "$OUTPUT_FILE" 2>&1
+EXIT_CODE=$?
+
+# Extract summary (last meaningful line, max 500 chars)
+SUMMARY=$(tail -5 "$OUTPUT_FILE" | grep -v "^$" | tail -1 | head -c 500 | sed "s/'/''/g")
+
+# Extract error if failed
+ERROR=""
+if [ $EXIT_CODE -ne 0 ]; then
+    ERROR=$(grep -i "error\|exception\|fail\|traceback" "$OUTPUT_FILE" | tail -1 | head -c 500 | sed "s/'/''/g")
+fi
+
+# Record end
+sqlite3 "$DB" "
+    UPDATE cron_runs SET
+        ended_at = datetime('now'),
+        exit_code = $EXIT_CODE,
+        summary = '$SUMMARY',
+        error = '$ERROR',
+        duration_secs = ROUND((julianday(datetime('now')) - julianday(started_at)) * 86400, 1)
+    WHERE id = $RUN_ID;
+" 2>/dev/null
+
+# Clean output
+rm -f "$OUTPUT_FILE"
+
+exit $EXIT_CODE

package/src/scripts/nexo-daily-self-audit.py

@@ -532,8 +532,10 @@ def main():
         "counts": {"error": errors, "warn": warns, "info": infos}
     }, indent=2))
 
-    # Stage B: CLI interpretation
-    interpret_findings(findings)
+    # Stage B: CLI interpretation (graceful fallback if CLI unavailable)
+    cli_ok = interpret_findings(findings)
+    if not cli_ok:
+        log("Stage B: CLI unavailable or failed. Stage A results saved to self-audit-summary.json.")
 
     # Register for catch-up
     try: