nexo-brain 5.3.19 → 5.3.21

package/src/scripts/deep-sleep/extract 2.py

@@ -0,0 +1,330 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+"""
+Deep Sleep v2 -- Phase 2: Extract findings from each session using the configured automation backend.
+
+For each session in the context file, sends the extract-prompt.md to Claude
+and collects structured findings. Merges all per-session results into
+$DATE-extractions.json.
+
+Environment variables:
+  NEXO_HOME  -- root of the NEXO installation (default: ~/.nexo)
+"""
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
+NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parents[2])))
+DEEP_SLEEP_DIR = NEXO_HOME / "operations" / "deep-sleep"
+PROMPT_FILE = Path(__file__).parent / "extract-prompt.md"
+
+if str(NEXO_CODE) not in sys.path:
+    sys.path.insert(0, str(NEXO_CODE))
+
+from agent_runner import AutomationBackendUnavailableError, run_automation_prompt
+try:
+    from client_preferences import resolve_user_model as _resolve_user_model
+    _USER_MODEL = _resolve_user_model()
+except Exception:
+    _USER_MODEL = ""
+
+
+# No timeout -- headless automation can take as long as needed
+CLAUDE_TIMEOUT = 21600  # 3h safety net (prevents zombie processes)
+
+
+def extract_json_from_response(text: str) -> dict | None:
+    """Parse JSON from Claude's response, handling markdown fences."""
+    text = text.strip()
+
+    # Strip markdown code fences if present
+    if text.startswith("```"):
+        lines = text.split("\n")
+        # Remove first line (```json or ```) and last line (```)
+        end = len(lines)
+        for i in range(len(lines) - 1, 0, -1):
+            if lines[i].strip() == "```":
+                end = i
+                break
+        text = "\n".join(lines[1:end]).strip()
+
+    # Find the outermost JSON object
+    brace_start = text.find("{")
+    if brace_start < 0:
+        return None
+
+    # Find matching closing brace
+    depth = 0
+    for i in range(brace_start, len(text)):
+        if text[i] == "{":
+            depth += 1
+        elif text[i] == "}":
+            depth -= 1
+            if depth == 0:
+                try:
+                    return json.loads(text[brace_start:i + 1])
+                except json.JSONDecodeError:
+                    break
+    return None
+
+
+def _safe_session_slug(session_id: str) -> str:
+    return (
+        session_id
+        .replace(".jsonl", "")
+        .replace(":", "-")
+        .replace("/", "-")
+    )
+
+
+def find_session_file(session_id: str, date_dir: Path, session_txt_map: dict[str, str] | None = None) -> Path | None:
+    """Find the individual .txt file for a session."""
+    if session_txt_map:
+        mapped = session_txt_map.get(session_id)
+        if mapped:
+            candidate = date_dir / mapped
+            if candidate.exists():
+                return candidate
+    if date_dir and date_dir.exists():
+        sid_short = _safe_session_slug(session_id)[:20]
+        for f in sorted(date_dir.glob("session-*.txt")):
+            if sid_short in f.name:
+                return f
+    return None
+
+
+def analyze_session(
+    session_id: str,
+    date_dir: Path,
+    shared_context_file: Path | None,
+    session_txt_map: dict[str, str] | None = None,
+) -> dict | None:
+    """Send a session to the automation backend for extraction analysis.
+
+    The backend reads the small per-session file + shared context file.
+    Prompt is short — the heavy lifting is in the Read tool calls.
+    """
+    session_file = find_session_file(session_id, date_dir, session_txt_map=session_txt_map)
+    if not session_file:
+        print(f"    No session file found for {session_id}", file=sys.stderr)
+        return None
+
+    print(f"    File: {session_file.name} ({session_file.stat().st_size / 1024:.0f} KB)")
+
+    # Build a short prompt — Claude reads the files itself
+    shared_ctx_instruction = ""
+    if shared_context_file and shared_context_file.exists():
+        shared_ctx_instruction = f"\n\nAlso read the shared context (followups, learnings, DB state) at: {shared_context_file}"
+
+    prompt_template = PROMPT_FILE.read_text()
+    prompt = prompt_template.replace("{{CONTEXT_FILE}}", str(session_file))
+    prompt = prompt.replace("{{SESSION_ID}}", session_id)
+    prompt += shared_ctx_instruction
+
+    try:
+        JSON_SYSTEM_PROMPT = (
+            "You are a JSON-only analyst. Your ENTIRE response must be a single valid JSON object. "
+            "No text before it. No text after it. No markdown fences. No explanations. "
+            "If you want to summarize, put it inside the JSON fields. Start with { and end with }."
+        )
+
+        result = run_automation_prompt(
+            prompt,
+            model=_USER_MODEL or "opus",
+            timeout=CLAUDE_TIMEOUT,
+            output_format="text",
+            append_system_prompt=JSON_SYSTEM_PROMPT,
+            allowed_tools="Read,Grep,Bash",
+        )
+
+        if result.returncode != 0:
+            print(f"    Automation backend error (exit {result.returncode}): {result.stderr[:300]}", file=sys.stderr)
+            return None
+
+        # Filter out stop hook contamination (e.g. "Post-mortem completo.")
+        output = "\n".join(
+            line for line in result.stdout.splitlines()
+            if not line.strip().startswith("Post-mortem") and line.strip()
+        )
+        parsed = extract_json_from_response(output)
+
+        # Fallback: if Claude returned text instead of JSON, ask a short conversion call
+        if not parsed and len(output.strip()) > 50:
+            print(f"    Got text instead of JSON ({len(output)} chars). Converting...")
+            convert_prompt = (
+                f"Convert the following analysis into the exact JSON schema required. "
+                f"Return ONLY the JSON object, nothing else.\n\n"
+                f"Analysis:\n{output[:8000]}\n\n"
+                f"Required schema: session_id, findings[], emotional_timeline[], "
+                f"abandoned_projects[], skill_candidates[], productivity_score, protocol_summary"
+            )
+            convert_result = run_automation_prompt(
+                convert_prompt,
+                model=_USER_MODEL or "sonnet",
+                timeout=120,
+                output_format="text",
+                append_system_prompt=JSON_SYSTEM_PROMPT,
+            )
+            if convert_result.returncode == 0:
+                parsed = extract_json_from_response(convert_result.stdout)
+                if parsed:
+                    print(f"    Conversion succeeded")
+
+        if not parsed:
+            # Save raw output for debugging
+            debug_file = DEEP_SLEEP_DIR / f"debug-extract-{session_id[:20]}.txt"
+            debug_file.write_text(result.stdout[:5000])
+            print(f"    Failed to parse JSON. Raw output saved to {debug_file}", file=sys.stderr)
+            return None
+
+        return parsed
+
+    except AutomationBackendUnavailableError as exc:
+        print(f"    Automation backend unavailable: {exc}", file=sys.stderr)
+        return None
+    except subprocess.TimeoutExpired:
+        print(f"    Automation backend timeout ({CLAUDE_TIMEOUT}s)", file=sys.stderr)
+        return None
+
+
+def main():
+    target_date = sys.argv[1] if len(sys.argv) > 1 else datetime.now().strftime("%Y-%m-%d")
+
+    context_file = DEEP_SLEEP_DIR / f"{target_date}-context.txt"
+    meta_file = DEEP_SLEEP_DIR / f"{target_date}-meta.json"
+    date_dir = DEEP_SLEEP_DIR / target_date
+
+    if not context_file.exists() and not date_dir.exists():
+        print(f"[extract] No context for {target_date}. Run collect.py first.")
+        sys.exit(1)
+
+    # Read metadata to get session list
+    if meta_file.exists():
+        with open(meta_file) as f:
+            meta = json.load(f)
+        session_files = meta.get("session_files", [])
+        session_txt_map = meta.get("session_txt_map", {})
+    else:
+        # Fallback: parse context file for session IDs
+        print("[extract] No meta file found, scanning context for sessions...")
+        session_files = []
+        session_txt_map = {}
+        if context_file.exists():
+            for line in context_file.read_text().splitlines():
+                if line.startswith("SESSION ") and ":" in line:
+                    parts = line.split(":", 1)
+                    if len(parts) == 2:
+                        sid = parts[1].strip()
+                        if sid.endswith(".jsonl"):
+                            session_files.append(sid)
+
+    if not session_files:
+        print(f"[extract] No sessions to analyze for {target_date}.")
+        output = {"date": target_date, "sessions_analyzed": 0, "extractions": []}
+        output_file = DEEP_SLEEP_DIR / f"{target_date}-extractions.json"
+        with open(output_file, "w") as f:
+            json.dump(output, f, indent=2, ensure_ascii=False)
+        print(f"[extract] Output: {output_file}")
+        return
+
+    # Shared context file (followups, learnings, DB state)
+    shared_context_file = date_dir / "shared-context.txt" if date_dir.exists() else None
+    if shared_context_file and shared_context_file.exists():
+        print(f"[extract] Shared context: {shared_context_file} ({shared_context_file.stat().st_size / 1024:.0f} KB)")
+    else:
+        shared_context_file = None
+        print("[extract] No shared context file")
+
+    print(f"[extract] Phase 2: Analyzing {len(session_files)} sessions for {target_date}")
+    print("[extract] Automation backend: schedule-configured")
+
+    # Checkpoint directory: one JSON per session, survives crashes
+    checkpoint_dir = date_dir / "checkpoints"
+    checkpoint_dir.mkdir(parents=True, exist_ok=True)
+
+    all_extractions = []
+    total_findings = 0
+    skipped = 0
+    MAX_RETRIES = 3
+
+    for i, session_id in enumerate(session_files):
+        sid_safe = _safe_session_slug(session_id)[:40]
+        checkpoint_file = checkpoint_dir / f"{sid_safe}.json"
+
+        # Resume: skip already-processed sessions
+        if checkpoint_file.exists():
+            try:
+                with open(checkpoint_file) as f:
+                    cached = json.load(f)
+                findings_count = len(cached.get("findings", []))
+                total_findings += findings_count
+                all_extractions.append(cached)
+                skipped += 1
+                print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id} (cached, {findings_count} findings)")
+                continue
+            except (json.JSONDecodeError, KeyError):
+                pass  # Corrupted checkpoint, re-process
+
+        print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id}")
+
+        # Retry loop
+        result = None
+        for attempt in range(1, MAX_RETRIES + 1):
+            result = analyze_session(
+                session_id,
+                date_dir,
+                shared_context_file,
+                session_txt_map=session_txt_map,
+            )
+            if result:
+                break
+            if attempt < MAX_RETRIES:
+                print(f"    -> Attempt {attempt}/{MAX_RETRIES} failed, retrying...")
+
+        if result:
+            findings_count = len(result.get("findings", []))
+            total_findings += findings_count
+            all_extractions.append(result)
+            # Save checkpoint
+            with open(checkpoint_file, "w") as f:
+                json.dump(result, f, indent=2, ensure_ascii=False)
+            print(f"    -> {findings_count} findings extracted (checkpointed)")
+        else:
+            print(f"    -> Failed after {MAX_RETRIES} attempts, marking as failed")
+            failed_entry = {
+                "session_id": session_id,
+                "findings": [],
+                "error": f"Extraction failed after {MAX_RETRIES} attempts"
+            }
+            all_extractions.append(failed_entry)
+            # Save failed checkpoint too (so we don't retry forever)
+            with open(checkpoint_file, "w") as f:
+                json.dump(failed_entry, f, indent=2, ensure_ascii=False)
+
+    # Merge into output
+    output = {
+        "date": target_date,
+        "sessions_analyzed": len(session_files),
+        "sessions_succeeded": len([e for e in all_extractions if "error" not in e]),
+        "sessions_cached": skipped,
+        "total_findings": total_findings,
+        "extractions": all_extractions
+    }
+
+    output_file = DEEP_SLEEP_DIR / f"{target_date}-extractions.json"
+    with open(output_file, "w") as f:
+        json.dump(output, f, indent=2, ensure_ascii=False)
+
+    if skipped:
+        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions ({skipped} cached, {len(session_files) - skipped} new).")
+    else:
+        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions.")
+    print(f"[extract] Output: {output_file}")
+
+
+if __name__ == "__main__":
+    main()

package/src/scripts/deep-sleep/extract-prompt 2.md

@@ -0,0 +1,285 @@
+# Deep Sleep v2 -- Phase 2: Session Extraction
+
+You are an overnight analyst for an AI agent's cognitive memory system. You have access to the complete transcript of a session between a user and their AI agent.
+
+## Setup
+
+FIRST: Call `nexo_startup` with `task='deep-sleep extraction'` to initialize the system.
+
+## Your Task
+
+Read the context file provided below. It contains:
+- The full session transcript (user messages, agent responses, tool usage)
+- Active followups, learnings, trust history, and other system state
+
+For the given session, extract the following categories of findings.
+
+### 1. Uncaptured Corrections
+The user corrected the agent but no learning was saved.
+Signals: frustration tone, repeating instructions 2+ times, explicit corrections ("no", "wrong", "that's not it"), the user having to explain something twice.
+For each, note whether it was already captured as a learning (check if `nexo_learning_add` was called after the correction).
+
+### 2. Self-Corrected Errors
+The agent searched in the wrong place, used the wrong approach, then found the right answer.
+These represent knowledge gaps that should become learnings so the agent goes to the right place next time.
+Example: agent looked for config in `/etc/` but it was in `~/.config/`.
+
+### 3. Unformalised Ideas
+The user mentioned an idea, plan, or intention that was never formalized into a followup or reminder.
+Signals: "we should...", "it would be nice...", "we could...", "I want to...", future-tense plans without deadlines.
+
+### 4. Missed Commitments
+The user or agent said "tomorrow", "next week", "when I have time", "I'll look at it later" -- but no followup was created.
+Check the tool usage log: was `nexo_followup_create` or `nexo_reminder_create` called after the commitment?
+
+### 5. Protocol Compliance
+Check whether the agent followed standard protocols:
+- `guard_check` called before editing production/shared files?
+- `heartbeat` called with meaningful context?
+- `change_log` called after production code changes?
+- `learning_add` called after resolving errors?
+- `followup_complete` called when user confirmed a task was done?
+- `feedback` captured after corrections?
+
+### 6. Emotional Signals
+Detect the user's emotional state throughout the session. Look for:
+- Frustration: short replies, cursing, "no", "otra vez", repeated corrections, tone shifts
+- Flow state: long productive stretches, "perfecto", "sí", rapid-fire instructions
+- Satisfaction: praise, "bien", "genial", accepting work without pushback
+- Disengagement: shorter messages over time, "ok", "vale", stopping mid-task
+- Stress: urgency words, deadlines, multiple tasks at once
+For each signal, note the approximate point in the session (early/mid/late) and what triggered it.
+
+### 7. Abandoned Projects
+Detect work that was started but not finished in this session:
+- Tasks the user mentioned wanting to do but never got to
+- Work that was interrupted by something else and never resumed
+- Features partially implemented then dropped
+- Investigations started but conclusions never reached
+Only flag if the work was NOT captured in a followup or reminder.
+
+### 9. Skill Candidates (Reusable Procedures)
+Detect multi-step tasks that were completed successfully and could be reused:
+- Tasks that required 3+ distinct steps to complete
+- Tasks where the agent followed a clear sequence of actions
+- Procedures that are likely to be repeated in the future
+- Examples: deploying code, configuring a service, running an audit, setting up infrastructure
+
+For each candidate, extract:
+- The full step-by-step procedure (what was actually done, in order)
+- Tags describing the domain (e.g., "shopify", "chrome", "deploy")
+- Trigger phrases that would indicate this procedure is needed (e.g., "deploy extension", "push theme")
+- Any gotchas or warnings discovered during execution
+- Whether the procedure looks scriptable (`scriptable: true|false`)
+- Automation scope: `read-only|local|remote`
+- Candidate params if the procedure clearly has repeated inputs (e.g. store, version, environment)
+- Which steps are automatable vs still manual
+
+Only flag if the procedure was SUCCESSFUL (the task was completed without major failures).
+Do NOT flag trivial tasks (single-step actions, simple file edits, quick lookups).
+
+### 8. Productivity Patterns
+Analyze how the session went in terms of efficiency:
+- How many times did the agent need correction before getting it right?
+- Did the agent anticipate needs or always wait for instructions?
+- Were there unnecessary back-and-forths that a better approach would have avoided?
+- Did the agent propose solutions or just ask questions?
+- Tool usage: which tools were used most? Any tools used unnecessarily or not used when they should have been?
+
+## Output Format
+
+Return ONLY valid JSON. No markdown code fences. No explanation text before or after.
+
+```json
+{
+  "session_id": "filename.jsonl",
+  "findings": [
+    {
+      "type": "uncaptured_correction",
+      "confidence": 0.9,
+      "impact": "high",
+      "reversibility": "reversible",
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 42,
+        "quote": "Exact user words (max 150 chars)"
+      },
+      "dedupe_key": "correction-<short-hash-of-content>",
+      "action_class": "auto_apply",
+      "description": "What the agent should have learned",
+      "suggested_action": "learning_add",
+      "suggested_content": {
+        "category": "process|code|ui|communication",
+        "title": "Short title for the learning",
+        "content": "Full learning content"
+      }
+    },
+    {
+      "type": "self_corrected_error",
+      "confidence": 0.85,
+      "impact": "medium",
+      "reversibility": "reversible",
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 15,
+        "quote": "What the agent did wrong and how it found the right answer"
+      },
+      "dedupe_key": "selfcorrect-<short-hash>",
+      "action_class": "auto_apply",
+      "description": "Knowledge gap identified",
+      "suggested_action": "learning_add",
+      "suggested_content": {
+        "category": "code",
+        "title": "Short title",
+        "content": "Full learning content"
+      }
+    },
+    {
+      "type": "unformalised_idea",
+      "confidence": 0.7,
+      "impact": "medium",
+      "reversibility": "reversible",
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 88,
+        "quote": "User's exact words"
+      },
+      "dedupe_key": "idea-<short-hash>",
+      "action_class": "draft_for_morning",
+      "description": "What the idea is about",
+      "suggested_action": "followup_create",
+      "suggested_content": {
+        "description": "Followup description",
+        "date": "YYYY-MM-DD or empty"
+      }
+    },
+    {
+      "type": "missed_commitment",
+      "confidence": 0.8,
+      "impact": "medium",
+      "reversibility": "reversible",
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 102,
+        "quote": "User's exact words"
+      },
+      "dedupe_key": "commitment-<short-hash>",
+      "action_class": "auto_apply",
+      "description": "What was promised",
+      "suggested_action": "followup_create",
+      "suggested_content": {
+        "description": "Followup description",
+        "date": "YYYY-MM-DD"
+      }
+    },
+    {
+      "type": "protocol_violation",
+      "confidence": 0.95,
+      "impact": "low",
+      "reversibility": "reversible",
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 50,
+        "quote": "What happened"
+      },
+      "dedupe_key": "protocol-<protocol-name>-<short-hash>",
+      "action_class": "auto_apply",
+      "description": "Which protocol was violated and how",
+      "suggested_action": "learning_add",
+      "suggested_content": {
+        "category": "process",
+        "title": "Protocol compliance: <protocol name>",
+        "content": "Details of what should have been done"
+      }
+    }
+  ],
+  "emotional_timeline": [
+    {
+      "phase": "early|mid|late",
+      "emotion": "frustrated|flow|satisfied|disengaged|stressed|neutral",
+      "trigger": "What caused this emotional state (max 100 chars)",
+      "intensity": 0.8
+    }
+  ],
+
+  "abandoned_projects": [
+    {
+      "description": "What was started but not finished",
+      "reason": "interrupted|dropped|forgotten|deferred",
+      "has_followup": false
+    }
+  ],
+
+  "skill_candidates": [
+    {
+      "name": "Short name for the procedure (e.g., Deploy Chrome Extension)",
+      "description": "What this procedure accomplishes (1-2 sentences)",
+      "steps": [
+        "Step 1: What was done first",
+        "Step 2: What was done next",
+        "Step 3: etc."
+      ],
+      "tags": ["domain1", "domain2"],
+      "trigger_phrases": ["phrase that would trigger this", "another trigger"],
+      "gotchas": ["Warning or caveat discovered during execution"],
+      "scriptable": false,
+      "automation_scope": "read-only|local|remote",
+      "candidate_params": {
+        "param_name": {
+          "type": "string|integer|number|boolean",
+          "required": true
+        }
+      },
+      "automatable_steps": ["Step that could be automated"],
+      "manual_steps": ["Step that still needs human judgment"],
+      "evidence": {
+        "type": "transcript",
+        "session_id": "filename.jsonl",
+        "message_index": 10,
+        "quote": "Start of the multi-step task"
+      },
+      "confidence": 0.85
+    }
+  ],
+
+  "productivity_score": {
+    "corrections_needed": 0,
+    "proactivity": "reactive|mixed|proactive",
+    "unnecessary_roundtrips": 0,
+    "tool_efficiency": "efficient|mixed|wasteful",
+    "notable_tools": ["tool1", "tool2"],
+    "summary": "One sentence assessment of session productivity"
+  },
+
+  "protocol_summary": {
+    "guard_check": {"required": 0, "executed": 0},
+    "heartbeat": {"total": 0, "with_context": 0},
+    "change_log": {"edits": 0, "logged": 0},
+    "learning_capture": {"errors": 0, "captured": 0},
+    "followup_complete": {"confirmed_done": 0, "marked": 0}
+  }
+}
+```
+
+## Rules
+
+- Be SPECIFIC. Quote the user's exact words in evidence.
+- Only flag REAL issues. If the agent did capture something correctly, do not flag it.
+- `confidence` is 0.0-1.0. Be honest -- if you're unsure, use a lower value.
+- `action_class`: use `auto_apply` when confidence >= 0.8 and the action is reversible. Use `draft_for_morning` when confidence < 0.8 or when the impact is high.
+- `dedupe_key`: create a short, deterministic key from the finding type + core content so duplicate findings across runs are suppressed.
+- Do NOT use any specific agent name -- refer to "the agent" throughout.
+- If no issues found, return `{"session_id": "...", "findings": [], "protocol_summary": {...}}`.
+- Do NOT invent problems. Empty findings are perfectly fine.
+- Respond in the user's language (check calibration.json if available). The JSON keys stay in English, but `description`, `title`, `content` fields should be in the user's language.
+
+## Context
+
+Read the session transcript at this path: {{CONTEXT_FILE}}
+
+Analyze session: {{SESSION_ID}}