nexo-brain 5.3.26 → 5.3.27

package/src/scripts/deep-sleep/extract 2.py

@@ -1,330 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-"""
-Deep Sleep v2 -- Phase 2: Extract findings from each session using the configured automation backend.
-
-For each session in the context file, sends the extract-prompt.md to Claude
-and collects structured findings. Merges all per-session results into
-$DATE-extractions.json.
-
-Environment variables:
-  NEXO_HOME  -- root of the NEXO installation (default: ~/.nexo)
-"""
-import json
-import os
-import subprocess
-import sys
-from datetime import datetime
-from pathlib import Path
-
-NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
-NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parents[2])))
-DEEP_SLEEP_DIR = NEXO_HOME / "operations" / "deep-sleep"
-PROMPT_FILE = Path(__file__).parent / "extract-prompt.md"
-
-if str(NEXO_CODE) not in sys.path:
-    sys.path.insert(0, str(NEXO_CODE))
-
-from agent_runner import AutomationBackendUnavailableError, run_automation_prompt
-try:
-    from client_preferences import resolve_user_model as _resolve_user_model
-    _USER_MODEL = _resolve_user_model()
-except Exception:
-    _USER_MODEL = ""
-
-
-# No timeout -- headless automation can take as long as needed
-CLAUDE_TIMEOUT = 21600  # 3h safety net (prevents zombie processes)
-
-
-def extract_json_from_response(text: str) -> dict | None:
-    """Parse JSON from Claude's response, handling markdown fences."""
-    text = text.strip()
-
-    # Strip markdown code fences if present
-    if text.startswith("```"):
-        lines = text.split("\n")
-        # Remove first line (```json or ```) and last line (```)
-        end = len(lines)
-        for i in range(len(lines) - 1, 0, -1):
-            if lines[i].strip() == "```":
-                end = i
-                break
-        text = "\n".join(lines[1:end]).strip()
-
-    # Find the outermost JSON object
-    brace_start = text.find("{")
-    if brace_start < 0:
-        return None
-
-    # Find matching closing brace
-    depth = 0
-    for i in range(brace_start, len(text)):
-        if text[i] == "{":
-            depth += 1
-        elif text[i] == "}":
-            depth -= 1
-            if depth == 0:
-                try:
-                    return json.loads(text[brace_start:i + 1])
-                except json.JSONDecodeError:
-                    break
-    return None
-
-
-def _safe_session_slug(session_id: str) -> str:
-    return (
-        session_id
-        .replace(".jsonl", "")
-        .replace(":", "-")
-        .replace("/", "-")
-    )
-
-
-def find_session_file(session_id: str, date_dir: Path, session_txt_map: dict[str, str] | None = None) -> Path | None:
-    """Find the individual .txt file for a session."""
-    if session_txt_map:
-        mapped = session_txt_map.get(session_id)
-        if mapped:
-            candidate = date_dir / mapped
-            if candidate.exists():
-                return candidate
-    if date_dir and date_dir.exists():
-        sid_short = _safe_session_slug(session_id)[:20]
-        for f in sorted(date_dir.glob("session-*.txt")):
-            if sid_short in f.name:
-                return f
-    return None
-
-
-def analyze_session(
-    session_id: str,
-    date_dir: Path,
-    shared_context_file: Path | None,
-    session_txt_map: dict[str, str] | None = None,
-) -> dict | None:
-    """Send a session to the automation backend for extraction analysis.
-
-    The backend reads the small per-session file + shared context file.
-    Prompt is short — the heavy lifting is in the Read tool calls.
-    """
-    session_file = find_session_file(session_id, date_dir, session_txt_map=session_txt_map)
-    if not session_file:
-        print(f"    No session file found for {session_id}", file=sys.stderr)
-        return None
-
-    print(f"    File: {session_file.name} ({session_file.stat().st_size / 1024:.0f} KB)")
-
-    # Build a short prompt — Claude reads the files itself
-    shared_ctx_instruction = ""
-    if shared_context_file and shared_context_file.exists():
-        shared_ctx_instruction = f"\n\nAlso read the shared context (followups, learnings, DB state) at: {shared_context_file}"
-
-    prompt_template = PROMPT_FILE.read_text()
-    prompt = prompt_template.replace("{{CONTEXT_FILE}}", str(session_file))
-    prompt = prompt.replace("{{SESSION_ID}}", session_id)
-    prompt += shared_ctx_instruction
-
-    try:
-        JSON_SYSTEM_PROMPT = (
-            "You are a JSON-only analyst. Your ENTIRE response must be a single valid JSON object. "
-            "No text before it. No text after it. No markdown fences. No explanations. "
-            "If you want to summarize, put it inside the JSON fields. Start with { and end with }."
-        )
-
-        result = run_automation_prompt(
-            prompt,
-            model=_USER_MODEL or "opus",
-            timeout=CLAUDE_TIMEOUT,
-            output_format="text",
-            append_system_prompt=JSON_SYSTEM_PROMPT,
-            allowed_tools="Read,Grep,Bash",
-        )
-
-        if result.returncode != 0:
-            print(f"    Automation backend error (exit {result.returncode}): {result.stderr[:300]}", file=sys.stderr)
-            return None
-
-        # Filter out stop hook contamination (e.g. "Post-mortem completo.")
-        output = "\n".join(
-            line for line in result.stdout.splitlines()
-            if not line.strip().startswith("Post-mortem") and line.strip()
-        )
-        parsed = extract_json_from_response(output)
-
-        # Fallback: if Claude returned text instead of JSON, ask a short conversion call
-        if not parsed and len(output.strip()) > 50:
-            print(f"    Got text instead of JSON ({len(output)} chars). Converting...")
-            convert_prompt = (
-                f"Convert the following analysis into the exact JSON schema required. "
-                f"Return ONLY the JSON object, nothing else.\n\n"
-                f"Analysis:\n{output[:8000]}\n\n"
-                f"Required schema: session_id, findings[], emotional_timeline[], "
-                f"abandoned_projects[], skill_candidates[], productivity_score, protocol_summary"
-            )
-            convert_result = run_automation_prompt(
-                convert_prompt,
-                model=_USER_MODEL or "sonnet",
-                timeout=120,
-                output_format="text",
-                append_system_prompt=JSON_SYSTEM_PROMPT,
-            )
-            if convert_result.returncode == 0:
-                parsed = extract_json_from_response(convert_result.stdout)
-                if parsed:
-                    print(f"    Conversion succeeded")
-
-        if not parsed:
-            # Save raw output for debugging
-            debug_file = DEEP_SLEEP_DIR / f"debug-extract-{session_id[:20]}.txt"
-            debug_file.write_text(result.stdout[:5000])
-            print(f"    Failed to parse JSON. Raw output saved to {debug_file}", file=sys.stderr)
-            return None
-
-        return parsed
-
-    except AutomationBackendUnavailableError as exc:
-        print(f"    Automation backend unavailable: {exc}", file=sys.stderr)
-        return None
-    except subprocess.TimeoutExpired:
-        print(f"    Automation backend timeout ({CLAUDE_TIMEOUT}s)", file=sys.stderr)
-        return None
-
-
-def main():
-    target_date = sys.argv[1] if len(sys.argv) > 1 else datetime.now().strftime("%Y-%m-%d")
-
-    context_file = DEEP_SLEEP_DIR / f"{target_date}-context.txt"
-    meta_file = DEEP_SLEEP_DIR / f"{target_date}-meta.json"
-    date_dir = DEEP_SLEEP_DIR / target_date
-
-    if not context_file.exists() and not date_dir.exists():
-        print(f"[extract] No context for {target_date}. Run collect.py first.")
-        sys.exit(1)
-
-    # Read metadata to get session list
-    if meta_file.exists():
-        with open(meta_file) as f:
-            meta = json.load(f)
-        session_files = meta.get("session_files", [])
-        session_txt_map = meta.get("session_txt_map", {})
-    else:
-        # Fallback: parse context file for session IDs
-        print("[extract] No meta file found, scanning context for sessions...")
-        session_files = []
-        session_txt_map = {}
-        if context_file.exists():
-            for line in context_file.read_text().splitlines():
-                if line.startswith("SESSION ") and ":" in line:
-                    parts = line.split(":", 1)
-                    if len(parts) == 2:
-                        sid = parts[1].strip()
-                        if sid.endswith(".jsonl"):
-                            session_files.append(sid)
-
-    if not session_files:
-        print(f"[extract] No sessions to analyze for {target_date}.")
-        output = {"date": target_date, "sessions_analyzed": 0, "extractions": []}
-        output_file = DEEP_SLEEP_DIR / f"{target_date}-extractions.json"
-        with open(output_file, "w") as f:
-            json.dump(output, f, indent=2, ensure_ascii=False)
-        print(f"[extract] Output: {output_file}")
-        return
-
-    # Shared context file (followups, learnings, DB state)
-    shared_context_file = date_dir / "shared-context.txt" if date_dir.exists() else None
-    if shared_context_file and shared_context_file.exists():
-        print(f"[extract] Shared context: {shared_context_file} ({shared_context_file.stat().st_size / 1024:.0f} KB)")
-    else:
-        shared_context_file = None
-        print("[extract] No shared context file")
-
-    print(f"[extract] Phase 2: Analyzing {len(session_files)} sessions for {target_date}")
-    print("[extract] Automation backend: schedule-configured")
-
-    # Checkpoint directory: one JSON per session, survives crashes
-    checkpoint_dir = date_dir / "checkpoints"
-    checkpoint_dir.mkdir(parents=True, exist_ok=True)
-
-    all_extractions = []
-    total_findings = 0
-    skipped = 0
-    MAX_RETRIES = 3
-
-    for i, session_id in enumerate(session_files):
-        sid_safe = _safe_session_slug(session_id)[:40]
-        checkpoint_file = checkpoint_dir / f"{sid_safe}.json"
-
-        # Resume: skip already-processed sessions
-        if checkpoint_file.exists():
-            try:
-                with open(checkpoint_file) as f:
-                    cached = json.load(f)
-                findings_count = len(cached.get("findings", []))
-                total_findings += findings_count
-                all_extractions.append(cached)
-                skipped += 1
-                print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id} (cached, {findings_count} findings)")
-                continue
-            except (json.JSONDecodeError, KeyError):
-                pass  # Corrupted checkpoint, re-process
-
-        print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id}")
-
-        # Retry loop
-        result = None
-        for attempt in range(1, MAX_RETRIES + 1):
-            result = analyze_session(
-                session_id,
-                date_dir,
-                shared_context_file,
-                session_txt_map=session_txt_map,
-            )
-            if result:
-                break
-            if attempt < MAX_RETRIES:
-                print(f"    -> Attempt {attempt}/{MAX_RETRIES} failed, retrying...")
-
-        if result:
-            findings_count = len(result.get("findings", []))
-            total_findings += findings_count
-            all_extractions.append(result)
-            # Save checkpoint
-            with open(checkpoint_file, "w") as f:
-                json.dump(result, f, indent=2, ensure_ascii=False)
-            print(f"    -> {findings_count} findings extracted (checkpointed)")
-        else:
-            print(f"    -> Failed after {MAX_RETRIES} attempts, marking as failed")
-            failed_entry = {
-                "session_id": session_id,
-                "findings": [],
-                "error": f"Extraction failed after {MAX_RETRIES} attempts"
-            }
-            all_extractions.append(failed_entry)
-            # Save failed checkpoint too (so we don't retry forever)
-            with open(checkpoint_file, "w") as f:
-                json.dump(failed_entry, f, indent=2, ensure_ascii=False)
-
-    # Merge into output
-    output = {
-        "date": target_date,
-        "sessions_analyzed": len(session_files),
-        "sessions_succeeded": len([e for e in all_extractions if "error" not in e]),
-        "sessions_cached": skipped,
-        "total_findings": total_findings,
-        "extractions": all_extractions
-    }
-
-    output_file = DEEP_SLEEP_DIR / f"{target_date}-extractions.json"
-    with open(output_file, "w") as f:
-        json.dump(output, f, indent=2, ensure_ascii=False)
-
-    if skipped:
-        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions ({skipped} cached, {len(session_files) - skipped} new).")
-    else:
-        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions.")
-    print(f"[extract] Output: {output_file}")
-
-
-if __name__ == "__main__":
-    main()

package/src/scripts/deep-sleep/extract-prompt 2.md

@@ -1,285 +0,0 @@
-# Deep Sleep v2 -- Phase 2: Session Extraction
-
-You are an overnight analyst for an AI agent's cognitive memory system. You have access to the complete transcript of a session between a user and their AI agent.
-
-## Setup
-
-FIRST: Call `nexo_startup` with `task='deep-sleep extraction'` to initialize the system.
-
-## Your Task
-
-Read the context file provided below. It contains:
-- The full session transcript (user messages, agent responses, tool usage)
-- Active followups, learnings, trust history, and other system state
-
-For the given session, extract the following categories of findings.
-
-### 1. Uncaptured Corrections
-The user corrected the agent but no learning was saved.
-Signals: frustration tone, repeating instructions 2+ times, explicit corrections ("no", "wrong", "that's not it"), the user having to explain something twice.
-For each, note whether it was already captured as a learning (check if `nexo_learning_add` was called after the correction).
-
-### 2. Self-Corrected Errors
-The agent searched in the wrong place, used the wrong approach, then found the right answer.
-These represent knowledge gaps that should become learnings so the agent goes to the right place next time.
-Example: agent looked for config in `/etc/` but it was in `~/.config/`.
-
-### 3. Unformalised Ideas
-The user mentioned an idea, plan, or intention that was never formalized into a followup or reminder.
-Signals: "we should...", "it would be nice...", "we could...", "I want to...", future-tense plans without deadlines.
-
-### 4. Missed Commitments
-The user or agent said "tomorrow", "next week", "when I have time", "I'll look at it later" -- but no followup was created.
-Check the tool usage log: was `nexo_followup_create` or `nexo_reminder_create` called after the commitment?
-
-### 5. Protocol Compliance
-Check whether the agent followed standard protocols:
-- `guard_check` called before editing production/shared files?
-- `heartbeat` called with meaningful context?
-- `change_log` called after production code changes?
-- `learning_add` called after resolving errors?
-- `followup_complete` called when user confirmed a task was done?
-- `feedback` captured after corrections?
-
-### 6. Emotional Signals
-Detect the user's emotional state throughout the session. Look for:
-- Frustration: short replies, cursing, "no", "otra vez", repeated corrections, tone shifts
-- Flow state: long productive stretches, "perfecto", "sí", rapid-fire instructions
-- Satisfaction: praise, "bien", "genial", accepting work without pushback
-- Disengagement: shorter messages over time, "ok", "vale", stopping mid-task
-- Stress: urgency words, deadlines, multiple tasks at once
-For each signal, note the approximate point in the session (early/mid/late) and what triggered it.
-
-### 7. Abandoned Projects
-Detect work that was started but not finished in this session:
-- Tasks the user mentioned wanting to do but never got to
-- Work that was interrupted by something else and never resumed
-- Features partially implemented then dropped
-- Investigations started but conclusions never reached
-Only flag if the work was NOT captured in a followup or reminder.
-
-### 9. Skill Candidates (Reusable Procedures)
-Detect multi-step tasks that were completed successfully and could be reused:
-- Tasks that required 3+ distinct steps to complete
-- Tasks where the agent followed a clear sequence of actions
-- Procedures that are likely to be repeated in the future
-- Examples: deploying code, configuring a service, running an audit, setting up infrastructure
-
-For each candidate, extract:
-- The full step-by-step procedure (what was actually done, in order)
-- Tags describing the domain (e.g., "shopify", "chrome", "deploy")
-- Trigger phrases that would indicate this procedure is needed (e.g., "deploy extension", "push theme")
-- Any gotchas or warnings discovered during execution
-- Whether the procedure looks scriptable (`scriptable: true|false`)
-- Automation scope: `read-only|local|remote`
-- Candidate params if the procedure clearly has repeated inputs (e.g. store, version, environment)
-- Which steps are automatable vs still manual
-
-Only flag if the procedure was SUCCESSFUL (the task was completed without major failures).
-Do NOT flag trivial tasks (single-step actions, simple file edits, quick lookups).
-
-### 8. Productivity Patterns
-Analyze how the session went in terms of efficiency:
-- How many times did the agent need correction before getting it right?
-- Did the agent anticipate needs or always wait for instructions?
-- Were there unnecessary back-and-forths that a better approach would have avoided?
-- Did the agent propose solutions or just ask questions?
-- Tool usage: which tools were used most? Any tools used unnecessarily or not used when they should have been?
-
-## Output Format
-
-Return ONLY valid JSON. No markdown code fences. No explanation text before or after.
-
-```json
-{
-  "session_id": "filename.jsonl",
-  "findings": [
-    {
-      "type": "uncaptured_correction",
-      "confidence": 0.9,
-      "impact": "high",
-      "reversibility": "reversible",
-      "evidence": {
-        "type": "transcript",
-        "session_id": "filename.jsonl",
-        "message_index": 42,
-        "quote": "Exact user words (max 150 chars)"
-      },
-      "dedupe_key": "correction-<short-hash-of-content>",
-      "action_class": "auto_apply",
-      "description": "What the agent should have learned",
-      "suggested_action": "learning_add",
-      "suggested_content": {
-        "category": "process|code|ui|communication",
-        "title": "Short title for the learning",
-        "content": "Full learning content"
-      }
-    },
-    {
-      "type": "self_corrected_error",
-      "confidence": 0.85,
-      "impact": "medium",
-      "reversibility": "reversible",
-      "evidence": {
-        "type": "transcript",
-        "session_id": "filename.jsonl",
-        "message_index": 15,
-        "quote": "What the agent did wrong and how it found the right answer"
-      },
-      "dedupe_key": "selfcorrect-<short-hash>",
-      "action_class": "auto_apply",
-      "description": "Knowledge gap identified",
-      "suggested_action": "learning_add",
-      "suggested_content": {
-        "category": "code",
-        "title": "Short title",
-        "content": "Full learning content"
-      }
-    },
-    {
-      "type": "unformalised_idea",
-      "confidence": 0.7,
-      "impact": "medium",
-      "reversibility": "reversible",
-      "evidence": {
-        "type": "transcript",
-        "session_id": "filename.jsonl",
-        "message_index": 88,
-        "quote": "User's exact words"
-      },
-      "dedupe_key": "idea-<short-hash>",
-      "action_class": "draft_for_morning",
-      "description": "What the idea is about",
-      "suggested_action": "followup_create",
-      "suggested_content": {
-        "description": "Followup description",
-        "date": "YYYY-MM-DD or empty"
-      }
-    },
-    {
-      "type": "missed_commitment",
-      "confidence": 0.8,
-      "impact": "medium",
-      "reversibility": "reversible",
-      "evidence": {
-        "type": "transcript",
-        "session_id": "filename.jsonl",
-        "message_index": 102,
-        "quote": "User's exact words"
-      },
-      "dedupe_key": "commitment-<short-hash>",
-      "action_class": "auto_apply",
-      "description": "What was promised",
-      "suggested_action": "followup_create",
-      "suggested_content": {
-        "description": "Followup description",
-        "date": "YYYY-MM-DD"
-      }
-    },
-    {
-      "type": "protocol_violation",
-      "confidence": 0.95,
-      "impact": "low",
-      "reversibility": "reversible",
-      "evidence": {
-        "type": "transcript",
-        "session_id": "filename.jsonl",
-        "message_index": 50,
-        "quote": "What happened"
-      },
-      "dedupe_key": "protocol-<protocol-name>-<short-hash>",
-      "action_class": "auto_apply",
-      "description": "Which protocol was violated and how",
-      "suggested_action": "learning_add",
-      "suggested_content": {
-        "category": "process",
-        "title": "Protocol compliance: <protocol name>",
-        "content": "Details of what should have been done"
-      }
-    }
-  ],
-  "emotional_timeline": [
-    {
-      "phase": "early|mid|late",
-      "emotion": "frustrated|flow|satisfied|disengaged|stressed|neutral",
-      "trigger": "What caused this emotional state (max 100 chars)",
-      "intensity": 0.8
-    }
-  ],
-
-  "abandoned_projects": [
-    {
-      "description": "What was started but not finished",
-      "reason": "interrupted|dropped|forgotten|deferred",
-      "has_followup": false
-    }
-  ],
-
-  "skill_candidates": [
-    {
-      "name": "Short name for the procedure (e.g., Deploy Chrome Extension)",
-      "description": "What this procedure accomplishes (1-2 sentences)",
-      "steps": [
-        "Step 1: What was done first",
-        "Step 2: What was done next",
-        "Step 3: etc."
-      ],
-      "tags": ["domain1", "domain2"],
-      "trigger_phrases": ["phrase that would trigger this", "another trigger"],
-      "gotchas": ["Warning or caveat discovered during execution"],
-      "scriptable": false,
-      "automation_scope": "read-only|local|remote",
-      "candidate_params": {
-        "param_name": {
-          "type": "string|integer|number|boolean",
-          "required": true
-        }
-      },
-      "automatable_steps": ["Step that could be automated"],
-      "manual_steps": ["Step that still needs human judgment"],
-      "evidence": {
-        "type": "transcript",
-        "session_id": "filename.jsonl",
-        "message_index": 10,
-        "quote": "Start of the multi-step task"
-      },
-      "confidence": 0.85
-    }
-  ],
-
-  "productivity_score": {
-    "corrections_needed": 0,
-    "proactivity": "reactive|mixed|proactive",
-    "unnecessary_roundtrips": 0,
-    "tool_efficiency": "efficient|mixed|wasteful",
-    "notable_tools": ["tool1", "tool2"],
-    "summary": "One sentence assessment of session productivity"
-  },
-
-  "protocol_summary": {
-    "guard_check": {"required": 0, "executed": 0},
-    "heartbeat": {"total": 0, "with_context": 0},
-    "change_log": {"edits": 0, "logged": 0},
-    "learning_capture": {"errors": 0, "captured": 0},
-    "followup_complete": {"confirmed_done": 0, "marked": 0}
-  }
-}
-```
-
-## Rules
-
-- Be SPECIFIC. Quote the user's exact words in evidence.
-- Only flag REAL issues. If the agent did capture something correctly, do not flag it.
-- `confidence` is 0.0-1.0. Be honest -- if you're unsure, use a lower value.
-- `action_class`: use `auto_apply` when confidence >= 0.8 and the action is reversible. Use `draft_for_morning` when confidence < 0.8 or when the impact is high.
-- `dedupe_key`: create a short, deterministic key from the finding type + core content so duplicate findings across runs are suppressed.
-- Do NOT use any specific agent name -- refer to "the agent" throughout.
-- If no issues found, return `{"session_id": "...", "findings": [], "protocol_summary": {...}}`.
-- Do NOT invent problems. Empty findings are perfectly fine.
-- Respond in the user's language (check calibration.json if available). The JSON keys stay in English, but `description`, `title`, `content` fields should be in the user's language.
-
-## Context
-
-Read the session transcript at this path: {{CONTEXT_FILE}}
-
-Analyze session: {{SESSION_ID}}