nexo-brain 5.7.0 → 5.8.1

package/src/scripts/deep-sleep/extract.py

@@ -38,6 +38,56 @@ except Exception:
 # still leaving enough headroom for legitimate long per-session extractions.
 CLAUDE_TIMEOUT = AUTOMATION_SUBPROCESS_TIMEOUT
 
+# Poison detection: a session checkpoint records the number of failed attempts
+# across runs. Once it reaches this limit we stop trying to extract findings
+# from that session — repeated failures on the same session (deterministic
+# JSON parse errors, unreadable transcripts) only burn API credits and stall
+# the whole deep-sleep cycle behind the poisoned session. The session is still
+# kept in the output (with the error) so synthesize.py can account for it.
+MAX_POISON_ATTEMPTS = 3
+
+# Transient error types worth retrying on the next deep-sleep run instead of
+# being counted as a poisoned attempt. `overloaded_error` comes from the
+# Anthropic API when it is under load and is the cause of the stuck
+# deep-sleep between 2026-04-14 and 2026-04-17 — the first attempt hit it,
+# the checkpoint flagged it as permanent failure, and later runs kept
+# re-processing the same session forever.
+TRANSIENT_ERROR_KINDS = {
+    "overloaded_error",
+    "rate_limit_error",
+    "api_error",
+    "timeout",
+    "signal",
+}
+
+
+def _classify_cli_result(result) -> tuple[str, str]:
+    """Return (kind, short_message) describing a failed automation backend call.
+
+    Kinds:
+      - "overloaded_error" / "rate_limit_error" / "api_error"
+          Anthropic API transient failure — do not poison the checkpoint.
+      - "signal"   Claude CLI killed by external signal (SIGTERM / SIGKILL / exit>=128).
+      - "timeout"  Subprocess hit CLAUDE_TIMEOUT — extremely long session.
+      - "json_parse" Claude responded, but output wasn't parseable JSON.
+      - "unknown"  Fallback.
+    """
+    rc = getattr(result, "returncode", -1)
+    stderr = (getattr(result, "stderr", "") or "")[:800]
+    stdout = (getattr(result, "stdout", "") or "")[:800]
+    blob = f"{stderr}\n{stdout}".lower()
+    if "overloaded" in blob:
+        return "overloaded_error", "Anthropic API overloaded"
+    if "rate_limit" in blob or "rate-limit" in blob or "429" in blob:
+        return "rate_limit_error", "Anthropic rate-limit hit"
+    if '"type":"error"' in blob and '"api_error"' in blob:
+        return "api_error", "Anthropic API error"
+    if rc >= 128:
+        return "signal", f"killed by signal (exit {rc})"
+    if rc < 0:
+        return "signal", f"subprocess terminated (exit {rc})"
+    return "unknown", f"exit {rc}"
+
 
 def extract_json_from_response(text: str) -> dict | None:
     """Parse JSON from Claude's response, handling markdown fences."""
@@ -104,20 +154,23 @@ def analyze_session(
     date_dir: Path,
     shared_context_file: Path | None,
     session_txt_map: dict[str, str] | None = None,
-) -> dict | None:
+) -> tuple[dict | None, str | None]:
     """Send a session to the automation backend for extraction analysis.
 
-    The backend reads the small per-session file + shared context file.
-    Prompt is short — the heavy lifting is in the Read tool calls.
+    Returns (parsed_result, error_kind). `error_kind` is only set on failure.
+    See `_classify_cli_result` for possible values.
     """
     session_file = find_session_file(session_id, date_dir, session_txt_map=session_txt_map)
     if not session_file:
         print(f"    No session file found for {session_id}", file=sys.stderr)
-        return None
+        return None, "missing_session_file"
 
     print(f"    File: {session_file.name} ({session_file.stat().st_size / 1024:.0f} KB)")
 
-    # Build a short prompt — Claude reads the files itself
+    # Build a short prompt — Claude reads the files itself. We point at the
+    # slim shared context rather than the full 400+KB dump so the Claude CLI
+    # process doesn't have to stream hundreds of kilobytes of followups /
+    # learnings into its context window on every per-session extraction.
     shared_ctx_instruction = ""
     if shared_context_file and shared_context_file.exists():
         shared_ctx_instruction = f"\n\nAlso read the shared context (followups, learnings, DB state) at: {shared_context_file}"
@@ -148,8 +201,9 @@ def analyze_session(
         )
 
         if result.returncode != 0:
-            print(f"    Automation backend error (exit {result.returncode}): {result.stderr[:300]}", file=sys.stderr)
-            return None
+            kind, message = _classify_cli_result(result)
+            print(f"    Automation backend {kind} (exit {result.returncode}): {message}", file=sys.stderr)
+            return None, kind
 
         # Filter out stop hook contamination (e.g. "Post-mortem completo.")
         output = "\n".join(
@@ -185,18 +239,65 @@ def analyze_session(
             debug_file = DEEP_SLEEP_DIR / f"debug-extract-{session_id[:20]}.txt"
             debug_file.write_text(result.stdout[:5000])
             print(f"    Failed to parse JSON. Raw output saved to {debug_file}", file=sys.stderr)
-            return None
+            return None, "json_parse"
 
-        return parsed
+        return parsed, None
 
     except AutomationBackendUnavailableError as exc:
         print(f"    Automation backend unavailable: {exc}", file=sys.stderr)
-        return None
+        return None, "backend_unavailable"
     except subprocess.TimeoutExpired:
         print(f"    Automation backend timeout ({CLAUDE_TIMEOUT}s)", file=sys.stderr)
+        return None, "timeout"
+
+
+def _write_slim_shared_context(full_path: Path) -> Path:
+    """Generate (once per run) a slim version of shared-context.txt.
+
+    The full shared context can exceed 400KB — feeding that to every
+    per-session extraction means the Claude CLI subprocess spends most of its
+    context window on repeated DB metadata instead of the session transcript.
+    The slim version keeps the top-level structure + the first ~200 lines so
+    the model still has a summary of followups/learnings/diary samples.
+    """
+    slim_path = full_path.with_suffix(".slim.txt")
+    try:
+        raw = full_path.read_text(errors="replace")
+    except OSError:
+        return full_path
+    lines = raw.splitlines()
+    head = lines[:200]
+    header = [
+        "# Shared context (slim) — " + full_path.name,
+        f"# original_bytes={full_path.stat().st_size} original_lines={len(lines)}",
+        f"# trimmed_to=first_{len(head)}_lines",
+        "",
+    ]
+    try:
+        slim_path.write_text("\n".join(header + head), encoding="utf-8")
+    except OSError:
+        return full_path
+    return slim_path
+
+
+def _load_checkpoint(path: Path) -> dict | None:
+    if not path.exists():
+        return None
+    try:
+        with path.open() as fh:
+            return json.load(fh)
+    except (json.JSONDecodeError, OSError):
         return None
 
 
+def _save_checkpoint(path: Path, payload: dict) -> None:
+    try:
+        with path.open("w") as fh:
+            json.dump(payload, fh, indent=2, ensure_ascii=False)
+    except OSError as exc:
+        print(f"    Warning: could not persist checkpoint {path}: {exc}", file=sys.stderr)
+
+
 def main():
     target_date = sys.argv[1] if len(sys.argv) > 1 else datetime.now().strftime("%Y-%m-%d")
 
@@ -237,12 +338,17 @@ def main():
         print(f"[extract] Output: {output_file}")
         return
 
-    # Shared context file (followups, learnings, DB state)
-    shared_context_file = date_dir / "shared-context.txt" if date_dir.exists() else None
-    if shared_context_file and shared_context_file.exists():
-        print(f"[extract] Shared context: {shared_context_file} ({shared_context_file.stat().st_size / 1024:.0f} KB)")
+    # Shared context file (followups, learnings, DB state).
+    # Use a slim copy for the per-session prompts so the Claude CLI doesn't
+    # re-read the full 400+KB dump for every single session.
+    full_shared_context = date_dir / "shared-context.txt" if date_dir.exists() else None
+    shared_context_file: Path | None = None
+    if full_shared_context and full_shared_context.exists():
+        shared_context_file = _write_slim_shared_context(full_shared_context)
+        full_kb = full_shared_context.stat().st_size / 1024
+        slim_kb = shared_context_file.stat().st_size / 1024
+        print(f"[extract] Shared context: {shared_context_file} ({slim_kb:.0f} KB slim, {full_kb:.0f} KB full)")
     else:
-        shared_context_file = None
         print("[extract] No shared context file")
 
     print(f"[extract] Phase 2: Analyzing {len(session_files)} sessions for {target_date}")
@@ -255,6 +361,7 @@ def main():
     all_extractions = []
     total_findings = 0
     skipped = 0
+    poisoned = 0
     # Two attempts is enough: if a session's extraction fails twice, the cause is
     # almost always deterministic (JSON parse, schema violation) rather than transient,
     # so further retries just burn time. Skip and continue instead.
@@ -264,26 +371,42 @@ def main():
         sid_safe = _safe_session_slug(session_id)[:40]
         checkpoint_file = checkpoint_dir / f"{sid_safe}.json"
 
-        # Resume: skip already-processed sessions
-        if checkpoint_file.exists():
-            try:
-                with open(checkpoint_file) as f:
-                    cached = json.load(f)
-                findings_count = len(cached.get("findings", []))
-                total_findings += findings_count
-                all_extractions.append(cached)
-                skipped += 1
-                print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id} (cached, {findings_count} findings)")
-                continue
-            except (json.JSONDecodeError, KeyError):
-                pass  # Corrupted checkpoint, re-process
+        cached = _load_checkpoint(checkpoint_file)
+        cached_error_count = int((cached or {}).get("error_count", 0))
+        cached_last_error_kind = (cached or {}).get("last_error_kind", "")
+
+        # Successful prior checkpoint → reuse as-is
+        if cached and not cached.get("error") and cached.get("findings") is not None:
+            findings_count = len(cached.get("findings", []))
+            total_findings += findings_count
+            all_extractions.append(cached)
+            skipped += 1
+            print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id} (cached, {findings_count} findings)")
+            continue
+
+        # Poisoned checkpoint → skip without burning API calls
+        if cached_error_count >= MAX_POISON_ATTEMPTS:
+            poisoned += 1
+            all_extractions.append(cached or {
+                "session_id": session_id,
+                "findings": [],
+                "error": "poisoned",
+                "error_count": cached_error_count,
+                "last_error_kind": cached_last_error_kind,
+            })
+            print(
+                f"[extract] Session {i + 1}/{len(session_files)}: {session_id} "
+                f"(poisoned, {cached_error_count} prior failures — skip)"
+            )
+            continue
 
         print(f"[extract] Session {i + 1}/{len(session_files)}: {session_id}")
 
-        # Retry loop
+        # Retry loop within this run
         result = None
+        last_error_kind = ""
         for attempt in range(1, MAX_RETRIES + 1):
-            result = analyze_session(
+            result, error_kind = analyze_session(
                 session_id,
                 date_dir,
                 shared_context_file,
@@ -291,47 +414,77 @@ def main():
             )
             if result:
                 break
+            last_error_kind = error_kind or "unknown"
             if attempt < MAX_RETRIES:
-                print(f"    -> Attempt {attempt}/{MAX_RETRIES} failed, retrying...")
+                print(f"    -> Attempt {attempt}/{MAX_RETRIES} failed ({last_error_kind}), retrying...")
 
         if result:
             findings_count = len(result.get("findings", []))
             total_findings += findings_count
+            # Persist success and reset error_count so transient past failures
+            # don't keep counting against the session.
+            result.setdefault("session_id", session_id)
+            result["error_count"] = 0
+            result["last_error_kind"] = ""
             all_extractions.append(result)
-            # Save checkpoint
-            with open(checkpoint_file, "w") as f:
-                json.dump(result, f, indent=2, ensure_ascii=False)
+            _save_checkpoint(checkpoint_file, result)
             print(f"    -> {findings_count} findings extracted (checkpointed)")
         else:
-            print(f"    -> Failed after {MAX_RETRIES} attempts, marking as failed")
+            # Transient errors (API overloaded, rate-limit, timeout, killed
+            # by signal) should NOT increment the poison counter — they're
+            # not the session's fault. They also don't persist a fresh
+            # checkpoint, so the next deep-sleep run will retry cleanly.
+            transient = last_error_kind in TRANSIENT_ERROR_KINDS
+            if transient:
+                print(f"    -> Transient failure ({last_error_kind}), will retry on next run.")
+                all_extractions.append({
+                    "session_id": session_id,
+                    "findings": [],
+                    "error": "transient",
+                    "error_count": cached_error_count,
+                    "last_error_kind": last_error_kind,
+                })
+                # Do not touch the checkpoint — the next run gets a clean retry.
+                continue
+
+            new_count = cached_error_count + 1
+            state = "poisoned" if new_count >= MAX_POISON_ATTEMPTS else "failed"
+            print(
+                f"    -> Deterministic failure #{new_count}/{MAX_POISON_ATTEMPTS} "
+                f"({last_error_kind}); marked as {state}."
+            )
             failed_entry = {
                 "session_id": session_id,
                 "findings": [],
-                "error": f"Extraction failed after {MAX_RETRIES} attempts"
+                "error": state,
+                "error_count": new_count,
+                "last_error_kind": last_error_kind,
             }
             all_extractions.append(failed_entry)
-            # Save failed checkpoint too (so we don't retry forever)
-            with open(checkpoint_file, "w") as f:
-                json.dump(failed_entry, f, indent=2, ensure_ascii=False)
+            _save_checkpoint(checkpoint_file, failed_entry)
+            if state == "poisoned":
+                poisoned += 1
 
     # Merge into output
     output = {
         "date": target_date,
         "sessions_analyzed": len(session_files),
-        "sessions_succeeded": len([e for e in all_extractions if "error" not in e]),
+        "sessions_succeeded": len([e for e in all_extractions if not e.get("error")]),
         "sessions_cached": skipped,
+        "sessions_poisoned": poisoned,
         "total_findings": total_findings,
-        "extractions": all_extractions
+        "extractions": all_extractions,
     }
 
     output_file = DEEP_SLEEP_DIR / f"{target_date}-extractions.json"
     with open(output_file, "w") as f:
         json.dump(output, f, indent=2, ensure_ascii=False)
 
-    if skipped:
-        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions ({skipped} cached, {len(session_files) - skipped} new).")
-    else:
-        print(f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions.")
+    fresh_runs = len(session_files) - skipped - poisoned
+    print(
+        f"\n[extract] Done. {total_findings} findings from {len(session_files)} sessions "
+        f"({skipped} cached, {fresh_runs} fresh, {poisoned} poisoned)."
+    )
     print(f"[extract] Output: {output_file}")
 
 

package/src/scripts/nexo-cron-wrapper.sh

@@ -5,6 +5,16 @@
 #
 # Wraps any cron command to automatically record start/end/exit_code/summary.
 # Used by sync.py when generating LaunchAgents from manifest.json.
+#
+# Two-phase recording (start → end):
+# 1. INSERT cron_runs row at start with ended_at=NULL so the watchdog can
+#    distinguish "currently running" from "missed / stuck". Without this,
+#    any job that exceeds the next watchdog tick (interval_seconds=1800 by
+#    default) looks stale and the watchdog may kickstart -k over it — which
+#    is exactly the loop that broke deep-sleep between 2026-04-14 and 2026-04-17.
+# 2. UPDATE the row at end with ended_at + exit_code + summary.
+# 3. Trap SIGTERM / SIGINT so wrappers killed mid-flight still close their
+#    row (exit_code=143 or 130) instead of leaving it NULL forever.
 
 set -uo pipefail
 
@@ -33,84 +43,159 @@ print(datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))
 PY
 )
 
-# Run the actual command, capture output
+# Phase 1: INSERT row at start (ended_at NULL = "running").
+# ROW_ID empty on DB failure; spool-fallback at the end handles that.
+ROW_ID=""
+ROW_ID=$(python3 - "$DB" "$CRON_ID" "$STARTED_AT" <<'PY' 2>/dev/null
+from __future__ import annotations
+import sqlite3
+import sys
+db_path, cron_id, started_at = sys.argv[1:]
+conn = sqlite3.connect(db_path)
+try:
+    cur = conn.execute(
+        "INSERT INTO cron_runs (cron_id, started_at, ended_at) VALUES (?, ?, NULL)",
+        (cron_id, started_at),
+    )
+    conn.commit()
+    print(cur.lastrowid)
+finally:
+    conn.close()
+PY
+)
+
 OUTPUT_FILE=$(mktemp)
-trap 'rm -f "$OUTPUT_FILE"' EXIT
-"$@" > "$OUTPUT_FILE" 2>&1
-EXIT_CODE=$?
-ENDED_AT=$(python3 - <<'PY'
+EXIT_CODE=0
+SIGNAL_NAME=""
+
+# finalize_row DB writer — also used by signal traps.
+# Reads $EXIT_CODE / $SIGNAL_NAME / $OUTPUT_FILE from the outer scope.
+finalize_row() {
+    local ended_at duration summary error
+    ended_at=$(python3 - <<'PY'
 from datetime import datetime, timezone
 print(datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))
 PY
 )
-DURATION_SECS=$(python3 - <<PY
+    duration=$(python3 - <<PY
 start = float("$START_EPOCH")
 import time
 print(round(time.time() - start, 1))
 PY
 )
+    summary=$(tail -5 "$OUTPUT_FILE" 2>/dev/null | grep -v "^$" | tail -1 | head -c 500)
+    error=""
+    if [ "$EXIT_CODE" -ne 0 ]; then
+        if [ -n "$SIGNAL_NAME" ]; then
+            error="Killed by $SIGNAL_NAME (exit $EXIT_CODE)"
+        else
+            error=$(grep -i "error\|exception\|fail\|traceback" "$OUTPUT_FILE" 2>/dev/null | tail -1 | head -c 500)
+        fi
+    fi
 
-# Extract summary (last meaningful line, max 500 chars)
-SUMMARY=$(tail -5 "$OUTPUT_FILE" | grep -v "^$" | tail -1 | head -c 500)
-
-# Extract error if failed
-ERROR=""
-if [ $EXIT_CODE -ne 0 ]; then
-    ERROR=$(grep -i "error\|exception\|fail\|traceback" "$OUTPUT_FILE" | tail -1 | head -c 500)
-fi
-
-if ! python3 - "$DB" "$CRON_ID" "$STARTED_AT" "$ENDED_AT" "$EXIT_CODE" "$SUMMARY" "$ERROR" "$DURATION_SECS" <<'PY'
+    # Update the row we inserted at start — or INSERT fresh if the start write failed.
+    if ! python3 - "$DB" "$ROW_ID" "$CRON_ID" "$STARTED_AT" "$ended_at" "$EXIT_CODE" "$summary" "$error" "$duration" <<'PY' 2>/dev/null
 from __future__ import annotations
-
 import sqlite3
 import sys
-
-db_path, cron_id, started_at, ended_at, exit_code, summary, error, duration_secs = sys.argv[1:]
+db_path, row_id, cron_id, started_at, ended_at, exit_code, summary, error, duration_secs = sys.argv[1:]
 conn = sqlite3.connect(db_path)
 try:
-    conn.execute(
-        """
-        INSERT INTO cron_runs (
-            cron_id, started_at, ended_at, exit_code, summary, error, duration_secs
-        ) VALUES (?, ?, ?, ?, ?, ?, ?)
-        """,
-        (
-            cron_id,
-            started_at,
-            ended_at,
-            int(exit_code),
-            summary,
-            error,
-            float(duration_secs),
-        ),
-    )
+    if row_id:
+        conn.execute(
+            """
+            UPDATE cron_runs
+               SET ended_at=?, exit_code=?, summary=?, error=?, duration_secs=?
+             WHERE id=?
+            """,
+            (ended_at, int(exit_code), summary, error, float(duration_secs), int(row_id)),
+        )
+    else:
+        conn.execute(
+            """
+            INSERT INTO cron_runs (cron_id, started_at, ended_at, exit_code, summary, error, duration_secs)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (cron_id, started_at, ended_at, int(exit_code), summary, error, float(duration_secs)),
+        )
     conn.commit()
 finally:
     conn.close()
 PY
-then
-    mkdir -p "$SPOOL_DIR"
-    SPOOL_FILE="$SPOOL_DIR/${CRON_ID}-$(date +%Y%m%d-%H%M%S)-$$.json"
-    python3 - "$SPOOL_FILE" "$CRON_ID" "$STARTED_AT" "$ENDED_AT" "$EXIT_CODE" "$SUMMARY" "$ERROR" "$DURATION_SECS" <<'PY'
+    then
+        mkdir -p "$SPOOL_DIR"
+        local spool_file="$SPOOL_DIR/${CRON_ID}-$(date +%Y%m%d-%H%M%S)-$$.json"
+        python3 - "$spool_file" "$CRON_ID" "$STARTED_AT" "$ended_at" "$EXIT_CODE" "$summary" "$error" "$duration" <<'PY'
 from __future__ import annotations
-
 import json
 import sys
 from pathlib import Path
-
 spool_file, cron_id, started_at, ended_at, exit_code, summary, error, duration_secs = sys.argv[1:]
-payload = {
-    "cron_id": cron_id,
-    "started_at": started_at,
-    "ended_at": ended_at,
-    "exit_code": int(exit_code),
-    "summary": summary,
-    "error": error,
-    "duration_secs": float(duration_secs),
-}
-Path(spool_file).write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+Path(spool_file).write_text(
+    json.dumps({
+        "cron_id": cron_id,
+        "started_at": started_at,
+        "ended_at": ended_at,
+        "exit_code": int(exit_code),
+        "summary": summary,
+        "error": error,
+        "duration_secs": float(duration_secs),
+    }, indent=2, ensure_ascii=False) + "\n",
+    encoding="utf-8",
+)
 PY
-    echo "[nexo-cron-wrapper] DB write failed; spooled run to $SPOOL_FILE" >&2
-fi
+        echo "[nexo-cron-wrapper] DB write failed; spooled run to $spool_file" >&2
+    fi
+}
+
+cleanup() {
+    rm -f "$OUTPUT_FILE"
+}
+
+CHILD_PID=""
+
+on_signal() {
+    local sig="$1"
+    local code="$2"
+    SIGNAL_NAME="$sig"
+    EXIT_CODE="$code"
+    # Forward the signal to the child. Bash traps run AFTER the foreground
+    # command completes, which is why we launch the command in background
+    # and wait on its PID — otherwise a SIGTERM to the wrapper would be
+    # delivered only when the child finishes naturally, defeating the
+    # purpose of closing the cron_runs row on kill.
+    if [ -n "$CHILD_PID" ] && kill -0 "$CHILD_PID" 2>/dev/null; then
+        kill -TERM "$CHILD_PID" 2>/dev/null
+        # Brief grace period before escalating to SIGKILL so the child gets
+        # a chance to clean up on its own.
+        local waited=0
+        while [ $waited -lt 5 ] && kill -0 "$CHILD_PID" 2>/dev/null; do
+            sleep 1
+            waited=$((waited + 1))
+        done
+        kill -KILL "$CHILD_PID" 2>/dev/null
+    fi
+    finalize_row
+    cleanup
+    exit "$code"
+}
+
+trap cleanup EXIT
+trap 'on_signal SIGTERM 143' TERM
+trap 'on_signal SIGINT 130' INT
+trap 'on_signal SIGHUP 129' HUP
+
+"$@" > "$OUTPUT_FILE" 2>&1 &
+CHILD_PID=$!
+
+# `wait` is interruptible by signals — when the trap fires, wait returns
+# immediately and on_signal() takes over. When the child finishes
+# normally, wait yields its exit code and we fall through to finalize_row
+# for the happy path.
+wait "$CHILD_PID"
+EXIT_CODE=$?
+CHILD_PID=""
+
+finalize_row
 
-exit $EXIT_CODE
+exit "$EXIT_CODE"