nexo-brain 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +241 -0
- package/bin/create-nexo.js +593 -0
- package/package.json +32 -0
- package/scripts/pre-commit-check.sh +55 -0
- package/src/cognitive.py +1224 -0
- package/src/db.py +2283 -0
- package/src/hooks/caffeinate-guard.sh +8 -0
- package/src/hooks/capture-session.sh +19 -0
- package/src/hooks/session-start.sh +27 -0
- package/src/hooks/session-stop.sh +11 -0
- package/src/plugin_loader.py +136 -0
- package/src/plugins/__init__.py +0 -0
- package/src/plugins/agents.py +52 -0
- package/src/plugins/backup.py +103 -0
- package/src/plugins/cognitive_memory.py +305 -0
- package/src/plugins/entities.py +61 -0
- package/src/plugins/episodic_memory.py +391 -0
- package/src/plugins/evolution.py +113 -0
- package/src/plugins/guard.py +346 -0
- package/src/plugins/preferences.py +47 -0
- package/src/scripts/nexo-auto-update.py +213 -0
- package/src/scripts/nexo-catchup.py +179 -0
- package/src/scripts/nexo-cognitive-decay.py +82 -0
- package/src/scripts/nexo-daily-self-audit.py +532 -0
- package/src/scripts/nexo-postmortem-consolidator.py +594 -0
- package/src/scripts/nexo-sleep.py +762 -0
- package/src/scripts/nexo-synthesis.py +537 -0
- package/src/server.py +560 -0
- package/src/tools_coordination.py +102 -0
- package/src/tools_credentials.py +64 -0
- package/src/tools_learnings.py +180 -0
- package/src/tools_menu.py +208 -0
- package/src/tools_reminders.py +80 -0
- package/src/tools_reminders_crud.py +157 -0
- package/src/tools_sessions.py +169 -0
- package/src/tools_task_history.py +57 -0
- package/templates/CLAUDE.md.template +89 -0
|
@@ -0,0 +1,762 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
NEXO Sleep System — Daily memory cleanup and pruning.
|
|
4
|
+
|
|
5
|
+
Triggered hourly via LaunchAgent. Runs ONCE per day, first time the Mac is awake.
|
|
6
|
+
If interrupted (power loss, crash), resumes on next trigger.
|
|
7
|
+
|
|
8
|
+
Stage A — Mechanical cleanup (Python pure, always runs):
|
|
9
|
+
A1: Delete daily_summaries >90 days
|
|
10
|
+
A2: Delete session_archive >30 days
|
|
11
|
+
A3: Rotate coordination stdout logs >5MB
|
|
12
|
+
A4: Delete compressed_memories/week_*.md >180 days
|
|
13
|
+
A5: Trim heartbeat-log.json to 200 entries
|
|
14
|
+
A6: Trim reflection-log.json to 60 entries
|
|
15
|
+
A7: Delete daemon/logs/ dirs >14 days
|
|
16
|
+
|
|
17
|
+
Stage C — Learning Consolidation (Python pure, always runs):
|
|
18
|
+
C1: Duplicate detection (>80% word overlap in titles)
|
|
19
|
+
C2: Age distribution of learnings
|
|
20
|
+
C3: Category health (counts, hottest last 7d, categories >20)
|
|
21
|
+
C4: Contradiction detection (NUNCA pairs in same category)
|
|
22
|
+
|
|
23
|
+
Stage B — Intelligent pruning (Claude CLI, conditional):
|
|
24
|
+
Only activates if MEMORY.md >170 lines, nexo.db preferences table has >5 rows,
|
|
25
|
+
Uses Claude CLI (sonnet) to compress and prune.
|
|
26
|
+
|
|
27
|
+
Zero external dependencies beyond stdlib + sqlite3. Claude CLI for Stage B only.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
import fcntl
|
|
31
|
+
import json
|
|
32
|
+
import os
|
|
33
|
+
import re
|
|
34
|
+
import shutil
|
|
35
|
+
import sqlite3
|
|
36
|
+
import subprocess
|
|
37
|
+
import sys
|
|
38
|
+
from datetime import datetime, date, timedelta
|
|
39
|
+
from pathlib import Path
|
|
40
|
+
|
|
41
|
+
# ─── Paths ────────────────────────────────────────────────────────────────────
|
|
42
|
+
CLAUDE_DIR = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
|
|
43
|
+
BRAIN_DIR = CLAUDE_DIR / "brain"
|
|
44
|
+
COORD_DIR = CLAUDE_DIR / "coordination"
|
|
45
|
+
MEMORY_DIR = CLAUDE_DIR / "memory"
|
|
46
|
+
DAEMON_LOGS_DIR = CLAUDE_DIR / "daemon" / "logs"
|
|
47
|
+
|
|
48
|
+
DAILY_SUMMARIES_DIR = BRAIN_DIR / "daily_summaries"
|
|
49
|
+
SESSION_ARCHIVE_DIR = BRAIN_DIR / "session_archive"
|
|
50
|
+
COMPRESSED_MEMORIES_DIR = BRAIN_DIR / "compressed_memories"
|
|
51
|
+
|
|
52
|
+
HEARTBEAT_LOG = COORD_DIR / "heartbeat-log.json"
|
|
53
|
+
REFLECTION_LOG = COORD_DIR / "reflection-log.json"
|
|
54
|
+
SLEEP_LOG = COORD_DIR / "sleep-log.json"
|
|
55
|
+
|
|
56
|
+
MEMORY_MD = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo"))) / "brain" / "MEMORY.md"
|
|
57
|
+
NEXO_DB = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo"))) / "nexo.db"
|
|
58
|
+
|
|
59
|
+
LAST_RUN_FILE = COORD_DIR / "sleep-last-run"
|
|
60
|
+
LOCK_FILE = COORD_DIR / "sleep.lock"
|
|
61
|
+
PROCESS_LOCK = COORD_DIR / "sleep-process.lock"
|
|
62
|
+
|
|
63
|
+
TODAY = date.today()
|
|
64
|
+
NOW = datetime.now()
|
|
65
|
+
TIMESTAMP = NOW.strftime("%Y-%m-%d %H:%M")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# ─── Run-once & resume logic ────────────────────────────────────────────────
|
|
69
|
+
|
|
70
|
+
def already_ran_today() -> bool:
|
|
71
|
+
"""Check if sleep already completed today."""
|
|
72
|
+
if not LAST_RUN_FILE.exists():
|
|
73
|
+
return False
|
|
74
|
+
try:
|
|
75
|
+
last_date = LAST_RUN_FILE.read_text().strip()
|
|
76
|
+
return last_date == str(TODAY)
|
|
77
|
+
except Exception:
|
|
78
|
+
return False
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def was_interrupted() -> bool:
|
|
82
|
+
"""Check if a previous run was interrupted (lock file exists with dead PID)."""
|
|
83
|
+
if not LOCK_FILE.exists():
|
|
84
|
+
return False
|
|
85
|
+
try:
|
|
86
|
+
lock_data = json.loads(LOCK_FILE.read_text())
|
|
87
|
+
lock_date = lock_data.get("date", "")
|
|
88
|
+
if lock_date != str(TODAY):
|
|
89
|
+
LOCK_FILE.unlink()
|
|
90
|
+
return False
|
|
91
|
+
|
|
92
|
+
lock_pid = lock_data.get("pid")
|
|
93
|
+
if lock_pid:
|
|
94
|
+
try:
|
|
95
|
+
os.kill(lock_pid, 0)
|
|
96
|
+
log(f"Another instance running (PID {lock_pid}). Exiting.")
|
|
97
|
+
return False
|
|
98
|
+
except ProcessLookupError:
|
|
99
|
+
log(f"Interrupted run detected (phase: {lock_data.get('phase', '?')}, dead PID {lock_pid}). Resuming.")
|
|
100
|
+
return True
|
|
101
|
+
except PermissionError:
|
|
102
|
+
return False
|
|
103
|
+
else:
|
|
104
|
+
LOCK_FILE.unlink()
|
|
105
|
+
return False
|
|
106
|
+
except Exception:
|
|
107
|
+
LOCK_FILE.unlink(missing_ok=True)
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def get_interrupted_phase() -> str:
|
|
112
|
+
"""Get which phase was interrupted."""
|
|
113
|
+
try:
|
|
114
|
+
lock_data = json.loads(LOCK_FILE.read_text())
|
|
115
|
+
return lock_data.get("phase", "stage_a")
|
|
116
|
+
except Exception:
|
|
117
|
+
return "stage_a"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def set_lock(phase: str):
|
|
121
|
+
"""Set lock file indicating current phase with PID for race detection."""
|
|
122
|
+
save_json(LOCK_FILE, {
|
|
123
|
+
"date": str(TODAY),
|
|
124
|
+
"phase": phase,
|
|
125
|
+
"started": TIMESTAMP,
|
|
126
|
+
"pid": os.getpid()
|
|
127
|
+
})
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def mark_complete():
|
|
131
|
+
"""Mark today's run as complete."""
|
|
132
|
+
LAST_RUN_FILE.write_text(str(TODAY))
|
|
133
|
+
LOCK_FILE.unlink(missing_ok=True)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
|
137
|
+
|
|
138
|
+
def log(msg: str):
|
|
139
|
+
print(f"[{TIMESTAMP}] {msg}")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def load_json(path: Path, default=None):
|
|
143
|
+
if not path.exists():
|
|
144
|
+
return default if default is not None else {}
|
|
145
|
+
try:
|
|
146
|
+
return json.loads(path.read_text())
|
|
147
|
+
except Exception as e:
|
|
148
|
+
log(f"WARN: Failed to load {path}: {e}")
|
|
149
|
+
return default if default is not None else {}
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def save_json(path: Path, data):
|
|
153
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def parse_date_from_stem(stem: str) -> date | None:
|
|
158
|
+
"""Extract YYYY-MM-DD date from a filename stem."""
|
|
159
|
+
m = re.search(r'(\d{4}-\d{2}-\d{2})', stem)
|
|
160
|
+
if m:
|
|
161
|
+
try:
|
|
162
|
+
return date.fromisoformat(m.group(1))
|
|
163
|
+
except ValueError:
|
|
164
|
+
return None
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def append_sleep_log(entry: dict):
|
|
169
|
+
"""Append entry to sleep-log.json, keeping last 90 entries."""
|
|
170
|
+
entries = load_json(SLEEP_LOG, [])
|
|
171
|
+
if not isinstance(entries, list):
|
|
172
|
+
entries = []
|
|
173
|
+
entries.append(entry)
|
|
174
|
+
# Keep last 90
|
|
175
|
+
if len(entries) > 90:
|
|
176
|
+
entries = entries[-90:]
|
|
177
|
+
save_json(SLEEP_LOG, entries)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# ─── Stage A: Mechanical cleanup ─────────────────────────────────────────────
|
|
181
|
+
|
|
182
|
+
def stage_a_cleanup() -> dict:
|
|
183
|
+
"""
|
|
184
|
+
Pure Python cleanup. No LLM calls.
|
|
185
|
+
Returns stats dict with counts per sub-task.
|
|
186
|
+
"""
|
|
187
|
+
stats = {
|
|
188
|
+
"a1_daily_summaries_deleted": 0,
|
|
189
|
+
"a2_session_archives_deleted": 0,
|
|
190
|
+
"a3_logs_rotated": 0,
|
|
191
|
+
"a4_compressed_memories_deleted": 0,
|
|
192
|
+
"a5_heartbeat_trimmed": False,
|
|
193
|
+
"a6_reflection_trimmed": False,
|
|
194
|
+
"a7_daemon_logs_deleted": 0,
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
# A1: Delete daily_summaries/*.md >90 days
|
|
198
|
+
cutoff_90 = TODAY - timedelta(days=90)
|
|
199
|
+
if DAILY_SUMMARIES_DIR.exists():
|
|
200
|
+
for f in DAILY_SUMMARIES_DIR.glob("*.md"):
|
|
201
|
+
d = parse_date_from_stem(f.stem)
|
|
202
|
+
if d and d < cutoff_90:
|
|
203
|
+
try:
|
|
204
|
+
f.unlink()
|
|
205
|
+
stats["a1_daily_summaries_deleted"] += 1
|
|
206
|
+
log(f"A1: Deleted {f.name} (>{90}d)")
|
|
207
|
+
except Exception as e:
|
|
208
|
+
log(f"A1: WARN: Could not delete {f.name}: {e}")
|
|
209
|
+
|
|
210
|
+
# A2: Delete session_archive/*.jsonl >30 days
|
|
211
|
+
cutoff_30 = TODAY - timedelta(days=30)
|
|
212
|
+
if SESSION_ARCHIVE_DIR.exists():
|
|
213
|
+
for f in SESSION_ARCHIVE_DIR.glob("*.jsonl"):
|
|
214
|
+
d = parse_date_from_stem(f.stem)
|
|
215
|
+
if d and d < cutoff_30:
|
|
216
|
+
try:
|
|
217
|
+
f.unlink()
|
|
218
|
+
stats["a2_session_archives_deleted"] += 1
|
|
219
|
+
log(f"A2: Deleted {f.name} (>{30}d)")
|
|
220
|
+
except Exception as e:
|
|
221
|
+
log(f"A2: WARN: Could not delete {f.name}: {e}")
|
|
222
|
+
|
|
223
|
+
# A3: Rotate coordination/*-stdout.log if >5MB (keep last 500 lines)
|
|
224
|
+
if COORD_DIR.exists():
|
|
225
|
+
for f in COORD_DIR.glob("*-stdout.log"):
|
|
226
|
+
try:
|
|
227
|
+
if f.stat().st_size > 5 * 1024 * 1024: # >5MB
|
|
228
|
+
lines = f.read_text().splitlines()
|
|
229
|
+
keep = lines[-500:] if len(lines) > 500 else lines
|
|
230
|
+
f.write_text("\n".join(keep) + "\n")
|
|
231
|
+
stats["a3_logs_rotated"] += 1
|
|
232
|
+
log(f"A3: Rotated {f.name} ({len(lines)}→{len(keep)} lines)")
|
|
233
|
+
except Exception as e:
|
|
234
|
+
log(f"A3: WARN: Could not rotate {f.name}: {e}")
|
|
235
|
+
|
|
236
|
+
# A4: Delete compressed_memories/week_*.md >180 days
|
|
237
|
+
cutoff_180 = TODAY - timedelta(days=180)
|
|
238
|
+
if COMPRESSED_MEMORIES_DIR.exists():
|
|
239
|
+
for f in COMPRESSED_MEMORIES_DIR.glob("week_*.md"):
|
|
240
|
+
d = parse_date_from_stem(f.stem)
|
|
241
|
+
if d and d < cutoff_180:
|
|
242
|
+
try:
|
|
243
|
+
f.unlink()
|
|
244
|
+
stats["a4_compressed_memories_deleted"] += 1
|
|
245
|
+
log(f"A4: Deleted {f.name} (>{180}d)")
|
|
246
|
+
except Exception as e:
|
|
247
|
+
log(f"A4: WARN: Could not delete {f.name}: {e}")
|
|
248
|
+
|
|
249
|
+
# A5: Trim heartbeat-log.json to 200 entries
|
|
250
|
+
if HEARTBEAT_LOG.exists():
|
|
251
|
+
try:
|
|
252
|
+
data = load_json(HEARTBEAT_LOG, [])
|
|
253
|
+
if isinstance(data, list) and len(data) > 200:
|
|
254
|
+
before = len(data)
|
|
255
|
+
data = data[-200:]
|
|
256
|
+
save_json(HEARTBEAT_LOG, data)
|
|
257
|
+
stats["a5_heartbeat_trimmed"] = True
|
|
258
|
+
log(f"A5: Trimmed heartbeat-log.json {before}→200 entries")
|
|
259
|
+
except Exception as e:
|
|
260
|
+
log(f"A5: WARN: {e}")
|
|
261
|
+
|
|
262
|
+
# A6: Trim reflection-log.json to 60 entries
|
|
263
|
+
if REFLECTION_LOG.exists():
|
|
264
|
+
try:
|
|
265
|
+
data = load_json(REFLECTION_LOG, [])
|
|
266
|
+
if isinstance(data, list) and len(data) > 60:
|
|
267
|
+
before = len(data)
|
|
268
|
+
data = data[-60:]
|
|
269
|
+
save_json(REFLECTION_LOG, data)
|
|
270
|
+
stats["a6_reflection_trimmed"] = True
|
|
271
|
+
log(f"A6: Trimmed reflection-log.json {before}→60 entries")
|
|
272
|
+
except Exception as e:
|
|
273
|
+
log(f"A6: WARN: {e}")
|
|
274
|
+
|
|
275
|
+
# A7: Delete daemon/logs/ dirs >14 days (subdirs named YYYY-MM-DD)
|
|
276
|
+
cutoff_14 = TODAY - timedelta(days=14)
|
|
277
|
+
if DAEMON_LOGS_DIR.exists():
|
|
278
|
+
for d_path in sorted(DAEMON_LOGS_DIR.iterdir()):
|
|
279
|
+
if not d_path.is_dir():
|
|
280
|
+
continue
|
|
281
|
+
d = parse_date_from_stem(d_path.name)
|
|
282
|
+
if d and d < cutoff_14:
|
|
283
|
+
try:
|
|
284
|
+
shutil.rmtree(d_path)
|
|
285
|
+
stats["a7_daemon_logs_deleted"] += 1
|
|
286
|
+
log(f"A7: Deleted daemon/logs/{d_path.name}/ (>{14}d)")
|
|
287
|
+
except Exception as e:
|
|
288
|
+
log(f"A7: WARN: Could not delete {d_path.name}: {e}")
|
|
289
|
+
|
|
290
|
+
# A8: Delete cortex/logs/*.log >7 days, truncate launchd logs >5MB
|
|
291
|
+
cutoff_7 = TODAY - timedelta(days=7)
|
|
292
|
+
cortex_logs = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo"))) / "cortex" / "logs"
|
|
293
|
+
if cortex_logs.exists():
|
|
294
|
+
for f in cortex_logs.glob("*.log"):
|
|
295
|
+
if f.name.startswith("launchd-"):
|
|
296
|
+
try:
|
|
297
|
+
if f.stat().st_size > 5 * 1024 * 1024:
|
|
298
|
+
lines = f.read_text().splitlines()
|
|
299
|
+
keep = lines[-500:] if len(lines) > 500 else lines
|
|
300
|
+
f.write_text("\n".join(keep) + "\n")
|
|
301
|
+
stats["a3_logs_rotated"] += 1
|
|
302
|
+
log(f"A8: Truncated cortex {f.name}")
|
|
303
|
+
except Exception as e:
|
|
304
|
+
log(f"A8: WARN: {e}")
|
|
305
|
+
continue
|
|
306
|
+
d = parse_date_from_stem(f.stem)
|
|
307
|
+
if d and d < cutoff_7:
|
|
308
|
+
try:
|
|
309
|
+
f.unlink()
|
|
310
|
+
log(f"A8: Deleted cortex log {f.name} (>7d)")
|
|
311
|
+
except Exception as e:
|
|
312
|
+
log(f"A8: WARN: Could not delete {f.name}: {e}")
|
|
313
|
+
|
|
314
|
+
return stats
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
# ─── Stage C: Learning Consolidation ─────────────────────────────────────────
|
|
318
|
+
|
|
319
|
+
STOPWORDS = {
|
|
320
|
+
"el", "la", "los", "las", "un", "una", "unos", "unas",
|
|
321
|
+
"de", "del", "al", "en", "y", "o", "a", "con", "por", "para",
|
|
322
|
+
"que", "es", "se", "no", "si", "lo", "le", "su", "sus",
|
|
323
|
+
"the", "a", "an", "of", "in", "and", "or", "to", "for", "is",
|
|
324
|
+
"it", "on", "at", "by", "from", "with", "not", "be", "as",
|
|
325
|
+
"this", "that", "are", "was", "were",
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _title_words(title: str) -> set:
|
|
330
|
+
"""Lowercase, tokenize, remove stopwords from a title."""
|
|
331
|
+
words = re.findall(r'[a-záéíóúüñA-ZÁÉÍÓÚÜÑ\w]+', title.lower())
|
|
332
|
+
return {w for w in words if w not in STOPWORDS and len(w) > 2}
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _word_overlap(words_a: set, words_b: set) -> float:
|
|
336
|
+
"""Jaccard-like overlap: intersection / union."""
|
|
337
|
+
if not words_a or not words_b:
|
|
338
|
+
return 0.0
|
|
339
|
+
return len(words_a & words_b) / len(words_a | words_b)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def stage_c_learning_consolidation() -> dict:
|
|
343
|
+
"""
|
|
344
|
+
Pure Python analysis of the learnings table in nexo.db.
|
|
345
|
+
Reads only — no deletions.
|
|
346
|
+
Returns stats dict stored under run_log['stage_c'].
|
|
347
|
+
"""
|
|
348
|
+
stats = {
|
|
349
|
+
"total_learnings": 0,
|
|
350
|
+
"potential_duplicates": [], # max 10
|
|
351
|
+
"age_distribution": {"<7d": 0, "7-30d": 0, "30-90d": 0, ">90d": 0},
|
|
352
|
+
"category_counts": {},
|
|
353
|
+
"hottest_category_7d": None,
|
|
354
|
+
"categories_over_20": [],
|
|
355
|
+
"potential_contradictions": [], # max 5
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
if not NEXO_DB.exists():
|
|
359
|
+
log("Stage C: nexo.db not found, skipping.")
|
|
360
|
+
return stats
|
|
361
|
+
|
|
362
|
+
try:
|
|
363
|
+
conn = sqlite3.connect(str(NEXO_DB))
|
|
364
|
+
conn.row_factory = sqlite3.Row
|
|
365
|
+
cursor = conn.cursor()
|
|
366
|
+
|
|
367
|
+
# Check table exists
|
|
368
|
+
cursor.execute(
|
|
369
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='learnings'"
|
|
370
|
+
)
|
|
371
|
+
if not cursor.fetchone():
|
|
372
|
+
log("Stage C: learnings table not found, skipping.")
|
|
373
|
+
conn.close()
|
|
374
|
+
return stats
|
|
375
|
+
|
|
376
|
+
cursor.execute(
|
|
377
|
+
"SELECT id, title, content, category, created_at FROM learnings ORDER BY id"
|
|
378
|
+
)
|
|
379
|
+
rows = cursor.fetchall()
|
|
380
|
+
conn.close()
|
|
381
|
+
except Exception as e:
|
|
382
|
+
log(f"Stage C: DB error: {e}")
|
|
383
|
+
return stats
|
|
384
|
+
|
|
385
|
+
if not rows:
|
|
386
|
+
log("Stage C: No learnings found.")
|
|
387
|
+
return stats
|
|
388
|
+
|
|
389
|
+
stats["total_learnings"] = len(rows)
|
|
390
|
+
now_dt = datetime.now()
|
|
391
|
+
cutoff_7 = now_dt - timedelta(days=7)
|
|
392
|
+
cutoff_30 = now_dt - timedelta(days=30)
|
|
393
|
+
cutoff_90 = now_dt - timedelta(days=90)
|
|
394
|
+
|
|
395
|
+
# Pre-compute per-row data
|
|
396
|
+
parsed = []
|
|
397
|
+
category_7d_counts: dict[str, int] = {}
|
|
398
|
+
|
|
399
|
+
for row in rows:
|
|
400
|
+
# Parse created_at (stored as epoch float or ISO string)
|
|
401
|
+
created_dt = None
|
|
402
|
+
raw_ts = row["created_at"]
|
|
403
|
+
if raw_ts:
|
|
404
|
+
# Try epoch first (nexo.db uses epoch floats)
|
|
405
|
+
try:
|
|
406
|
+
ts_float = float(raw_ts)
|
|
407
|
+
if ts_float > 1_000_000_000: # reasonable epoch
|
|
408
|
+
created_dt = datetime.fromtimestamp(ts_float)
|
|
409
|
+
except (ValueError, TypeError, OSError):
|
|
410
|
+
pass
|
|
411
|
+
# Fallback to ISO string formats
|
|
412
|
+
if created_dt is None:
|
|
413
|
+
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
|
|
414
|
+
try:
|
|
415
|
+
created_dt = datetime.strptime(str(raw_ts)[:19], fmt)
|
|
416
|
+
break
|
|
417
|
+
except ValueError:
|
|
418
|
+
continue
|
|
419
|
+
|
|
420
|
+
words = _title_words(row["title"] or "")
|
|
421
|
+
cat = (row["category"] or "uncategorized").strip()
|
|
422
|
+
|
|
423
|
+
parsed.append({
|
|
424
|
+
"id": row["id"],
|
|
425
|
+
"title": row["title"] or "",
|
|
426
|
+
"words": words,
|
|
427
|
+
"category": cat,
|
|
428
|
+
"created_dt": created_dt,
|
|
429
|
+
})
|
|
430
|
+
|
|
431
|
+
# C2: age distribution
|
|
432
|
+
if created_dt:
|
|
433
|
+
if created_dt >= cutoff_7:
|
|
434
|
+
stats["age_distribution"]["<7d"] += 1
|
|
435
|
+
category_7d_counts[cat] = category_7d_counts.get(cat, 0) + 1
|
|
436
|
+
elif created_dt >= cutoff_30:
|
|
437
|
+
stats["age_distribution"]["7-30d"] += 1
|
|
438
|
+
elif created_dt >= cutoff_90:
|
|
439
|
+
stats["age_distribution"]["30-90d"] += 1
|
|
440
|
+
else:
|
|
441
|
+
stats["age_distribution"][">90d"] += 1
|
|
442
|
+
else:
|
|
443
|
+
# Unknown age → bucket as >90d
|
|
444
|
+
stats["age_distribution"][">90d"] += 1
|
|
445
|
+
|
|
446
|
+
# C3: category counts
|
|
447
|
+
stats["category_counts"][cat] = stats["category_counts"].get(cat, 0) + 1
|
|
448
|
+
|
|
449
|
+
# C3: hottest category last 7d + categories over 20
|
|
450
|
+
if category_7d_counts:
|
|
451
|
+
stats["hottest_category_7d"] = max(category_7d_counts, key=lambda k: category_7d_counts[k])
|
|
452
|
+
stats["categories_over_20"] = [
|
|
453
|
+
cat for cat, cnt in stats["category_counts"].items() if cnt > 20
|
|
454
|
+
]
|
|
455
|
+
|
|
456
|
+
# C1: Duplicate detection — O(n²) but learnings table is small
|
|
457
|
+
duplicates = []
|
|
458
|
+
for i in range(len(parsed)):
|
|
459
|
+
if len(duplicates) >= 10:
|
|
460
|
+
break
|
|
461
|
+
for j in range(i + 1, len(parsed)):
|
|
462
|
+
if len(duplicates) >= 10:
|
|
463
|
+
break
|
|
464
|
+
overlap = _word_overlap(parsed[i]["words"], parsed[j]["words"])
|
|
465
|
+
if overlap >= 0.80:
|
|
466
|
+
duplicates.append({
|
|
467
|
+
"id1": parsed[i]["id"],
|
|
468
|
+
"id2": parsed[j]["id"],
|
|
469
|
+
"title1": parsed[i]["title"],
|
|
470
|
+
"title2": parsed[j]["title"],
|
|
471
|
+
"overlap": round(overlap, 2),
|
|
472
|
+
})
|
|
473
|
+
stats["potential_duplicates"] = duplicates
|
|
474
|
+
|
|
475
|
+
# C4: Contradiction detection — NUNCA pairs in same category
|
|
476
|
+
nunca_entries = [p for p in parsed if "nunca" in p["title"].lower()]
|
|
477
|
+
contradictions = []
|
|
478
|
+
for nunca in nunca_entries:
|
|
479
|
+
if len(contradictions) >= 5:
|
|
480
|
+
break
|
|
481
|
+
# Look for same-category entries that don't contain NUNCA
|
|
482
|
+
# and whose remaining words overlap significantly (same subject, opposite stance)
|
|
483
|
+
nunca_words_no_nunca = nunca["words"] - {"nunca"}
|
|
484
|
+
for other in parsed:
|
|
485
|
+
if len(contradictions) >= 5:
|
|
486
|
+
break
|
|
487
|
+
if other["id"] == nunca["id"]:
|
|
488
|
+
continue
|
|
489
|
+
if other["category"] != nunca["category"]:
|
|
490
|
+
continue
|
|
491
|
+
if "nunca" in other["title"].lower():
|
|
492
|
+
continue
|
|
493
|
+
# Check if they share meaningful subject words
|
|
494
|
+
overlap = _word_overlap(nunca_words_no_nunca, other["words"])
|
|
495
|
+
if overlap >= 0.50:
|
|
496
|
+
contradictions.append({
|
|
497
|
+
"id1": nunca["id"],
|
|
498
|
+
"id2": other["id"],
|
|
499
|
+
"title1": nunca["title"],
|
|
500
|
+
"title2": other["title"],
|
|
501
|
+
})
|
|
502
|
+
stats["potential_contradictions"] = contradictions
|
|
503
|
+
|
|
504
|
+
log(f"Stage C: {stats['total_learnings']} learnings analyzed. "
|
|
505
|
+
f"Potential duplicates: {len(duplicates)}. "
|
|
506
|
+
f"Categories over 20: {len(stats['categories_over_20'])}. "
|
|
507
|
+
f"Potential contradictions: {len(contradictions)}.")
|
|
508
|
+
if stats["hottest_category_7d"]:
|
|
509
|
+
log(f"Stage C: Hottest category last 7d: {stats['hottest_category_7d']} "
|
|
510
|
+
f"({category_7d_counts.get(stats['hottest_category_7d'], 0)} new).")
|
|
511
|
+
|
|
512
|
+
return stats
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
# ─── Stage B: Intelligent pruning (Claude CLI) ──────────────────────────────
|
|
516
|
+
|
|
517
|
+
def check_stage_b_conditions() -> dict:
|
|
518
|
+
"""
|
|
519
|
+
Check if Stage B should activate.
|
|
520
|
+
Returns dict with condition results and whether to trigger.
|
|
521
|
+
"""
|
|
522
|
+
conditions = {
|
|
523
|
+
"memory_md_lines": 0,
|
|
524
|
+
"memory_md_over_limit": False,
|
|
525
|
+
"preferences_auto_sections": 0,
|
|
526
|
+
"preferences_over_limit": False,
|
|
527
|
+
"should_trigger": False,
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
# Check MEMORY.md line count
|
|
531
|
+
if MEMORY_MD.exists():
|
|
532
|
+
try:
|
|
533
|
+
lines = MEMORY_MD.read_text().splitlines()
|
|
534
|
+
conditions["memory_md_lines"] = len(lines)
|
|
535
|
+
conditions["memory_md_over_limit"] = len(lines) > 170
|
|
536
|
+
except Exception as e:
|
|
537
|
+
log(f"Stage B check: WARN reading MEMORY.md: {e}")
|
|
538
|
+
|
|
539
|
+
# Check preferences count in SQLite
|
|
540
|
+
if NEXO_DB.exists():
|
|
541
|
+
try:
|
|
542
|
+
conn = sqlite3.connect(str(NEXO_DB))
|
|
543
|
+
cursor = conn.cursor()
|
|
544
|
+
cursor.execute("SELECT COUNT(*) FROM preferences")
|
|
545
|
+
count = cursor.fetchone()[0]
|
|
546
|
+
conn.close()
|
|
547
|
+
conditions["preferences_auto_sections"] = count
|
|
548
|
+
conditions["preferences_over_limit"] = count > 5
|
|
549
|
+
except Exception as e:
|
|
550
|
+
log(f"Stage B check: WARN reading nexo.db preferences: {e}")
|
|
551
|
+
|
|
552
|
+
conditions["should_trigger"] = (
|
|
553
|
+
conditions["memory_md_over_limit"]
|
|
554
|
+
or conditions["preferences_over_limit"]
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
return conditions
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def build_stage_b_prompt(conditions: dict) -> str:
|
|
561
|
+
"""Build the prompt for Claude CLI based on which conditions triggered."""
|
|
562
|
+
tasks = []
|
|
563
|
+
|
|
564
|
+
if conditions["memory_md_over_limit"]:
|
|
565
|
+
tasks.append(f"""TAREA 1: MEMORY.md ({conditions['memory_md_lines']} lineas, limite 200)
|
|
566
|
+
Archivo: {MEMORY_MD}
|
|
567
|
+
Lee con Read tool, comprime incidentes resueltos >21 dias, fusiona duplicados, mantener <180 lineas.
|
|
568
|
+
PRESERVA toda la estructura de secciones existente. No elimines secciones enteras.""")
|
|
569
|
+
|
|
570
|
+
if conditions["preferences_over_limit"]:
|
|
571
|
+
tasks.append(f"""TAREA 2: preferences en SQLite ({conditions['preferences_auto_sections']} registros)
|
|
572
|
+
DB: {NEXO_DB}, tabla: preferences (columnas: key, value, category, updated_at)
|
|
573
|
+
Conecta con sqlite3. Elimina preferencias duplicadas (mismo key) manteniendo la mas reciente.
|
|
574
|
+
Elimina preferencias con updated_at mas antiguo de 30 dias si hay un duplicado mas reciente.
|
|
575
|
+
Reporta cuantos registros eliminaste.""")
|
|
576
|
+
|
|
577
|
+
if not tasks:
|
|
578
|
+
return ""
|
|
579
|
+
|
|
580
|
+
tasks_str = "\n\n".join(tasks)
|
|
581
|
+
|
|
582
|
+
return f"""You are the NEXO Sleep System. Your job is to PRUNE memory.
|
|
583
|
+
You are NOT interactive. Do NOT wait for input. Execute these tasks and exit.
|
|
584
|
+
|
|
585
|
+
ABSOLUTE RULES:
|
|
586
|
+
- NEVER delete credentials, tokens, account IDs, API endpoints, keys, secrets.
|
|
587
|
+
- NEVER delete operational rules marked as critical or high priority.
|
|
588
|
+
- NEVER delete infrastructure information (servers, repos, deploys).
|
|
589
|
+
- You CAN merge redundant sections.
|
|
590
|
+
- You CAN remove technical info that was fixed >30 days ago and never referenced since.
|
|
591
|
+
- You CAN compress long paragraphs into concise bullets.
|
|
592
|
+
- Every line you remove must have a clear reason. When in doubt, DO NOT delete.
|
|
593
|
+
|
|
594
|
+
{tasks_str}
|
|
595
|
+
|
|
596
|
+
Al terminar, imprime un resumen JSON con las acciones realizadas."""
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def run_stage_b(conditions: dict) -> dict:
|
|
600
|
+
"""Run Stage B using Claude CLI."""
|
|
601
|
+
prompt = build_stage_b_prompt(conditions)
|
|
602
|
+
if not prompt:
|
|
603
|
+
return {"skipped": True, "reason": "No tasks to run"}
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
log("Stage B: Invoking Claude CLI (sonnet)...")
|
|
607
|
+
|
|
608
|
+
try:
|
|
609
|
+
env = os.environ.copy()
|
|
610
|
+
# Remove env vars that would cause Claude CLI to think it's inside Claude Code
|
|
611
|
+
env.pop("CLAUDECODE", None)
|
|
612
|
+
env.pop("CLAUDE_CODE", None)
|
|
613
|
+
|
|
614
|
+
result = subprocess.run(
|
|
615
|
+
capture_output=True,
|
|
616
|
+
text=True,
|
|
617
|
+
timeout=300,
|
|
618
|
+
env=env
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
stdout = result.stdout.strip() if result.stdout else ""
|
|
622
|
+
stderr = result.stderr.strip() if result.stderr else ""
|
|
623
|
+
|
|
624
|
+
if result.returncode != 0:
|
|
625
|
+
log(f"Stage B: Claude CLI returned code {result.returncode}")
|
|
626
|
+
if stderr:
|
|
627
|
+
log(f"Stage B: stderr: {stderr[:500]}")
|
|
628
|
+
return {
|
|
629
|
+
"returncode": result.returncode,
|
|
630
|
+
"stderr": stderr[:500],
|
|
631
|
+
"stdout": stdout[:500],
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
log(f"Stage B: Completed. Output length: {len(stdout)} chars")
|
|
635
|
+
return {
|
|
636
|
+
"returncode": 0,
|
|
637
|
+
"output_length": len(stdout),
|
|
638
|
+
"output_preview": stdout[:800],
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
except subprocess.TimeoutExpired:
|
|
642
|
+
log("Stage B: Claude CLI timed out (300s)")
|
|
643
|
+
return {"error": "timeout"}
|
|
644
|
+
except Exception as e:
|
|
645
|
+
log(f"Stage B: Exception: {e}")
|
|
646
|
+
return {"error": str(e)}
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
# ─── Main ─────────────────────────────────────────────────────────────────────
|
|
650
|
+
|
|
651
|
+
def main():
|
|
652
|
+
log("=" * 60)
|
|
653
|
+
log("NEXO Sleep System starting")
|
|
654
|
+
|
|
655
|
+
# Process lock via fcntl to prevent concurrent instances
|
|
656
|
+
try:
|
|
657
|
+
lock_fd = open(PROCESS_LOCK, "w")
|
|
658
|
+
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
659
|
+
lock_fd.write(str(os.getpid()))
|
|
660
|
+
lock_fd.flush()
|
|
661
|
+
except (IOError, OSError):
|
|
662
|
+
log("Another sleep instance is already running. Exiting.")
|
|
663
|
+
sys.exit(0)
|
|
664
|
+
|
|
665
|
+
try:
|
|
666
|
+
# Check if already completed today
|
|
667
|
+
if already_ran_today():
|
|
668
|
+
log("Already ran today. Exiting.")
|
|
669
|
+
sys.exit(0)
|
|
670
|
+
|
|
671
|
+
# Determine start phase (for resume after interruption)
|
|
672
|
+
start_phase = "stage_a"
|
|
673
|
+
if was_interrupted():
|
|
674
|
+
start_phase = get_interrupted_phase()
|
|
675
|
+
log(f"Resuming from phase: {start_phase}")
|
|
676
|
+
|
|
677
|
+
run_log = {
|
|
678
|
+
"date": str(TODAY),
|
|
679
|
+
"started": TIMESTAMP,
|
|
680
|
+
"stage_a": None,
|
|
681
|
+
"stage_c": None,
|
|
682
|
+
"stage_b_conditions": None,
|
|
683
|
+
"stage_b": None,
|
|
684
|
+
"completed": None,
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
# Stage A: Mechanical cleanup
|
|
688
|
+
if start_phase in ("stage_a",):
|
|
689
|
+
set_lock("stage_a")
|
|
690
|
+
log("─── Stage A: Mechanical cleanup ───")
|
|
691
|
+
stage_a_stats = stage_a_cleanup()
|
|
692
|
+
run_log["stage_a"] = stage_a_stats
|
|
693
|
+
|
|
694
|
+
total_cleaned = (
|
|
695
|
+
stage_a_stats["a1_daily_summaries_deleted"]
|
|
696
|
+
+ stage_a_stats["a2_session_archives_deleted"]
|
|
697
|
+
+ stage_a_stats["a3_logs_rotated"]
|
|
698
|
+
+ stage_a_stats["a4_compressed_memories_deleted"]
|
|
699
|
+
+ stage_a_stats["a7_daemon_logs_deleted"]
|
|
700
|
+
)
|
|
701
|
+
log(f"Stage A complete: {total_cleaned} items cleaned, "
|
|
702
|
+
f"heartbeat trimmed={stage_a_stats['a5_heartbeat_trimmed']}, "
|
|
703
|
+
f"reflection trimmed={stage_a_stats['a6_reflection_trimmed']}")
|
|
704
|
+
|
|
705
|
+
# Stage C: Learning Consolidation (always runs, pure Python)
|
|
706
|
+
if start_phase in ("stage_a", "stage_c", "stage_b"):
|
|
707
|
+
set_lock("stage_c")
|
|
708
|
+
log("─── Stage C: Learning Consolidation ───")
|
|
709
|
+
stage_c_stats = stage_c_learning_consolidation()
|
|
710
|
+
run_log["stage_c"] = stage_c_stats
|
|
711
|
+
|
|
712
|
+
# Stage B: Intelligent pruning (conditional)
|
|
713
|
+
if start_phase in ("stage_a", "stage_c", "stage_b"):
|
|
714
|
+
set_lock("stage_b")
|
|
715
|
+
log("─── Stage B: Checking conditions ───")
|
|
716
|
+
conditions = check_stage_b_conditions()
|
|
717
|
+
run_log["stage_b_conditions"] = conditions
|
|
718
|
+
|
|
719
|
+
log(f" MEMORY.md: {conditions['memory_md_lines']} lines "
|
|
720
|
+
f"(trigger={conditions['memory_md_over_limit']})")
|
|
721
|
+
log(f" nexo.db preferences: {conditions['preferences_auto_sections']} rows "
|
|
722
|
+
f"(trigger={conditions['preferences_over_limit']})")
|
|
723
|
+
|
|
724
|
+
if conditions["should_trigger"]:
|
|
725
|
+
log("Stage B: Conditions met, running intelligent pruning...")
|
|
726
|
+
stage_b_result = run_stage_b(conditions)
|
|
727
|
+
run_log["stage_b"] = stage_b_result
|
|
728
|
+
else:
|
|
729
|
+
log("Stage B: No conditions met, skipping.")
|
|
730
|
+
run_log["stage_b"] = {"skipped": True, "reason": "No conditions met"}
|
|
731
|
+
|
|
732
|
+
# Mark complete
|
|
733
|
+
run_log["completed"] = datetime.now().strftime("%Y-%m-%d %H:%M")
|
|
734
|
+
mark_complete()
|
|
735
|
+
append_sleep_log(run_log)
|
|
736
|
+
|
|
737
|
+
log(f"NEXO Sleep complete at {run_log['completed']}")
|
|
738
|
+
|
|
739
|
+
# Register successful run for catch-up
|
|
740
|
+
try:
|
|
741
|
+
import json as _json
|
|
742
|
+
_state_file = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo"))) / "operations" / ".catchup-state.json"
|
|
743
|
+
_state = _json.loads(_state_file.read_text()) if _state_file.exists() else {}
|
|
744
|
+
_state["sleep"] = datetime.now().isoformat()
|
|
745
|
+
_state_file.write_text(_json.dumps(_state, indent=2))
|
|
746
|
+
except Exception:
|
|
747
|
+
pass
|
|
748
|
+
|
|
749
|
+
log("=" * 60)
|
|
750
|
+
|
|
751
|
+
finally:
|
|
752
|
+
# Release process lock
|
|
753
|
+
try:
|
|
754
|
+
fcntl.flock(lock_fd, fcntl.LOCK_UN)
|
|
755
|
+
lock_fd.close()
|
|
756
|
+
PROCESS_LOCK.unlink(missing_ok=True)
|
|
757
|
+
except Exception:
|
|
758
|
+
pass
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
if __name__ == "__main__":
|
|
762
|
+
main()
|