nexo-brain 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +33 -0
- package/.mcp.json +12 -0
- package/README.md +48 -23
- package/bin/nexo-brain.js +65 -33
- package/hooks/hooks.json +14 -0
- package/package.json +15 -3
- package/src/auto_update.py +79 -2
- package/src/cli.py +490 -11
- package/src/cron_recovery.py +283 -0
- package/src/crons/manifest.json +79 -21
- package/src/crons/sync.py +132 -27
- package/src/db/__init__.py +11 -0
- package/src/db/_personal_scripts.py +548 -0
- package/src/db/_schema.py +44 -1
- package/src/doctor/providers/runtime.py +272 -75
- package/src/evolution_cycle.py +90 -7
- package/src/nexo.db +0 -0
- package/src/plugins/evolution.py +9 -2
- package/src/plugins/personal_scripts.py +117 -0
- package/src/plugins/schedule.py +116 -27
- package/src/script_registry.py +877 -28
- package/src/scripts/nexo-catchup.py +74 -109
- package/src/scripts/nexo-evolution-run.py +178 -67
- package/src/scripts/nexo-watchdog.sh +242 -54
- package/src/tools_learnings.py +8 -0
- package/templates/launchagents/com.nexo.catchup.plist +7 -6
- package/templates/script-template.py +3 -0
- package/templates/script-template.sh +13 -0
- package/src/scripts/nexo-day-orchestrator.sh +0 -139
|
@@ -16,6 +16,7 @@ NEXO_DIR="$NEXO_HOME"
|
|
|
16
16
|
CORTEX_DIR="$NEXO_HOME/brain"
|
|
17
17
|
OPS_DIR="$NEXO_HOME/operations"
|
|
18
18
|
LOG_DIR="$NEXO_HOME/logs"
|
|
19
|
+
DB_PATH="$NEXO_HOME/data/nexo.db"
|
|
19
20
|
LOG="$LOG_DIR/watchdog.log"
|
|
20
21
|
STATUS_JSON="$OPS_DIR/watchdog-status.json"
|
|
21
22
|
REPORT_TXT="$OPS_DIR/watchdog-report.txt"
|
|
@@ -62,12 +63,25 @@ import json, sys, platform
|
|
|
62
63
|
|
|
63
64
|
nexo_home = '$NEXO_HOME'
|
|
64
65
|
is_mac = platform.system() == 'Darwin'
|
|
66
|
+
optionals_file = '$NEXO_HOME/config/optionals.json'
|
|
67
|
+
optionals = {}
|
|
65
68
|
|
|
66
69
|
with open('$MANIFEST_FILE') as f:
|
|
67
70
|
data = json.load(f)
|
|
68
71
|
|
|
72
|
+
try:
|
|
73
|
+
with open(optionals_file) as f:
|
|
74
|
+
maybe = json.load(f)
|
|
75
|
+
if isinstance(maybe, dict):
|
|
76
|
+
optionals = {str(k): bool(v) for k, v in maybe.items()}
|
|
77
|
+
except Exception:
|
|
78
|
+
optionals = {}
|
|
79
|
+
|
|
69
80
|
for c in data.get('crons', []):
|
|
70
81
|
cid = c['id']
|
|
82
|
+
optional_key = c.get('optional')
|
|
83
|
+
if optional_key and not optionals.get(optional_key, False):
|
|
84
|
+
continue
|
|
71
85
|
name = cid.replace('-', ' ').title()
|
|
72
86
|
# Use the right service identifier per platform
|
|
73
87
|
if is_mac:
|
|
@@ -77,10 +91,21 @@ for c in data.get('crons', []):
|
|
|
77
91
|
stdout_log = nexo_home + '/logs/' + cid + '-stdout.log'
|
|
78
92
|
stderr_log = nexo_home + '/logs/' + cid + '-stderr.log'
|
|
79
93
|
|
|
94
|
+
recovery_policy = c.get('recovery_policy')
|
|
95
|
+
if not recovery_policy:
|
|
96
|
+
if c.get('keep_alive') or 'interval_seconds' in c:
|
|
97
|
+
recovery_policy = 'restart'
|
|
98
|
+
elif 'schedule' in c:
|
|
99
|
+
recovery_policy = 'catchup'
|
|
100
|
+
else:
|
|
101
|
+
recovery_policy = 'none'
|
|
102
|
+
run_at_load = bool(c.get('run_at_load') or (c.get('run_on_boot') and 'interval_seconds' in c and not c.get('keep_alive')))
|
|
103
|
+
|
|
80
104
|
# Derive max_stale_secs and schedule_desc from schedule config
|
|
81
|
-
if c.get('
|
|
105
|
+
if c.get('keep_alive'):
|
|
82
106
|
max_stale = 0
|
|
83
|
-
schedule_desc = '
|
|
107
|
+
schedule_desc = 'KeepAlive'
|
|
108
|
+
proc_grep = c.get('script', '').split('/')[-1]
|
|
84
109
|
elif 'interval_seconds' in c:
|
|
85
110
|
iv = c['interval_seconds']
|
|
86
111
|
# Allow 2x the interval before WARN
|
|
@@ -89,6 +114,9 @@ for c in data.get('crons', []):
|
|
|
89
114
|
schedule_desc = f'Every {iv // 3600}h'
|
|
90
115
|
else:
|
|
91
116
|
schedule_desc = f'Every {iv // 60} min'
|
|
117
|
+
if run_at_load:
|
|
118
|
+
schedule_desc += ' + boot'
|
|
119
|
+
proc_grep = ''
|
|
92
120
|
elif 'schedule' in c:
|
|
93
121
|
s = c['schedule']
|
|
94
122
|
h = s.get('hour', 0)
|
|
@@ -100,14 +128,19 @@ for c in data.get('crons', []):
|
|
|
100
128
|
else:
|
|
101
129
|
schedule_desc = f'Daily {h}:{m:02d}'
|
|
102
130
|
max_stale = 90000 # ~25h
|
|
131
|
+
proc_grep = ''
|
|
132
|
+
elif run_at_load:
|
|
133
|
+
max_stale = 0
|
|
134
|
+
schedule_desc = 'RunAtLoad once'
|
|
135
|
+
proc_grep = ''
|
|
103
136
|
else:
|
|
104
137
|
max_stale = 0
|
|
105
138
|
schedule_desc = 'unknown'
|
|
139
|
+
proc_grep = ''
|
|
106
140
|
|
|
107
141
|
mon_type = 'core' if c.get('core') else 'personal'
|
|
108
|
-
proc_grep = '' # manifest crons are one-shot, no persistent process
|
|
109
142
|
|
|
110
|
-
print(f'{name}|{svc_id}|{stdout_log}|{stderr_log}|{max_stale}|{proc_grep}|{schedule_desc}|{mon_type}')
|
|
143
|
+
print(f'{name}|{svc_id}|{stdout_log}|{stderr_log}|{max_stale}|{proc_grep}|{schedule_desc}|{mon_type}|{recovery_policy}')
|
|
111
144
|
" 2>/dev/null
|
|
112
145
|
}
|
|
113
146
|
|
|
@@ -134,7 +167,7 @@ if [ "${NEXO_MAINTAINER:-}" = "1" ]; then
|
|
|
134
167
|
fi
|
|
135
168
|
|
|
136
169
|
# Error patterns to search in stderr logs (last 50 lines)
|
|
137
|
-
ERROR_PATTERNS="Traceback|Error:|CRITICAL|FATAL|ModuleNotFoundError|PermissionError|FileNotFoundError|ConnectionRefused|Errno"
|
|
170
|
+
ERROR_PATTERNS="Traceback|Error:|CRITICAL|FATAL|ModuleNotFoundError|PermissionError|FileNotFoundError|ConnectionRefused|Errno|Operation not permitted|SyntaxError|sqlite3\\.OperationalError"
|
|
138
171
|
|
|
139
172
|
# ============================================================================
|
|
140
173
|
# HELPER FUNCTIONS
|
|
@@ -150,7 +183,7 @@ log_repair() { echo "[$TS] REPAIR: $1" >> "$REPAIR_LOG"; log "REPAIR: $1"; }
|
|
|
150
183
|
|
|
151
184
|
is_loaded() {
|
|
152
185
|
if $IS_MACOS; then
|
|
153
|
-
launchctl
|
|
186
|
+
launchctl print "gui/$UID_NUM/$1" &>/dev/null
|
|
154
187
|
else
|
|
155
188
|
# On Linux, check if the systemd timer is enabled
|
|
156
189
|
systemctl --user is-enabled "$1" &>/dev/null
|
|
@@ -242,45 +275,13 @@ try_reexecute_missed_cron() {
|
|
|
242
275
|
local svc_id="$1"
|
|
243
276
|
|
|
244
277
|
if $IS_MACOS; then
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
if [ ! -f "$plist_file" ]; then
|
|
249
|
-
log "Re-execute skipped: no plist for $svc_id"
|
|
250
|
-
return 1
|
|
251
|
-
fi
|
|
252
|
-
|
|
253
|
-
local cmd
|
|
254
|
-
cmd=$(python3 -c "
|
|
255
|
-
import plistlib, sys
|
|
256
|
-
try:
|
|
257
|
-
with open('$plist_file', 'rb') as f:
|
|
258
|
-
d = plistlib.load(f)
|
|
259
|
-
args = d.get('ProgramArguments', [])
|
|
260
|
-
if d.get('KeepAlive'):
|
|
261
|
-
sys.exit(1)
|
|
262
|
-
if not d.get('StartCalendarInterval') and not d.get('StartInterval'):
|
|
263
|
-
sys.exit(1)
|
|
264
|
-
print(' '.join(args))
|
|
265
|
-
except:
|
|
266
|
-
sys.exit(1)
|
|
267
|
-
" 2>/dev/null)
|
|
268
|
-
|
|
269
|
-
if [ -z "$cmd" ] || [ $? -ne 0 ]; then
|
|
270
|
-
return 1
|
|
271
|
-
fi
|
|
272
|
-
|
|
273
|
-
log "Re-executing missed cron: $svc_id → $cmd"
|
|
274
|
-
timeout 300 bash -c "$cmd" >> "$LOG_DIR/watchdog-reexec.log" 2>&1 &
|
|
275
|
-
local pid=$!
|
|
276
|
-
sleep 2
|
|
277
|
-
if kill -0 "$pid" 2>/dev/null || wait "$pid" 2>/dev/null; then
|
|
278
|
-
log_repair "$svc_id: re-executed missed cron (PID $pid)"
|
|
278
|
+
log "Re-executing missed cron via launchctl kickstart: $svc_id"
|
|
279
|
+
if launchctl kickstart -k "gui/$UID_NUM/$svc_id" >> "$LOG_DIR/watchdog-reexec.log" 2>&1; then
|
|
280
|
+
log_repair "$svc_id: re-executed missed cron via launchctl kickstart"
|
|
279
281
|
return 0
|
|
280
|
-
else
|
|
281
|
-
log "Re-execute failed for $svc_id"
|
|
282
|
-
return 1
|
|
283
282
|
fi
|
|
283
|
+
log "Re-execute failed for $svc_id"
|
|
284
|
+
return 1
|
|
284
285
|
else
|
|
285
286
|
# Linux: start the corresponding service unit directly
|
|
286
287
|
local service_unit="${svc_id%.timer}.service"
|
|
@@ -295,11 +296,30 @@ except:
|
|
|
295
296
|
fi
|
|
296
297
|
}
|
|
297
298
|
|
|
299
|
+
CATCHUP_REQUESTED=false
|
|
300
|
+
try_request_catchup() {
|
|
301
|
+
if $CATCHUP_REQUESTED; then
|
|
302
|
+
return 0
|
|
303
|
+
fi
|
|
304
|
+
local catchup_svc
|
|
305
|
+
if $IS_MACOS; then
|
|
306
|
+
catchup_svc="com.nexo.catchup"
|
|
307
|
+
else
|
|
308
|
+
catchup_svc="nexo-catchup.timer"
|
|
309
|
+
fi
|
|
310
|
+
if try_reexecute_missed_cron "$catchup_svc"; then
|
|
311
|
+
CATCHUP_REQUESTED=true
|
|
312
|
+
return 0
|
|
313
|
+
fi
|
|
314
|
+
return 1
|
|
315
|
+
}
|
|
316
|
+
|
|
298
317
|
try_verify_repair() {
|
|
299
318
|
# After Level 2 repair, wait and verify the service is healthy
|
|
300
319
|
local plist_id="$1"
|
|
301
320
|
local log_stdout="$2"
|
|
302
321
|
local proc_grep="$3"
|
|
322
|
+
local mon_type="${4:-core}"
|
|
303
323
|
local max_wait=30
|
|
304
324
|
|
|
305
325
|
log "Verifying repair for $plist_id..."
|
|
@@ -325,7 +345,22 @@ try_verify_repair() {
|
|
|
325
345
|
return 1
|
|
326
346
|
fi
|
|
327
347
|
|
|
328
|
-
# Check 3: For scheduled crons, check if
|
|
348
|
+
# Check 3: For scheduled crons, check if cron_runs/logs were updated recently
|
|
349
|
+
if [ "$mon_type" = "core" ]; then
|
|
350
|
+
local cron_id
|
|
351
|
+
cron_id=$(cron_id_from_service "$plist_id")
|
|
352
|
+
local run_info
|
|
353
|
+
run_info=$(cron_last_run_info "$cron_id" || true)
|
|
354
|
+
if [ -n "$run_info" ]; then
|
|
355
|
+
local run_age
|
|
356
|
+
IFS='|' read -r run_age _ _ _ _ _ <<< "$run_info"
|
|
357
|
+
if [ -n "$run_age" ] && [ "$run_age" -lt 300 ]; then
|
|
358
|
+
log "Verify OK: $plist_id cron_runs updated ${run_age}s ago"
|
|
359
|
+
return 0
|
|
360
|
+
fi
|
|
361
|
+
fi
|
|
362
|
+
fi
|
|
363
|
+
|
|
329
364
|
if [ -n "$log_stdout" ] && [ -f "$log_stdout" ]; then
|
|
330
365
|
local age
|
|
331
366
|
age=$(file_age "$log_stdout")
|
|
@@ -408,6 +443,51 @@ process_running() {
|
|
|
408
443
|
fi
|
|
409
444
|
}
|
|
410
445
|
|
|
446
|
+
cron_id_from_service() {
|
|
447
|
+
local svc_id="$1"
|
|
448
|
+
if $IS_MACOS; then
|
|
449
|
+
echo "${svc_id#com.nexo.}"
|
|
450
|
+
else
|
|
451
|
+
echo "${svc_id#nexo-}" | sed 's/\.timer$//'
|
|
452
|
+
fi
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
cron_last_run_info() {
|
|
456
|
+
local cron_id="$1"
|
|
457
|
+
[ ! -f "$DB_PATH" ] && return 1
|
|
458
|
+
sqlite3 -separator '|' "$DB_PATH" "
|
|
459
|
+
SELECT
|
|
460
|
+
CAST(strftime('%s','now') - strftime('%s', started_at) AS INTEGER) AS age_secs,
|
|
461
|
+
COALESCE(started_at, ''),
|
|
462
|
+
COALESCE(ended_at, ''),
|
|
463
|
+
COALESCE(exit_code, ''),
|
|
464
|
+
COALESCE(error, ''),
|
|
465
|
+
COALESCE(summary, '')
|
|
466
|
+
FROM cron_runs
|
|
467
|
+
WHERE cron_id = '$cron_id'
|
|
468
|
+
ORDER BY id DESC
|
|
469
|
+
LIMIT 1;
|
|
470
|
+
" 2>/dev/null
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
classify_log_issue() {
|
|
474
|
+
local logfile="$1"
|
|
475
|
+
if [ ! -f "$logfile" ] || [ ! -s "$logfile" ]; then
|
|
476
|
+
return 0
|
|
477
|
+
fi
|
|
478
|
+
local tail_text
|
|
479
|
+
tail_text=$(tail -50 "$logfile" 2>/dev/null || true)
|
|
480
|
+
if echo "$tail_text" | grep -q "Operation not permitted"; then
|
|
481
|
+
echo "tcc"
|
|
482
|
+
elif echo "$tail_text" | grep -q "ModuleNotFoundError"; then
|
|
483
|
+
echo "dependency"
|
|
484
|
+
elif echo "$tail_text" | grep -q "SyntaxError"; then
|
|
485
|
+
echo "syntax"
|
|
486
|
+
elif echo "$tail_text" | grep -q "sqlite3.OperationalError"; then
|
|
487
|
+
echo "schema"
|
|
488
|
+
fi
|
|
489
|
+
}
|
|
490
|
+
|
|
411
491
|
# Escape strings for JSON
|
|
412
492
|
json_escape() {
|
|
413
493
|
echo "$1" | sed 's/\\/\\\\/g; s/"/\\"/g; s/ / /g' | tr '\n' ' '
|
|
@@ -427,8 +507,9 @@ FAILED_MONITORS=() # Track failed monitors for Level 2 repair
|
|
|
427
507
|
for monitor in "${MONITORS[@]}"; do
|
|
428
508
|
# Skip comment lines
|
|
429
509
|
[[ "$monitor" =~ ^[[:space:]]*# ]] && continue
|
|
430
|
-
IFS='|' read -r name plist_id log_stdout log_stderr max_stale proc_grep schedule mon_type <<< "$monitor"
|
|
510
|
+
IFS='|' read -r name plist_id log_stdout log_stderr max_stale proc_grep schedule mon_type recovery_policy <<< "$monitor"
|
|
431
511
|
mon_type="${mon_type:-core}"
|
|
512
|
+
recovery_policy="${recovery_policy:-restart}"
|
|
432
513
|
|
|
433
514
|
status="PASS"
|
|
434
515
|
details=""
|
|
@@ -436,6 +517,10 @@ for monitor in "${MONITORS[@]}"; do
|
|
|
436
517
|
stale_age="n/a"
|
|
437
518
|
error_count=0
|
|
438
519
|
proc_alive="n/a"
|
|
520
|
+
error_kind=""
|
|
521
|
+
cron_id=$(cron_id_from_service "$plist_id")
|
|
522
|
+
latest_run_has_record=false
|
|
523
|
+
latest_run_failed=false
|
|
439
524
|
|
|
440
525
|
# Check 1: Service loaded? (launchd on macOS, systemd on Linux)
|
|
441
526
|
if is_loaded "$plist_id"; then
|
|
@@ -490,8 +575,69 @@ for monitor in "${MONITORS[@]}"; do
|
|
|
490
575
|
fi
|
|
491
576
|
fi
|
|
492
577
|
|
|
493
|
-
# Check 3:
|
|
494
|
-
if [
|
|
578
|
+
# Check 3: Staleness + AUTO RE-EXECUTE missed crons
|
|
579
|
+
if [ "$mon_type" = "core" ] && [ "$max_stale" -gt 0 ]; then
|
|
580
|
+
run_info=$(cron_last_run_info "$cron_id" || true)
|
|
581
|
+
if [ -n "$run_info" ]; then
|
|
582
|
+
latest_run_has_record=true
|
|
583
|
+
IFS='|' read -r age _ _ last_exit last_error last_summary <<< "$run_info"
|
|
584
|
+
age="${age:-999999}"
|
|
585
|
+
stale_age=$(format_age "$age")
|
|
586
|
+
if [ -n "$last_exit" ] && [ "$last_exit" != "0" ]; then
|
|
587
|
+
latest_run_failed=true
|
|
588
|
+
status="FAIL"
|
|
589
|
+
details="${details}Last run exited ${last_exit}. "
|
|
590
|
+
[ -n "$last_error" ] && details="${details}Error: ${last_error}. "
|
|
591
|
+
fi
|
|
592
|
+
if [ "$age" -gt $(( max_stale * 3 )) ]; then
|
|
593
|
+
if [ "$recovery_policy" = "catchup" ]; then
|
|
594
|
+
if try_request_catchup; then
|
|
595
|
+
status="HEALED"
|
|
596
|
+
details="${details}Self-healed: requested catchup for missed window (last run: $stale_age). "
|
|
597
|
+
TOTAL_HEALED=$((TOTAL_HEALED + 1))
|
|
598
|
+
else
|
|
599
|
+
status="FAIL"
|
|
600
|
+
details="${details}cron_runs stale: $stale_age (limit: $(format_age "$max_stale")). Catchup request failed. "
|
|
601
|
+
fi
|
|
602
|
+
else
|
|
603
|
+
if try_reexecute_missed_cron "$plist_id"; then
|
|
604
|
+
status="HEALED"
|
|
605
|
+
details="${details}Self-healed: re-executed missed cron (last run: $stale_age). "
|
|
606
|
+
TOTAL_HEALED=$((TOTAL_HEALED + 1))
|
|
607
|
+
else
|
|
608
|
+
status="FAIL"
|
|
609
|
+
details="${details}cron_runs stale: $stale_age (limit: $(format_age "$max_stale")). Re-execute failed. "
|
|
610
|
+
fi
|
|
611
|
+
fi
|
|
612
|
+
elif [ "$age" -gt "$max_stale" ]; then
|
|
613
|
+
[ "$status" = "PASS" ] && status="WARN"
|
|
614
|
+
details="${details}cron_runs slightly stale: $stale_age. "
|
|
615
|
+
elif [ -z "$details" ] && [ -n "$last_summary" ]; then
|
|
616
|
+
details="${details}Last run summary: ${last_summary}. "
|
|
617
|
+
fi
|
|
618
|
+
else
|
|
619
|
+
stale_age="no cron_runs entry"
|
|
620
|
+
if [ "$recovery_policy" = "catchup" ]; then
|
|
621
|
+
if try_request_catchup; then
|
|
622
|
+
status="HEALED"
|
|
623
|
+
details="${details}Self-healed: requested catchup for missing cron_runs entry. "
|
|
624
|
+
TOTAL_HEALED=$((TOTAL_HEALED + 1))
|
|
625
|
+
else
|
|
626
|
+
status="FAIL"
|
|
627
|
+
details="${details}No cron_runs entry recorded yet and catchup request failed. "
|
|
628
|
+
fi
|
|
629
|
+
else
|
|
630
|
+
if try_reexecute_missed_cron "$plist_id"; then
|
|
631
|
+
status="HEALED"
|
|
632
|
+
details="${details}Self-healed: executed missing cron for first run. "
|
|
633
|
+
TOTAL_HEALED=$((TOTAL_HEALED + 1))
|
|
634
|
+
else
|
|
635
|
+
status="FAIL"
|
|
636
|
+
details="${details}No cron_runs entry recorded yet. "
|
|
637
|
+
fi
|
|
638
|
+
fi
|
|
639
|
+
fi
|
|
640
|
+
elif [ -n "$log_stdout" ] && [ "$max_stale" -gt 0 ]; then
|
|
495
641
|
age=$(file_age "$log_stdout")
|
|
496
642
|
stale_age=$(format_age "$age")
|
|
497
643
|
if [ "$age" -gt $(( max_stale * 3 )) ]; then
|
|
@@ -519,10 +665,35 @@ for monitor in "${MONITORS[@]}"; do
|
|
|
519
665
|
|
|
520
666
|
# Check 4: Errors in stderr log
|
|
521
667
|
if [ -n "$log_stderr" ]; then
|
|
522
|
-
|
|
523
|
-
if [ "$
|
|
524
|
-
|
|
525
|
-
|
|
668
|
+
consider_stderr=true
|
|
669
|
+
if [ "$mon_type" = "core" ] && $latest_run_has_record && ! $latest_run_failed && [ "$loaded" = "yes" ]; then
|
|
670
|
+
consider_stderr=false
|
|
671
|
+
fi
|
|
672
|
+
if $consider_stderr; then
|
|
673
|
+
error_count=$(check_errors "$log_stderr")
|
|
674
|
+
error_kind=$(classify_log_issue "$log_stderr" || true)
|
|
675
|
+
if [ "$error_count" -gt 5 ]; then
|
|
676
|
+
[ "$status" = "PASS" ] && status="WARN"
|
|
677
|
+
details="${details}${error_count} errors in recent stderr. "
|
|
678
|
+
fi
|
|
679
|
+
case "$error_kind" in
|
|
680
|
+
tcc)
|
|
681
|
+
status="FAIL"
|
|
682
|
+
details="${details}Recent stderr shows macOS TCC/Sandbox denial ('Operation not permitted'). "
|
|
683
|
+
;;
|
|
684
|
+
dependency)
|
|
685
|
+
[ "$status" = "PASS" ] && status="WARN"
|
|
686
|
+
details="${details}Recent stderr shows missing Python dependency. "
|
|
687
|
+
;;
|
|
688
|
+
syntax)
|
|
689
|
+
status="FAIL"
|
|
690
|
+
details="${details}Recent stderr shows syntax error. "
|
|
691
|
+
;;
|
|
692
|
+
schema)
|
|
693
|
+
status="FAIL"
|
|
694
|
+
details="${details}Recent stderr shows DB/schema mismatch. "
|
|
695
|
+
;;
|
|
696
|
+
esac
|
|
526
697
|
fi
|
|
527
698
|
fi
|
|
528
699
|
|
|
@@ -557,7 +728,7 @@ done
|
|
|
557
728
|
# --- Cron job checks ---
|
|
558
729
|
CRON_JSON=""
|
|
559
730
|
CRON_REPORT=""
|
|
560
|
-
for cron_entry in "${CRON_MONITORS[@]}"; do
|
|
731
|
+
for cron_entry in ${CRON_MONITORS[@]+"${CRON_MONITORS[@]}"}; do
|
|
561
732
|
IFS='|' read -r name script check_path max_stale schedule <<< "$cron_entry"
|
|
562
733
|
|
|
563
734
|
c_status="PASS"
|
|
@@ -656,6 +827,7 @@ fi
|
|
|
656
827
|
# --- Immutable file integrity ---
|
|
657
828
|
IMMUTABLE_STATUS="PASS"
|
|
658
829
|
IMMUTABLE_DETAIL=""
|
|
830
|
+
OBJECTIVE="$CORTEX_DIR/evolution-objective.json"
|
|
659
831
|
if [ -f "$HASH_REGISTRY" ]; then
|
|
660
832
|
TAMPERED=0
|
|
661
833
|
while IFS='|' read -r filepath expected_hash; do
|
|
@@ -676,7 +848,6 @@ if [ -f "$HASH_REGISTRY" ]; then
|
|
|
676
848
|
IMMUTABLE_STATUS="FAIL"
|
|
677
849
|
IMMUTABLE_DETAIL="$TAMPERED immutable files tampered"
|
|
678
850
|
TOTAL_FAIL=$((TOTAL_FAIL + 1))
|
|
679
|
-
OBJECTIVE="$CORTEX_DIR/evolution-objective.json"
|
|
680
851
|
if [ -f "$OBJECTIVE" ]; then
|
|
681
852
|
python3 -c "
|
|
682
853
|
import json
|
|
@@ -690,6 +861,23 @@ with open('$OBJECTIVE', 'w') as f: json.dump(d, f, indent=2)
|
|
|
690
861
|
else
|
|
691
862
|
IMMUTABLE_DETAIL="All files intact"
|
|
692
863
|
TOTAL_PASS=$((TOTAL_PASS + 1))
|
|
864
|
+
if [ -f "$OBJECTIVE" ]; then
|
|
865
|
+
python3 -c "
|
|
866
|
+
import json
|
|
867
|
+
from pathlib import Path
|
|
868
|
+
obj = Path('$OBJECTIVE')
|
|
869
|
+
try:
|
|
870
|
+
data = json.loads(obj.read_text())
|
|
871
|
+
except Exception:
|
|
872
|
+
raise SystemExit(0)
|
|
873
|
+
reason = data.get('disabled_reason', '') or ''
|
|
874
|
+
if data.get('evolution_enabled') is False and 'watchdog disabled Evolution' in reason:
|
|
875
|
+
data['evolution_enabled'] = True
|
|
876
|
+
data.pop('disabled_reason', None)
|
|
877
|
+
obj.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
|
878
|
+
print('REENABLED')
|
|
879
|
+
" 2>/dev/null | grep -q "REENABLED" && log "REENABLED Evolution after immutable integrity recovered"
|
|
880
|
+
fi
|
|
693
881
|
fi
|
|
694
882
|
else
|
|
695
883
|
IMMUTABLE_DETAIL="No hash registry (skipped)"
|
|
@@ -973,7 +1161,7 @@ NEXOPROMPT
|
|
|
973
1161
|
VERIFY_FAIL=0
|
|
974
1162
|
for failed in ${FAILED_MONITORS[@]+"${FAILED_MONITORS[@]}"}; do
|
|
975
1163
|
IFS='|' read -r v_name v_plist v_stdout v_stderr v_proc v_sched v_type v_details <<< "$failed"
|
|
976
|
-
if try_verify_repair "$v_plist" "$v_stdout" "$v_proc"; then
|
|
1164
|
+
if try_verify_repair "$v_plist" "$v_stdout" "$v_proc" "$v_type"; then
|
|
977
1165
|
VERIFY_PASS=$((VERIFY_PASS + 1))
|
|
978
1166
|
log "VERIFY OK: $v_name"
|
|
979
1167
|
else
|
package/src/tools_learnings.py
CHANGED
|
@@ -23,6 +23,14 @@ def handle_learning_add(category: str, title: str, content: str, reasoning: str
|
|
|
23
23
|
category = category.lower().strip()
|
|
24
24
|
if not category:
|
|
25
25
|
return "ERROR: Category cannot be empty."
|
|
26
|
+
# Dedup guard: block exact title duplicates in same category
|
|
27
|
+
conn = get_db()
|
|
28
|
+
existing = conn.execute(
|
|
29
|
+
"SELECT id, title FROM learnings WHERE LOWER(title) = LOWER(?) AND category = ? AND status = 'active'",
|
|
30
|
+
(title.strip(), category)
|
|
31
|
+
).fetchone()
|
|
32
|
+
if existing:
|
|
33
|
+
return f"Learning #{existing['id']} already exists with same title in {category}: {existing['title']}. Use nexo_learning_update to modify it."
|
|
26
34
|
result = create_learning(
|
|
27
35
|
category, title, content, reasoning=reasoning
|
|
28
36
|
)
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
2
|
<!-- com.nexo.catchup
|
|
3
|
-
Runs nexo-catchup.py
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
after a reboot or prolonged sleep period.
|
|
3
|
+
Runs nexo-catchup.py at login and every 15 minutes so missed core
|
|
4
|
+
windows are recovered after boot, wake, or prolonged sleep.
|
|
5
|
+
Recovery is gated by cron_runs plus the manifest recovery contract,
|
|
6
|
+
so catchup only replays jobs that are safe and still inside their
|
|
7
|
+
allowed catch-up window.
|
|
9
8
|
-->
|
|
10
9
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
11
10
|
<plist version="1.0">
|
|
@@ -19,6 +18,8 @@
|
|
|
19
18
|
</array>
|
|
20
19
|
<key>RunAtLoad</key>
|
|
21
20
|
<true/>
|
|
21
|
+
<key>StartInterval</key>
|
|
22
|
+
<integer>900</integer>
|
|
22
23
|
<key>StandardOutPath</key>
|
|
23
24
|
<string>{{NEXO_HOME}}/logs/catchup-stdout.log</string>
|
|
24
25
|
<key>StandardErrorPath</key>
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
# nexo: name=example-script
|
|
3
3
|
# nexo: description=Example personal script using the stable NEXO CLI
|
|
4
|
+
# nexo: category=automation
|
|
4
5
|
# nexo: runtime=python
|
|
5
6
|
# nexo: timeout=60
|
|
6
7
|
# nexo: tools=nexo_learning_search,nexo_schedule_status
|
|
8
|
+
# nexo: interval_seconds=300
|
|
9
|
+
# nexo: schedule_required=false
|
|
7
10
|
|
|
8
11
|
"""Example personal script for NEXO.
|
|
9
12
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# nexo: name=example-script
|
|
3
|
+
# nexo: description=Example personal shell script using the stable NEXO CLI
|
|
4
|
+
# nexo: category=automation
|
|
5
|
+
# nexo: runtime=shell
|
|
6
|
+
# nexo: timeout=60
|
|
7
|
+
# nexo: schedule=08:00
|
|
8
|
+
# nexo: schedule_required=false
|
|
9
|
+
|
|
10
|
+
set -euo pipefail
|
|
11
|
+
|
|
12
|
+
echo "Hello from NEXO personal shell script"
|
|
13
|
+
echo "NEXO_HOME=${NEXO_HOME:-}"
|
|
@@ -1,139 +0,0 @@
|
|
|
1
|
-
#!/bin/bash
|
|
2
|
-
# ============================================================================
|
|
3
|
-
# NEXO Day Orchestrator — autonomous NEXO cycle every 15 min
|
|
4
|
-
# Schedule: keepAlive, self-enforced operating hours (default 8:00-23:00)
|
|
5
|
-
#
|
|
6
|
-
# This is NOT a Python script that simulates intelligence.
|
|
7
|
-
# This launches Claude Code as NEXO with full MCP access.
|
|
8
|
-
# NEXO thinks, acts, and reports — like any interactive session.
|
|
9
|
-
# ============================================================================
|
|
10
|
-
set -euo pipefail
|
|
11
|
-
|
|
12
|
-
NEXO_HOME="${NEXO_HOME:-$HOME/.nexo}"
|
|
13
|
-
LOG_DIR="$NEXO_HOME/logs"
|
|
14
|
-
mkdir -p "$LOG_DIR" "$NEXO_HOME/operations"
|
|
15
|
-
|
|
16
|
-
# --- Configuration ---
|
|
17
|
-
CYCLE_INTERVAL=900 # 15 minutes between cycles
|
|
18
|
-
CYCLE_TIMEOUT=600 # 10 min max per cycle
|
|
19
|
-
MAX_TURNS=30 # Claude max turns per cycle
|
|
20
|
-
HOUR_START=8
|
|
21
|
-
HOUR_END=23
|
|
22
|
-
|
|
23
|
-
# --- Find Claude CLI ---
|
|
24
|
-
find_claude() {
|
|
25
|
-
for candidate in \
|
|
26
|
-
"$(command -v claude 2>/dev/null)" \
|
|
27
|
-
"$HOME/.claude/local/claude" \
|
|
28
|
-
"/opt/homebrew/bin/claude" \
|
|
29
|
-
"/usr/local/bin/claude"; do
|
|
30
|
-
if [ -n "$candidate" ] && [ -x "$candidate" ]; then
|
|
31
|
-
echo "$candidate"
|
|
32
|
-
return 0
|
|
33
|
-
fi
|
|
34
|
-
done
|
|
35
|
-
return 1
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
CLAUDE=$(find_claude) || {
|
|
39
|
-
echo "$(date '+%Y-%m-%d %H:%M') ERROR: claude CLI not found" >&2
|
|
40
|
-
exit 1
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
# --- Prevent overlapping cycles ---
|
|
44
|
-
LOCKFILE="$NEXO_HOME/operations/.orchestrator.lock"
|
|
45
|
-
acquire_lock() {
|
|
46
|
-
if [ -f "$LOCKFILE" ]; then
|
|
47
|
-
local pid
|
|
48
|
-
pid=$(cat "$LOCKFILE" 2>/dev/null || echo "")
|
|
49
|
-
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
|
|
50
|
-
return 1 # Still running
|
|
51
|
-
fi
|
|
52
|
-
fi
|
|
53
|
-
echo $$ > "$LOCKFILE"
|
|
54
|
-
return 0
|
|
55
|
-
}
|
|
56
|
-
release_lock() { rm -f "$LOCKFILE"; }
|
|
57
|
-
|
|
58
|
-
# --- The orchestrator prompt ---
|
|
59
|
-
PROMPT='You are NEXO in autonomous orchestrator mode. The user is NOT present. You have 5 minutes max.
|
|
60
|
-
|
|
61
|
-
ABSOLUTE PRIORITY: act, do not list. If you can do something, do it. If you need the user, send email.
|
|
62
|
-
|
|
63
|
-
CHECKLIST (in this order):
|
|
64
|
-
|
|
65
|
-
1. OVERDUE FOLLOWUPS: nexo_reminders(filter="due") + nexo_reminders(filter="followups")
|
|
66
|
-
- NEXO tasks (verify, check, monitor) → DO THEM NOW
|
|
67
|
-
- Tasks needing user decision → accumulate for email
|
|
68
|
-
- Completed ones → nexo_followup_complete
|
|
69
|
-
|
|
70
|
-
2. EMAIL: nexo_email_inbox(unread_only=true, limit=10)
|
|
71
|
-
- Emails you can process → process them
|
|
72
|
-
- Important emails for user → accumulate for email
|
|
73
|
-
|
|
74
|
-
3. INFRASTRUCTURE: nexo_doctor(tier="runtime")
|
|
75
|
-
- If degraded/critical → try to fix
|
|
76
|
-
|
|
77
|
-
4. EMAIL TO USER (only if there is something to report):
|
|
78
|
-
- nexo_email_send with clean HTML summary
|
|
79
|
-
- Only what needs attention or decision
|
|
80
|
-
- Include what you ALREADY DID (not just pending items)
|
|
81
|
-
- If nothing relevant → DO NOT send email
|
|
82
|
-
- Max 1 email per cycle
|
|
83
|
-
|
|
84
|
-
5. DIARY: nexo_session_diary_write with what you did
|
|
85
|
-
|
|
86
|
-
RULES:
|
|
87
|
-
- DO NOT ask permission. autonomy=full
|
|
88
|
-
- DO NOT send empty or "all ok" emails
|
|
89
|
-
- DO NOT list things without acting
|
|
90
|
-
- If a followup is executable → execute it before reporting
|
|
91
|
-
- Use nexo_heartbeat at start
|
|
92
|
-
- Clean close: diary + nexo_stop'
|
|
93
|
-
|
|
94
|
-
# --- Main loop ---
|
|
95
|
-
echo "$(date '+%Y-%m-%d %H:%M') NEXO Day Orchestrator starting (PID $$)"
|
|
96
|
-
echo " Claude: $CLAUDE"
|
|
97
|
-
echo " Cycle: every ${CYCLE_INTERVAL}s, ${HOUR_START}:00-${HOUR_END}:00"
|
|
98
|
-
echo " Timeout: ${CYCLE_TIMEOUT}s, max turns: $MAX_TURNS"
|
|
99
|
-
|
|
100
|
-
while true; do
|
|
101
|
-
HOUR=$(date +%H | sed 's/^0//')
|
|
102
|
-
|
|
103
|
-
# Outside operating hours — sleep and check again
|
|
104
|
-
if [ "$HOUR" -lt "$HOUR_START" ] || [ "$HOUR" -ge "$HOUR_END" ]; then
|
|
105
|
-
sleep 300 # Check every 5 min if we're back in hours
|
|
106
|
-
continue
|
|
107
|
-
fi
|
|
108
|
-
|
|
109
|
-
# Try to acquire lock
|
|
110
|
-
if ! acquire_lock; then
|
|
111
|
-
echo "$(date '+%Y-%m-%d %H:%M') Previous cycle still running. Skipping."
|
|
112
|
-
sleep "$CYCLE_INTERVAL"
|
|
113
|
-
continue
|
|
114
|
-
fi
|
|
115
|
-
|
|
116
|
-
TIMESTAMP=$(date '+%Y-%m-%d_%H%M')
|
|
117
|
-
LOGFILE="$LOG_DIR/orchestrator-$TIMESTAMP.log"
|
|
118
|
-
echo "$(date '+%Y-%m-%d %H:%M') Cycle starting..."
|
|
119
|
-
|
|
120
|
-
# Launch Claude Code as NEXO
|
|
121
|
-
set +e
|
|
122
|
-
timeout "$CYCLE_TIMEOUT" "$CLAUDE" \
|
|
123
|
-
--dangerously-skip-permissions \
|
|
124
|
-
-p "$PROMPT" \
|
|
125
|
-
--max-turns "$MAX_TURNS" \
|
|
126
|
-
>>"$LOGFILE" 2>&1
|
|
127
|
-
EXIT_CODE=$?
|
|
128
|
-
set -e
|
|
129
|
-
|
|
130
|
-
echo "$(date '+%Y-%m-%d %H:%M') Cycle finished (exit $EXIT_CODE)" | tee -a "$LOGFILE"
|
|
131
|
-
|
|
132
|
-
release_lock
|
|
133
|
-
|
|
134
|
-
# Clean old logs (keep 7 days)
|
|
135
|
-
find "$LOG_DIR" -name "orchestrator-*.log" -mtime +7 -delete 2>/dev/null || true
|
|
136
|
-
|
|
137
|
-
# Sleep until next cycle
|
|
138
|
-
sleep "$CYCLE_INTERVAL"
|
|
139
|
-
done
|