nexo-brain 7.20.21 → 7.20.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/auto_update.py +44 -14
- package/src/cli.py +10 -4
- package/src/db_guard.py +7 -5
- package/src/interactive_db.py +59 -0
- package/src/local_context/api.py +3 -6
- package/src/local_context/db.py +318 -0
- package/src/local_context/logging.py +3 -4
- package/src/mcp_required_tools.py +31 -0
- package/src/plugins/episodic_memory.py +18 -0
- package/src/plugins/skills.py +14 -3
- package/src/plugins/update.py +29 -10
- package/src/scripts/nexo-backup.sh +131 -7
- package/src/server.py +95 -5
- package/src/tools_reminders.py +37 -8
- package/src/tools_sessions.py +0 -11
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.20.
|
|
3
|
+
"version": "7.20.22",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/auto_update.py
CHANGED
|
@@ -89,6 +89,7 @@ LOCAL_CONTEXT_BACKUP_TABLES = (
|
|
|
89
89
|
"local_index_checkpoints",
|
|
90
90
|
"local_index_state",
|
|
91
91
|
"local_index_errors",
|
|
92
|
+
"local_index_logs",
|
|
92
93
|
"local_assets",
|
|
93
94
|
"local_asset_versions",
|
|
94
95
|
"local_chunks",
|
|
@@ -98,7 +99,17 @@ LOCAL_CONTEXT_BACKUP_TABLES = (
|
|
|
98
99
|
"local_context_queries",
|
|
99
100
|
"local_index_dirs",
|
|
100
101
|
)
|
|
101
|
-
PROTECTED_BACKUP_TABLES = CRITICAL_BACKUP_TABLES
|
|
102
|
+
PROTECTED_BACKUP_TABLES = CRITICAL_BACKUP_TABLES
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _backup_validation_tables(db_file: Path) -> tuple[str, ...]:
|
|
106
|
+
if db_file.name == "nexo.db":
|
|
107
|
+
return PROTECTED_BACKUP_TABLES
|
|
108
|
+
if db_file.name == "local-context.db":
|
|
109
|
+
return LOCAL_CONTEXT_BACKUP_TABLES
|
|
110
|
+
return ()
|
|
111
|
+
|
|
112
|
+
|
|
102
113
|
CLASSIFIER_INSTALL_TIMEOUT_SECONDS = 1800
|
|
103
114
|
CLASSIFIER_INSTALL_JOIN_SECONDS = 1500
|
|
104
115
|
CLASSIFIER_INSTALL_LOG = paths.logs_dir() / "classifier-install.log"
|
|
@@ -338,7 +349,7 @@ def _validate_db_backup(source_db: Path, backup_db: Path) -> dict:
|
|
|
338
349
|
report["errors"].append(f"backup db missing: {backup_db}")
|
|
339
350
|
return report
|
|
340
351
|
|
|
341
|
-
for table in
|
|
352
|
+
for table in _backup_validation_tables(source_db):
|
|
342
353
|
source_count = _critical_table_count(source_db, table)
|
|
343
354
|
backup_count = _critical_table_count(backup_db, table)
|
|
344
355
|
report["source_counts"][table] = source_count
|
|
@@ -373,18 +384,29 @@ def _create_validated_db_backup() -> tuple[str | None, dict | None]:
|
|
|
373
384
|
if not backup_dir:
|
|
374
385
|
return None, None
|
|
375
386
|
|
|
376
|
-
|
|
377
|
-
|
|
387
|
+
source_dbs: list[Path] = []
|
|
388
|
+
primary_db = _find_primary_db_path()
|
|
389
|
+
if primary_db is not None:
|
|
390
|
+
source_dbs.append(primary_db)
|
|
391
|
+
local_context_db = paths.memory_dir() / "local-context.db"
|
|
392
|
+
if local_context_db.is_file():
|
|
393
|
+
source_dbs.append(local_context_db)
|
|
394
|
+
if not source_dbs:
|
|
378
395
|
return backup_dir, None
|
|
379
396
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
397
|
+
reports = []
|
|
398
|
+
ok = True
|
|
399
|
+
for source_db in source_dbs:
|
|
400
|
+
report = _validate_db_backup(source_db, Path(backup_dir) / source_db.name)
|
|
401
|
+
reports.append(report)
|
|
402
|
+
if not report["ok"]:
|
|
403
|
+
ok = False
|
|
404
|
+
details = ", ".join(
|
|
405
|
+
f"{item['table']} {item['source']}->{item['backup']}"
|
|
406
|
+
for item in report["regressions"]
|
|
407
|
+
) or "; ".join(report["errors"])
|
|
408
|
+
_log(f"DB backup validation failed ({source_db.name}): {details}")
|
|
409
|
+
return backup_dir, {"ok": ok, "reports": reports}
|
|
388
410
|
|
|
389
411
|
|
|
390
412
|
def _read_last_check() -> dict:
|
|
@@ -2047,6 +2069,9 @@ def _backup_dbs() -> str | None:
|
|
|
2047
2069
|
backup_dir = paths.backups_dir() / f"pre-autoupdate-{timestamp}"
|
|
2048
2070
|
|
|
2049
2071
|
db_files = list(DATA_DIR.glob("*.db")) if DATA_DIR.is_dir() else []
|
|
2072
|
+
local_context_db = paths.memory_dir() / "local-context.db"
|
|
2073
|
+
if local_context_db.is_file():
|
|
2074
|
+
db_files.append(local_context_db)
|
|
2050
2075
|
db_files += [f for f in NEXO_HOME.glob("*.db") if f.is_file()]
|
|
2051
2076
|
src_db = SRC_DIR / "nexo.db"
|
|
2052
2077
|
if src_db.is_file() and src_db not in db_files:
|
|
@@ -2090,11 +2115,16 @@ def _restore_dbs(backup_dir: str):
|
|
|
2090
2115
|
if not bdir.is_dir():
|
|
2091
2116
|
return
|
|
2092
2117
|
for db_backup in bdir.glob("*.db"):
|
|
2093
|
-
|
|
2094
|
-
|
|
2118
|
+
if db_backup.name == "local-context.db":
|
|
2119
|
+
candidates = [paths.memory_dir() / db_backup.name]
|
|
2120
|
+
else:
|
|
2121
|
+
candidates = [DATA_DIR / db_backup.name, NEXO_HOME / db_backup.name, SRC_DIR / db_backup.name]
|
|
2122
|
+
for candidate in candidates:
|
|
2123
|
+
if candidate.is_file() or db_backup.name == "local-context.db":
|
|
2095
2124
|
src_conn = None
|
|
2096
2125
|
dst_conn = None
|
|
2097
2126
|
try:
|
|
2127
|
+
candidate.parent.mkdir(parents=True, exist_ok=True)
|
|
2098
2128
|
src_conn = sqlite3.connect(str(db_backup))
|
|
2099
2129
|
dst_conn = sqlite3.connect(str(candidate))
|
|
2100
2130
|
src_conn.backup(dst_conn)
|
package/src/cli.py
CHANGED
|
@@ -52,6 +52,7 @@ from pathlib import Path
|
|
|
52
52
|
|
|
53
53
|
from runtime_home import export_resolved_nexo_home
|
|
54
54
|
from runtime_versioning import build_mcp_status, clear_restart_required_marker
|
|
55
|
+
from mcp_required_tools import BOOTSTRAP_REQUIRED_MCP_TOOLS, missing_required_tools
|
|
55
56
|
|
|
56
57
|
NEXO_HOME = export_resolved_nexo_home()
|
|
57
58
|
NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parent)))
|
|
@@ -200,8 +201,11 @@ def _mcp_probe(args) -> int:
|
|
|
200
201
|
server_path = NEXO_CODE / "server.py"
|
|
201
202
|
env = os.environ.copy()
|
|
202
203
|
env["NEXO_MCP_PROBE"] = "1"
|
|
203
|
-
env
|
|
204
|
-
|
|
204
|
+
# Probe must be deterministic: inherited env from an older Desktop/runtime
|
|
205
|
+
# must not force plugin loading or startup preflight and turn this into a
|
|
206
|
+
# slow/false-negative health check.
|
|
207
|
+
env["NEXO_MCP_PLUGIN_MODE"] = getattr(args, "plugin_mode", None) or "none"
|
|
208
|
+
env["NEXO_MCP_RUN_STARTUP_PREFLIGHT"] = "0"
|
|
205
209
|
client = str(getattr(args, "client", "") or "").strip()
|
|
206
210
|
if client:
|
|
207
211
|
env["NEXO_MCP_CLIENT"] = client
|
|
@@ -256,8 +260,8 @@ def _mcp_probe(args) -> int:
|
|
|
256
260
|
for tool in tools
|
|
257
261
|
if isinstance(tool, dict) and tool.get("name")
|
|
258
262
|
]
|
|
259
|
-
required =
|
|
260
|
-
missing =
|
|
263
|
+
required = list(BOOTSTRAP_REQUIRED_MCP_TOOLS)
|
|
264
|
+
missing = missing_required_tools(tool_names)
|
|
261
265
|
ok = not missing and len(tool_names) > 0
|
|
262
266
|
payload = {
|
|
263
267
|
"ok": ok,
|
|
@@ -265,6 +269,8 @@ def _mcp_probe(args) -> int:
|
|
|
265
269
|
"probe_ok": ok,
|
|
266
270
|
"tools_available": len(tool_names) > 0,
|
|
267
271
|
"tool_count": len(tool_names),
|
|
272
|
+
"required_tools": required,
|
|
273
|
+
"required_tool_count": len(required),
|
|
268
274
|
"required_tools_present": not missing,
|
|
269
275
|
"missing_required_tools": missing,
|
|
270
276
|
"client": client,
|
package/src/db_guard.py
CHANGED
|
@@ -75,6 +75,7 @@ LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
|
75
75
|
"local_index_checkpoints",
|
|
76
76
|
"local_index_state",
|
|
77
77
|
"local_index_errors",
|
|
78
|
+
"local_index_logs",
|
|
78
79
|
"local_assets",
|
|
79
80
|
"local_asset_versions",
|
|
80
81
|
"local_chunks",
|
|
@@ -85,10 +86,11 @@ LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
|
85
86
|
"local_index_dirs",
|
|
86
87
|
)
|
|
87
88
|
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
|
|
89
|
+
# Tables protected inside the operational Brain DB. Local memory now lives in
|
|
90
|
+
# runtime/memory/local-context.db and is validated with LOCAL_CONTEXT_TABLES
|
|
91
|
+
# separately; mixing both here makes old local-index backups look like the
|
|
92
|
+
# source of truth for nexo.db and can block every update.
|
|
93
|
+
PROTECTED_TABLES: tuple[str, ...] = CRITICAL_TABLES
|
|
92
94
|
|
|
93
95
|
# A reference backup must contain at least this many rows (summed across
|
|
94
96
|
# CRITICAL_TABLES) before we will treat it as "proof the user has real data".
|
|
@@ -478,7 +480,7 @@ def _backup_drift_is_safe(source_count: int, backup_count: int) -> bool:
|
|
|
478
480
|
|
|
479
481
|
|
|
480
482
|
def _quote_identifier(identifier: str) -> str:
|
|
481
|
-
if identifier not in PROTECTED_TABLES:
|
|
483
|
+
if identifier not in PROTECTED_TABLES and identifier not in LOCAL_CONTEXT_TABLES:
|
|
482
484
|
raise ValueError(f"refusing unsafe table identifier: {identifier!r}")
|
|
483
485
|
return '"' + identifier.replace('"', '""') + '"'
|
|
484
486
|
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Helpers for interactive MCP tools that must not block on SQLite locks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sqlite3
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
|
|
9
|
+
from db import get_db
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def interactive_db_timeout_ms() -> int:
|
|
13
|
+
"""Return the max SQLite busy wait for user-facing MCP calls."""
|
|
14
|
+
try:
|
|
15
|
+
return max(50, min(int(os.environ.get("NEXO_MCP_DB_BUSY_TIMEOUT_MS", "250")), 10000))
|
|
16
|
+
except Exception:
|
|
17
|
+
return 250
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_interactive_db_timeout() -> None:
|
|
21
|
+
"""Make the shared connection fail fast when a background writer owns the DB."""
|
|
22
|
+
try:
|
|
23
|
+
get_db().execute(f"PRAGMA busy_timeout={interactive_db_timeout_ms()}")
|
|
24
|
+
except Exception:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@contextmanager
|
|
29
|
+
def interactive_db_timeout():
|
|
30
|
+
"""Temporarily reduce SQLite busy wait for a user-facing MCP call."""
|
|
31
|
+
conn = None
|
|
32
|
+
previous = None
|
|
33
|
+
try:
|
|
34
|
+
conn = get_db()
|
|
35
|
+
row = conn.execute("PRAGMA busy_timeout").fetchone()
|
|
36
|
+
previous = int(row[0]) if row and row[0] is not None else None
|
|
37
|
+
conn.execute(f"PRAGMA busy_timeout={interactive_db_timeout_ms()}")
|
|
38
|
+
except Exception:
|
|
39
|
+
conn = None
|
|
40
|
+
try:
|
|
41
|
+
yield
|
|
42
|
+
finally:
|
|
43
|
+
if conn is not None and previous is not None:
|
|
44
|
+
try:
|
|
45
|
+
conn.execute(f"PRAGMA busy_timeout={previous}")
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_db_busy(exc: BaseException) -> bool:
|
|
51
|
+
msg = str(exc).lower()
|
|
52
|
+
return (
|
|
53
|
+
isinstance(exc, sqlite3.OperationalError)
|
|
54
|
+
and (
|
|
55
|
+
"database is locked" in msg
|
|
56
|
+
or "database is busy" in msg
|
|
57
|
+
or "database table is locked" in msg
|
|
58
|
+
)
|
|
59
|
+
) or "database is locked" in msg or "database is busy" in msg
|
package/src/local_context/api.py
CHANGED
|
@@ -11,10 +11,8 @@ import sys
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
|
-
from db import get_db, init_db
|
|
15
|
-
from db._schema import run_migrations
|
|
16
|
-
|
|
17
14
|
from . import embeddings
|
|
15
|
+
from .db import ensure_local_context_db, get_local_context_db
|
|
18
16
|
from .extractors import chunk_text, contains_secret, entities, extract_text, summarize
|
|
19
17
|
from .logging import log_event, tail
|
|
20
18
|
from .privacy import classify_path, is_local_email_tree, is_queryable_path, should_extract, should_skip_file, should_skip_tree
|
|
@@ -88,13 +86,12 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
88
86
|
|
|
89
87
|
|
|
90
88
|
def ensure_ready() -> None:
|
|
91
|
-
|
|
92
|
-
run_migrations()
|
|
89
|
+
ensure_local_context_db()
|
|
93
90
|
|
|
94
91
|
|
|
95
92
|
def _conn():
|
|
96
93
|
ensure_ready()
|
|
97
|
-
return
|
|
94
|
+
return get_local_context_db()
|
|
98
95
|
|
|
99
96
|
|
|
100
97
|
def add_root(path: str, *, mode: str = "normal", depth: int | None = None) -> dict:
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import sqlite3
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Iterable
|
|
8
|
+
|
|
9
|
+
import paths
|
|
10
|
+
from db._schema import _m63_local_context_layer, _m64_local_context_live_dirs
|
|
11
|
+
|
|
12
|
+
LOCAL_CONTEXT_DB_NAME = "local-context.db"
|
|
13
|
+
MIGRATION_STATE_KEY = "local_context_db_migrated_from_main"
|
|
14
|
+
MIGRATION_SKIPPED_KEY = "local_context_db_migration_skipped"
|
|
15
|
+
MAIN_CLEANUP_STATE_KEY = "local_context_main_tables_drained"
|
|
16
|
+
|
|
17
|
+
LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
18
|
+
"local_index_roots",
|
|
19
|
+
"local_index_exclusions",
|
|
20
|
+
"local_index_jobs",
|
|
21
|
+
"local_index_checkpoints",
|
|
22
|
+
"local_index_state",
|
|
23
|
+
"local_index_errors",
|
|
24
|
+
"local_index_logs",
|
|
25
|
+
"local_assets",
|
|
26
|
+
"local_asset_versions",
|
|
27
|
+
"local_chunks",
|
|
28
|
+
"local_entities",
|
|
29
|
+
"local_relations",
|
|
30
|
+
"local_embeddings",
|
|
31
|
+
"local_context_queries",
|
|
32
|
+
"local_index_dirs",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
_CONN: sqlite3.Connection | None = None
|
|
36
|
+
_CONN_PATH: Path | None = None
|
|
37
|
+
_READY = False
|
|
38
|
+
_LAST_MIGRATION_ATTEMPT = 0.0
|
|
39
|
+
_MIGRATION_RETRY_INTERVAL_SECONDS = 300.0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def local_context_db_path() -> Path:
|
|
43
|
+
override = os.environ.get("NEXO_LOCAL_CONTEXT_DB", "").strip()
|
|
44
|
+
if override:
|
|
45
|
+
return Path(override).expanduser()
|
|
46
|
+
test_db = os.environ.get("NEXO_TEST_DB", "").strip()
|
|
47
|
+
if test_db:
|
|
48
|
+
return Path(test_db).expanduser().with_name("test_local_context.db")
|
|
49
|
+
return paths.memory_dir() / LOCAL_CONTEXT_DB_NAME
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _main_db_path_for_migration() -> Path:
|
|
53
|
+
override = os.environ.get("NEXO_LOCAL_CONTEXT_MAIN_DB", "").strip()
|
|
54
|
+
if override:
|
|
55
|
+
return Path(override).expanduser()
|
|
56
|
+
test_db = os.environ.get("NEXO_TEST_DB", "").strip()
|
|
57
|
+
if test_db:
|
|
58
|
+
return Path(test_db).expanduser()
|
|
59
|
+
return paths.db_path()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _busy_timeout_ms() -> int:
|
|
63
|
+
raw = os.environ.get("NEXO_LOCAL_CONTEXT_DB_BUSY_TIMEOUT_MS", "15000")
|
|
64
|
+
try:
|
|
65
|
+
value = int(raw)
|
|
66
|
+
except Exception:
|
|
67
|
+
value = 15000
|
|
68
|
+
return max(1000, min(value, 60000))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _connect(db_path: Path) -> sqlite3.Connection:
|
|
72
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
73
|
+
conn = sqlite3.connect(str(db_path), timeout=max(_busy_timeout_ms() / 1000.0, 1.0), check_same_thread=False)
|
|
74
|
+
conn.row_factory = sqlite3.Row
|
|
75
|
+
conn.execute(f"PRAGMA busy_timeout={_busy_timeout_ms()}")
|
|
76
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
77
|
+
conn.execute("PRAGMA synchronous=NORMAL")
|
|
78
|
+
conn.execute("PRAGMA temp_store=MEMORY")
|
|
79
|
+
return conn
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
|
83
|
+
_m63_local_context_layer(conn)
|
|
84
|
+
_m64_local_context_live_dirs(conn)
|
|
85
|
+
conn.execute("PRAGMA user_version=64")
|
|
86
|
+
conn.commit()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _table_exists(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> bool:
|
|
90
|
+
row = conn.execute(
|
|
91
|
+
f"SELECT 1 FROM {schema}.sqlite_master WHERE type='table' AND name=? LIMIT 1",
|
|
92
|
+
(table,),
|
|
93
|
+
).fetchone()
|
|
94
|
+
return bool(row)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _table_count(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> int:
|
|
98
|
+
if not _table_exists(conn, table, schema=schema):
|
|
99
|
+
return 0
|
|
100
|
+
row = conn.execute(f"SELECT COUNT(*) AS total FROM {schema}.{_quoted(table)}").fetchone()
|
|
101
|
+
return int(row["total"] or 0)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _state(conn: sqlite3.Connection, key: str) -> str:
|
|
105
|
+
row = conn.execute("SELECT value FROM local_index_state WHERE key=?", (key,)).fetchone()
|
|
106
|
+
return str(row["value"] or "") if row else ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _set_state(conn: sqlite3.Connection, key: str, value: str) -> None:
|
|
110
|
+
conn.execute(
|
|
111
|
+
"""
|
|
112
|
+
INSERT INTO local_index_state(key, value, updated_at)
|
|
113
|
+
VALUES (?, ?, strftime('%s','now'))
|
|
114
|
+
ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at
|
|
115
|
+
""",
|
|
116
|
+
(key, value),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _quoted(name: str) -> str:
|
|
121
|
+
return '"' + name.replace('"', '""') + '"'
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _table_columns(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> list[str]:
|
|
125
|
+
try:
|
|
126
|
+
rows = conn.execute(f"PRAGMA {schema}.table_info({_quoted(table)})").fetchall()
|
|
127
|
+
except sqlite3.OperationalError:
|
|
128
|
+
return []
|
|
129
|
+
return [str(row["name"]) for row in rows if row["name"]]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _primary_key_columns(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> list[str]:
|
|
133
|
+
try:
|
|
134
|
+
rows = conn.execute(f"PRAGMA {schema}.table_info({_quoted(table)})").fetchall()
|
|
135
|
+
except sqlite3.OperationalError:
|
|
136
|
+
return []
|
|
137
|
+
ordered = sorted(
|
|
138
|
+
(
|
|
139
|
+
(int(row["pk"] or 0), str(row["name"]))
|
|
140
|
+
for row in rows
|
|
141
|
+
if int(row["pk"] or 0) > 0 and row["name"]
|
|
142
|
+
),
|
|
143
|
+
key=lambda item: item[0],
|
|
144
|
+
)
|
|
145
|
+
return [name for _order, name in ordered]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _source_rows_missing_in_target(conn: sqlite3.Connection, table: str) -> int:
|
|
149
|
+
target_pk = _primary_key_columns(conn, table)
|
|
150
|
+
source_columns = set(_table_columns(conn, table, schema="source"))
|
|
151
|
+
pk_columns = [column for column in target_pk if column in source_columns]
|
|
152
|
+
if not pk_columns:
|
|
153
|
+
raise RuntimeError(f"cannot verify local context migration for {table}: missing primary key")
|
|
154
|
+
join_sql = " AND ".join(f"s.{_quoted(column)} = t.{_quoted(column)}" for column in pk_columns)
|
|
155
|
+
null_check = f"t.{_quoted(pk_columns[0])} IS NULL"
|
|
156
|
+
row = conn.execute(
|
|
157
|
+
f"SELECT COUNT(*) AS total "
|
|
158
|
+
f"FROM source.{_quoted(table)} s "
|
|
159
|
+
f"LEFT JOIN {_quoted(table)} t ON {join_sql} "
|
|
160
|
+
f"WHERE {null_check}"
|
|
161
|
+
).fetchone()
|
|
162
|
+
return int(row["total"] or 0)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _copy_local_tables(conn: sqlite3.Connection, tables: Iterable[str]) -> dict[str, int]:
|
|
166
|
+
copied: dict[str, int] = {}
|
|
167
|
+
for table in tables:
|
|
168
|
+
if not _table_exists(conn, table, schema="source"):
|
|
169
|
+
continue
|
|
170
|
+
target_columns = _table_columns(conn, table)
|
|
171
|
+
source_columns = set(_table_columns(conn, table, schema="source"))
|
|
172
|
+
columns = [column for column in target_columns if column in source_columns]
|
|
173
|
+
if not columns:
|
|
174
|
+
continue
|
|
175
|
+
before = _table_count(conn, table)
|
|
176
|
+
column_sql = ", ".join(_quoted(column) for column in columns)
|
|
177
|
+
conn.execute(
|
|
178
|
+
f"INSERT OR IGNORE INTO {_quoted(table)} ({column_sql}) "
|
|
179
|
+
f"SELECT {column_sql} FROM source.{_quoted(table)}"
|
|
180
|
+
)
|
|
181
|
+
after = _table_count(conn, table)
|
|
182
|
+
copied[table] = max(0, after - before)
|
|
183
|
+
return copied
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _source_table_counts(conn: sqlite3.Connection, tables: Iterable[str]) -> dict[str, int]:
|
|
187
|
+
counts: dict[str, int] = {}
|
|
188
|
+
for table in tables:
|
|
189
|
+
if _table_exists(conn, table, schema="source"):
|
|
190
|
+
counts[table] = _table_count(conn, table, schema="source")
|
|
191
|
+
return counts
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _drain_source_local_tables_if_verified(
|
|
195
|
+
conn: sqlite3.Connection,
|
|
196
|
+
source_counts: dict[str, int],
|
|
197
|
+
) -> dict[str, int]:
|
|
198
|
+
"""Clear old local-memory rows from main DB after verifying the copy.
|
|
199
|
+
|
|
200
|
+
We keep the table schema in ``nexo.db`` for backward compatibility with
|
|
201
|
+
older binaries, but empty the rows after the new DB has at least the same
|
|
202
|
+
number of records per table. This prevents two sources of truth and stops
|
|
203
|
+
the main DB from carrying the huge local-memory payload.
|
|
204
|
+
"""
|
|
205
|
+
drained: dict[str, int] = {}
|
|
206
|
+
if not source_counts:
|
|
207
|
+
return drained
|
|
208
|
+
|
|
209
|
+
for table, source_count in source_counts.items():
|
|
210
|
+
local_count = _table_count(conn, table)
|
|
211
|
+
missing = _source_rows_missing_in_target(conn, table)
|
|
212
|
+
if local_count < source_count or missing:
|
|
213
|
+
raise RuntimeError(
|
|
214
|
+
f"local context migration verification failed for {table}: "
|
|
215
|
+
f"local={local_count} source={source_count} missing={missing}"
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
for table, source_count in source_counts.items():
|
|
219
|
+
if source_count <= 0:
|
|
220
|
+
continue
|
|
221
|
+
conn.execute(f"DELETE FROM source.{_quoted(table)}")
|
|
222
|
+
drained[table] = source_count
|
|
223
|
+
|
|
224
|
+
if drained:
|
|
225
|
+
_set_state(conn, MAIN_CLEANUP_STATE_KEY, ",".join(sorted(drained)))
|
|
226
|
+
return drained
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def migrate_from_main_if_needed(conn: sqlite3.Connection) -> dict:
|
|
230
|
+
if os.environ.get("NEXO_LOCAL_CONTEXT_DISABLE_MAIN_MIGRATION", "").strip().lower() in {"1", "true", "yes"}:
|
|
231
|
+
return {"ok": True, "skipped": "disabled"}
|
|
232
|
+
|
|
233
|
+
main_db = _main_db_path_for_migration()
|
|
234
|
+
if not main_db.is_file():
|
|
235
|
+
_set_state(conn, MIGRATION_SKIPPED_KEY, "main_db_missing")
|
|
236
|
+
conn.commit()
|
|
237
|
+
return {"ok": True, "skipped": "main_db_missing"}
|
|
238
|
+
|
|
239
|
+
source = str(main_db).replace("'", "''")
|
|
240
|
+
try:
|
|
241
|
+
conn.execute("PRAGMA busy_timeout=1000")
|
|
242
|
+
conn.execute(f"ATTACH DATABASE '{source}' AS source")
|
|
243
|
+
if not _table_exists(conn, "local_assets", schema="source") and not _table_exists(conn, "local_index_roots", schema="source"):
|
|
244
|
+
_set_state(conn, MIGRATION_SKIPPED_KEY, "main_local_tables_missing")
|
|
245
|
+
if _table_count(conn, "local_assets") > 0 or _table_count(conn, "local_index_jobs") > 0:
|
|
246
|
+
_set_state(conn, MIGRATION_STATE_KEY, "already_has_local_data")
|
|
247
|
+
conn.commit()
|
|
248
|
+
return {"ok": True, "skipped": "main_local_tables_missing"}
|
|
249
|
+
source_counts = _source_table_counts(conn, LOCAL_CONTEXT_TABLES)
|
|
250
|
+
if not any(source_counts.values()):
|
|
251
|
+
_set_state(conn, MIGRATION_STATE_KEY, "main_local_tables_empty")
|
|
252
|
+
_set_state(conn, MAIN_CLEANUP_STATE_KEY, "empty")
|
|
253
|
+
conn.commit()
|
|
254
|
+
return {"ok": True, "skipped": "main_local_tables_empty"}
|
|
255
|
+
copied = _copy_local_tables(conn, LOCAL_CONTEXT_TABLES)
|
|
256
|
+
drained = _drain_source_local_tables_if_verified(conn, source_counts)
|
|
257
|
+
_set_state(conn, MIGRATION_STATE_KEY, str(main_db))
|
|
258
|
+
_set_state(conn, "local_context_db_migrated_rows", str(sum(copied.values())))
|
|
259
|
+
if drained:
|
|
260
|
+
_set_state(conn, "local_context_main_tables_drained_rows", str(sum(drained.values())))
|
|
261
|
+
conn.commit()
|
|
262
|
+
return {"ok": True, "migrated_from": str(main_db), "copied": copied, "drained": drained}
|
|
263
|
+
except sqlite3.OperationalError as exc:
|
|
264
|
+
message = str(exc)
|
|
265
|
+
_set_state(conn, MIGRATION_SKIPPED_KEY, message[:240])
|
|
266
|
+
_set_state(conn, "local_context_db_drain_pending", "main_db_busy_or_unavailable")
|
|
267
|
+
conn.commit()
|
|
268
|
+
return {"ok": True, "skipped": "main_db_busy_or_unavailable", "error": message, "retry_pending": True}
|
|
269
|
+
finally:
|
|
270
|
+
try:
|
|
271
|
+
conn.execute("DETACH DATABASE source")
|
|
272
|
+
except Exception:
|
|
273
|
+
pass
|
|
274
|
+
conn.execute(f"PRAGMA busy_timeout={_busy_timeout_ms()}")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def ensure_local_context_db() -> None:
|
|
278
|
+
global _CONN, _CONN_PATH, _READY, _LAST_MIGRATION_ATTEMPT
|
|
279
|
+
db_path = local_context_db_path()
|
|
280
|
+
if _CONN is not None and _CONN_PATH != db_path:
|
|
281
|
+
close_local_context_db()
|
|
282
|
+
if _CONN is None:
|
|
283
|
+
_CONN = _connect(db_path)
|
|
284
|
+
_CONN_PATH = db_path
|
|
285
|
+
now = time.monotonic()
|
|
286
|
+
if _READY:
|
|
287
|
+
if now - _LAST_MIGRATION_ATTEMPT >= _MIGRATION_RETRY_INTERVAL_SECONDS:
|
|
288
|
+
_LAST_MIGRATION_ATTEMPT = now
|
|
289
|
+
try:
|
|
290
|
+
migrate_from_main_if_needed(_CONN)
|
|
291
|
+
except Exception:
|
|
292
|
+
# Sidecar data is still usable; cleanup from the old main DB is
|
|
293
|
+
# best-effort and will retry on later service cycles.
|
|
294
|
+
pass
|
|
295
|
+
return
|
|
296
|
+
_ensure_schema(_CONN)
|
|
297
|
+
_LAST_MIGRATION_ATTEMPT = now
|
|
298
|
+
migration = migrate_from_main_if_needed(_CONN)
|
|
299
|
+
_READY = True
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def get_local_context_db() -> sqlite3.Connection:
|
|
303
|
+
ensure_local_context_db()
|
|
304
|
+
assert _CONN is not None
|
|
305
|
+
return _CONN
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def close_local_context_db() -> None:
|
|
309
|
+
global _CONN, _CONN_PATH, _READY, _LAST_MIGRATION_ATTEMPT
|
|
310
|
+
if _CONN is not None:
|
|
311
|
+
try:
|
|
312
|
+
_CONN.close()
|
|
313
|
+
except Exception:
|
|
314
|
+
pass
|
|
315
|
+
_CONN = None
|
|
316
|
+
_CONN_PATH = None
|
|
317
|
+
_READY = False
|
|
318
|
+
_LAST_MIGRATION_ATTEMPT = 0.0
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from db import
|
|
4
|
-
|
|
3
|
+
from .db import get_local_context_db
|
|
5
4
|
from .util import json_dumps, now
|
|
6
5
|
|
|
7
6
|
|
|
8
7
|
def log_event(level: str, event: str, message: str, **metadata) -> None:
|
|
9
|
-
conn =
|
|
8
|
+
conn = get_local_context_db()
|
|
10
9
|
conn.execute(
|
|
11
10
|
"""
|
|
12
11
|
INSERT INTO local_index_logs(created_at, level, event, message, metadata_json)
|
|
@@ -18,7 +17,7 @@ def log_event(level: str, event: str, message: str, **metadata) -> None:
|
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
def tail(limit: int = 100) -> list[dict]:
|
|
21
|
-
conn =
|
|
20
|
+
conn = get_local_context_db()
|
|
22
21
|
rows = conn.execute(
|
|
23
22
|
"""
|
|
24
23
|
SELECT created_at, level, event, message, metadata_json
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
"""Required MCP tool contract shared by Brain and Desktop probes.
|
|
4
|
+
|
|
5
|
+
These tools are the minimum bootstrap surface NEXO Desktop needs before a
|
|
6
|
+
conversation can be considered healthy. Dynamic plugin loading may still add
|
|
7
|
+
more tools, but these names must always be present.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
BOOTSTRAP_REQUIRED_MCP_TOOLS: tuple[str, ...] = (
|
|
11
|
+
"nexo_startup",
|
|
12
|
+
"nexo_heartbeat",
|
|
13
|
+
"nexo_session_diary_read",
|
|
14
|
+
"nexo_reminders",
|
|
15
|
+
"nexo_smart_startup",
|
|
16
|
+
"nexo_task_open",
|
|
17
|
+
"nexo_task_close",
|
|
18
|
+
"nexo_task_acknowledge_guard",
|
|
19
|
+
"nexo_guard_check",
|
|
20
|
+
"nexo_learning_add",
|
|
21
|
+
"nexo_confidence_check",
|
|
22
|
+
"nexo_followup_create",
|
|
23
|
+
"nexo_protocol_debt_resolve",
|
|
24
|
+
"nexo_card_match",
|
|
25
|
+
"nexo_skill_match",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def missing_required_tools(tool_names: list[str] | tuple[str, ...] | set[str]) -> list[str]:
|
|
30
|
+
available = {str(name) for name in tool_names}
|
|
31
|
+
return [name for name in BOOTSTRAP_REQUIRED_MCP_TOOLS if name not in available]
|
|
@@ -10,6 +10,7 @@ from db import (
|
|
|
10
10
|
log_change, search_changes, update_change_commit,
|
|
11
11
|
recall, get_db, set_linked_outcomes_met,
|
|
12
12
|
)
|
|
13
|
+
from interactive_db import interactive_db_timeout, is_db_busy
|
|
13
14
|
|
|
14
15
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
15
16
|
_REPO_ABS_PREFIX = str(REPO_ROOT) + "/"
|
|
@@ -377,6 +378,23 @@ def handle_session_diary_read(session_id: str = '', last_n: int = 3, last_day: b
|
|
|
377
378
|
brief: If true, returns ONLY the last diary entry with summary + mental_state + context_next.
|
|
378
379
|
Use this at startup for fast context loading (~1K chars instead of full dump).
|
|
379
380
|
"""
|
|
381
|
+
with interactive_db_timeout():
|
|
382
|
+
try:
|
|
383
|
+
return _handle_session_diary_read_inner(session_id, last_n, last_day, domain, brief)
|
|
384
|
+
except Exception as exc:
|
|
385
|
+
if is_db_busy(exc):
|
|
386
|
+
return (
|
|
387
|
+
"SESSION DIARY DEGRADED:\n"
|
|
388
|
+
" local brain database is busy; continue with the user request and retry shortly."
|
|
389
|
+
)
|
|
390
|
+
return (
|
|
391
|
+
"SESSION DIARY DEGRADED:\n"
|
|
392
|
+
f" skipped ({type(exc).__name__}); continue with the user request and retry shortly."
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _handle_session_diary_read_inner(session_id: str = '', last_n: int = 3, last_day: bool = False,
|
|
397
|
+
domain: str = '', brief: bool = False) -> str:
|
|
380
398
|
if brief:
|
|
381
399
|
# Fast path: only the most recent diary entry, compact format
|
|
382
400
|
results = read_session_diary(session_id, 1, last_day=False, domain=domain)
|
package/src/plugins/skills.py
CHANGED
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
|
|
7
|
+
from interactive_db import interactive_db_timeout, is_db_busy
|
|
7
8
|
from db import (
|
|
8
9
|
create_skill,
|
|
9
10
|
delete_skill,
|
|
@@ -85,12 +86,22 @@ def handle_skill_create(
|
|
|
85
86
|
|
|
86
87
|
|
|
87
88
|
def handle_skill_match(task: str, level: str = "") -> str:
|
|
88
|
-
|
|
89
|
-
|
|
89
|
+
sync_warning = ""
|
|
90
|
+
with interactive_db_timeout():
|
|
91
|
+
try:
|
|
92
|
+
sync_skills()
|
|
93
|
+
except Exception as exc:
|
|
94
|
+
if is_db_busy(exc):
|
|
95
|
+
sync_warning = "SKILL SYNC DEGRADED: local brain database is busy; using existing skill index."
|
|
96
|
+
else:
|
|
97
|
+
raise
|
|
98
|
+
matches = match_skills(task, level=level)
|
|
90
99
|
if not matches:
|
|
91
|
-
return f"No skills found for: '{task}'"
|
|
100
|
+
return f"{sync_warning + chr(10) if sync_warning else ''}No skills found for: '{task}'"
|
|
92
101
|
|
|
93
102
|
lines = [f"SKILLS MATCHED ({len(matches)}) for '{task}':"]
|
|
103
|
+
if sync_warning:
|
|
104
|
+
lines.insert(0, sync_warning)
|
|
94
105
|
for match in matches:
|
|
95
106
|
match_method = match.pop("_match", "unknown")
|
|
96
107
|
lines.append(
|
package/src/plugins/update.py
CHANGED
|
@@ -39,6 +39,7 @@ try:
|
|
|
39
39
|
from db_guard import (
|
|
40
40
|
CRITICAL_TABLES,
|
|
41
41
|
HOURLY_BACKUP_MAX_AGE,
|
|
42
|
+
LOCAL_CONTEXT_TABLES,
|
|
42
43
|
MIN_REFERENCE_ROWS,
|
|
43
44
|
PROTECTED_TABLES,
|
|
44
45
|
WIPE_THRESHOLD_PCT,
|
|
@@ -56,6 +57,7 @@ except Exception as exc: # pragma: no cover - exercised only during stale insta
|
|
|
56
57
|
_DB_GUARD_AVAILABLE = False
|
|
57
58
|
_DB_GUARD_IMPORT_ERROR = str(exc)
|
|
58
59
|
CRITICAL_TABLES = ()
|
|
60
|
+
LOCAL_CONTEXT_TABLES = ()
|
|
59
61
|
PROTECTED_TABLES = ()
|
|
60
62
|
HOURLY_BACKUP_MAX_AGE = 48 * 3600
|
|
61
63
|
MIN_REFERENCE_ROWS = 50
|
|
@@ -98,6 +100,14 @@ REQUIRED_LOCAL_MEMORY_TABLES: tuple[str, ...] = (
|
|
|
98
100
|
"local_index_dirs",
|
|
99
101
|
)
|
|
100
102
|
|
|
103
|
+
|
|
104
|
+
def _backup_validation_tables(db_file: Path) -> tuple[str, ...]:
|
|
105
|
+
if db_file.name == "nexo.db":
|
|
106
|
+
return tuple(PROTECTED_TABLES)
|
|
107
|
+
if db_file.name == "local-context.db":
|
|
108
|
+
return tuple(LOCAL_CONTEXT_TABLES)
|
|
109
|
+
return ()
|
|
110
|
+
|
|
101
111
|
# Code root is the parent of plugins/:
|
|
102
112
|
# - source checkout: <repo>/src
|
|
103
113
|
# - packaged runtime: <NEXO_HOME>
|
|
@@ -404,11 +414,11 @@ def _db_guard_integrity_error() -> str | None:
|
|
|
404
414
|
f" import error: {_DB_GUARD_IMPORT_ERROR or 'unknown'}\n"
|
|
405
415
|
"Restart Desktop so it can sync the bundled Brain guard, then retry the update."
|
|
406
416
|
)
|
|
407
|
-
|
|
408
|
-
missing = [table for table in REQUIRED_LOCAL_MEMORY_TABLES if table not in
|
|
417
|
+
local_tables = set(LOCAL_CONTEXT_TABLES)
|
|
418
|
+
missing = [table for table in REQUIRED_LOCAL_MEMORY_TABLES if table not in local_tables]
|
|
409
419
|
if missing:
|
|
410
420
|
return (
|
|
411
|
-
"DB protection module is stale; refusing to update because local memory tables are not
|
|
421
|
+
"DB protection module is stale; refusing to update because local memory tables are not known.\n"
|
|
412
422
|
f" missing tables: {', '.join(missing)}\n"
|
|
413
423
|
"Restart Desktop so it can sync the bundled Brain guard, then retry the update."
|
|
414
424
|
)
|
|
@@ -505,6 +515,9 @@ def _backup_databases() -> tuple[str, str | None]:
|
|
|
505
515
|
backup_dir = BACKUP_BASE / f"pre-update-{timestamp}"
|
|
506
516
|
|
|
507
517
|
db_files = list(DATA_DIR.glob("*.db")) if DATA_DIR.is_dir() else []
|
|
518
|
+
local_context_db = paths.memory_dir() / "local-context.db"
|
|
519
|
+
if local_context_db.is_file():
|
|
520
|
+
db_files.append(local_context_db)
|
|
508
521
|
# Also check NEXO_HOME root for legacy db location
|
|
509
522
|
db_files += [f for f in NEXO_HOME.glob("*.db") if f.is_file()]
|
|
510
523
|
# And check src/ dir for nexo.db (dev mode)
|
|
@@ -522,10 +535,9 @@ def _backup_databases() -> tuple[str, str | None]:
|
|
|
522
535
|
ok, err = safe_sqlite_backup(db_file, dest)
|
|
523
536
|
if not ok:
|
|
524
537
|
return str(backup_dir), f"Failed to backup {db_file.name}: {err}"
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
valid, valid_err = validate_backup_matches_source(db_file, dest, PROTECTED_TABLES)
|
|
538
|
+
tables = _backup_validation_tables(db_file)
|
|
539
|
+
if tables:
|
|
540
|
+
valid, valid_err = validate_backup_matches_source(db_file, dest, tables)
|
|
529
541
|
if not valid:
|
|
530
542
|
return str(backup_dir), (
|
|
531
543
|
f"Backup of {db_file.name} did not preserve critical tables: {valid_err}"
|
|
@@ -544,12 +556,19 @@ def _restore_databases(backup_dir: str):
|
|
|
544
556
|
if not bdir.is_dir():
|
|
545
557
|
return
|
|
546
558
|
for db_backup in bdir.glob("*.db"):
|
|
547
|
-
# Try to find original location
|
|
548
|
-
|
|
549
|
-
|
|
559
|
+
# Try to find original location. local-context.db lives outside the
|
|
560
|
+
# operational DB directory and must be restored even if the target was
|
|
561
|
+
# missing after a failed update.
|
|
562
|
+
if db_backup.name == "local-context.db":
|
|
563
|
+
candidates = [paths.memory_dir() / db_backup.name]
|
|
564
|
+
else:
|
|
565
|
+
candidates = [DATA_DIR / db_backup.name, NEXO_HOME / db_backup.name, SRC_DIR / db_backup.name]
|
|
566
|
+
for candidate in candidates:
|
|
567
|
+
if candidate.is_file() or db_backup.name == "local-context.db":
|
|
550
568
|
src_conn = None
|
|
551
569
|
dst_conn = None
|
|
552
570
|
try:
|
|
571
|
+
candidate.parent.mkdir(parents=True, exist_ok=True)
|
|
553
572
|
src_conn = sqlite3.connect(str(db_backup))
|
|
554
573
|
dst_conn = sqlite3.connect(str(candidate))
|
|
555
574
|
src_conn.backup(dst_conn)
|
|
@@ -8,24 +8,130 @@ if [ ! -d "$BACKUP_DIR" ] && [ -d "$NEXO_HOME/backups" ]; then
|
|
|
8
8
|
fi
|
|
9
9
|
WEEKLY_DIR="$BACKUP_DIR/weekly"
|
|
10
10
|
DB="$NEXO_HOME/runtime/data/nexo.db"
|
|
11
|
-
|
|
11
|
+
LOCAL_CONTEXT_DB="$NEXO_HOME/runtime/memory/local-context.db"
|
|
12
|
+
LOCK_FILE="$NEXO_HOME/runtime/logs/local-index.lock"
|
|
13
|
+
RETENTION_HOURS="${NEXO_BACKUP_RETENTION_HOURS:-24}"
|
|
14
|
+
KEEP_LAST="${NEXO_BACKUP_KEEP_LAST:-3}"
|
|
15
|
+
FAMILY_KEEP_LAST="${NEXO_BACKUP_FAMILY_KEEP_LAST:-2}"
|
|
16
|
+
LOCAL_CONTEXT_RETENTION_HOURS="${NEXO_LOCAL_CONTEXT_BACKUP_RETENTION_HOURS:-24}"
|
|
17
|
+
LOCAL_CONTEXT_KEEP_LAST="${NEXO_LOCAL_CONTEXT_BACKUP_KEEP_LAST:-2}"
|
|
18
|
+
BUSY_TIMEOUT_MS="${NEXO_BACKUP_BUSY_TIMEOUT_MS:-5000}"
|
|
19
|
+
RECENT_BACKUP_HOURS="${NEXO_BACKUP_RECENT_BACKUP_HOURS:-6}"
|
|
12
20
|
|
|
13
21
|
mkdir -p "$BACKUP_DIR" "$WEEKLY_DIR"
|
|
14
22
|
|
|
23
|
+
cleanup_backups() {
|
|
24
|
+
python3 - "$BACKUP_DIR" "$RETENTION_HOURS" "$KEEP_LAST" "$FAMILY_KEEP_LAST" "$LOCAL_CONTEXT_RETENTION_HOURS" "$LOCAL_CONTEXT_KEEP_LAST" <<'PY'
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import shutil
|
|
28
|
+
import sys
|
|
29
|
+
import time
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
base = Path(sys.argv[1])
|
|
33
|
+
retention_hours = max(1, int(sys.argv[2]))
|
|
34
|
+
keep_last = max(1, int(sys.argv[3]))
|
|
35
|
+
family_keep_last = max(1, int(sys.argv[4]))
|
|
36
|
+
local_context_retention_hours = max(1, int(sys.argv[5]))
|
|
37
|
+
local_context_keep_last = max(1, int(sys.argv[6]))
|
|
38
|
+
now = time.time()
|
|
39
|
+
|
|
40
|
+
def delete_path(path: Path) -> None:
|
|
41
|
+
try:
|
|
42
|
+
if path.is_dir():
|
|
43
|
+
shutil.rmtree(path)
|
|
44
|
+
else:
|
|
45
|
+
path.unlink()
|
|
46
|
+
except FileNotFoundError:
|
|
47
|
+
pass
|
|
48
|
+
except Exception as exc:
|
|
49
|
+
print(f"NEXO backup cleanup warning: {path}: {exc}", file=sys.stderr)
|
|
50
|
+
|
|
51
|
+
for tmp in base.glob("*.tmp.*"):
|
|
52
|
+
try:
|
|
53
|
+
if now - tmp.stat().st_mtime > 1800:
|
|
54
|
+
delete_path(tmp)
|
|
55
|
+
except FileNotFoundError:
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
hourlies = sorted(base.glob("nexo-*.db"), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True)
|
|
59
|
+
for backup in hourlies[keep_last:]:
|
|
60
|
+
try:
|
|
61
|
+
age_hours = (now - backup.stat().st_mtime) / 3600
|
|
62
|
+
except FileNotFoundError:
|
|
63
|
+
continue
|
|
64
|
+
if age_hours > retention_hours:
|
|
65
|
+
delete_path(backup)
|
|
66
|
+
|
|
67
|
+
local_context_hourlies = sorted(
|
|
68
|
+
base.glob("local-context-*.db"),
|
|
69
|
+
key=lambda p: p.stat().st_mtime if p.exists() else 0,
|
|
70
|
+
reverse=True,
|
|
71
|
+
)
|
|
72
|
+
for backup in local_context_hourlies[local_context_keep_last:]:
|
|
73
|
+
try:
|
|
74
|
+
age_hours = (now - backup.stat().st_mtime) / 3600
|
|
75
|
+
except FileNotFoundError:
|
|
76
|
+
continue
|
|
77
|
+
if age_hours > local_context_retention_hours:
|
|
78
|
+
delete_path(backup)
|
|
79
|
+
|
|
80
|
+
for pattern in (
|
|
81
|
+
"pre-backfill-owner-*",
|
|
82
|
+
"pre-update-*",
|
|
83
|
+
"pre-autoupdate-*",
|
|
84
|
+
"pre-restore-*",
|
|
85
|
+
"app-reinstall-*",
|
|
86
|
+
"app-install-*",
|
|
87
|
+
"desktop-local-install-*",
|
|
88
|
+
"code-tree-*",
|
|
89
|
+
):
|
|
90
|
+
entries = sorted(base.glob(pattern), key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True)
|
|
91
|
+
for entry in entries[family_keep_last:]:
|
|
92
|
+
delete_path(entry)
|
|
93
|
+
|
|
94
|
+
weekly = base / "weekly"
|
|
95
|
+
if weekly.exists():
|
|
96
|
+
for backup in weekly.glob("weekly-*.db"):
|
|
97
|
+
try:
|
|
98
|
+
if now - backup.stat().st_mtime > 90 * 24 * 3600:
|
|
99
|
+
delete_path(backup)
|
|
100
|
+
except FileNotFoundError:
|
|
101
|
+
pass
|
|
102
|
+
PY
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
has_recent_backup() {
|
|
106
|
+
find "$BACKUP_DIR" -maxdepth 1 -name "nexo-*.db" -mmin "-$((RECENT_BACKUP_HOURS * 60))" -print -quit | grep -q .
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
has_recent_local_context_backup() {
|
|
110
|
+
find "$BACKUP_DIR" -maxdepth 1 -name "local-context-*.db" -mmin "-$((RECENT_BACKUP_HOURS * 60))" -print -quit | grep -q .
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
cleanup_backups
|
|
114
|
+
|
|
15
115
|
# Hourly backup
|
|
16
116
|
TIMESTAMP=$(date +%Y-%m-%d-%H%M)
|
|
17
117
|
BACKUP_FILE="$BACKUP_DIR/nexo-$TIMESTAMP.db"
|
|
18
118
|
TMP_BACKUP="$BACKUP_FILE.tmp.$$"
|
|
19
119
|
rm -f "$TMP_BACKUP"
|
|
20
|
-
if sqlite3 -cmd ".timeout
|
|
21
|
-
PRAGMA busy_timeout
|
|
120
|
+
if sqlite3 -cmd ".timeout $BUSY_TIMEOUT_MS" "$DB" <<SQL
|
|
121
|
+
PRAGMA busy_timeout=$BUSY_TIMEOUT_MS;
|
|
22
122
|
.backup '$TMP_BACKUP'
|
|
23
123
|
SQL
|
|
24
124
|
then
|
|
25
125
|
mv "$TMP_BACKUP" "$BACKUP_FILE"
|
|
26
126
|
else
|
|
27
127
|
rm -f "$TMP_BACKUP"
|
|
28
|
-
|
|
128
|
+
if has_recent_backup; then
|
|
129
|
+
echo "NEXO backup skipped: database busy and a recent backup exists" >&2
|
|
130
|
+
cleanup_backups
|
|
131
|
+
exit 0
|
|
132
|
+
fi
|
|
133
|
+
echo "NEXO backup failed: database busy or unavailable and no recent backup exists" >&2
|
|
134
|
+
cleanup_backups
|
|
29
135
|
exit 1
|
|
30
136
|
fi
|
|
31
137
|
|
|
@@ -36,6 +142,24 @@ if [ ! -f "$WEEKLY_FILE" ] && [ "$(date +%u)" = "7" ] && [ -f "$BACKUP_FILE" ];
|
|
|
36
142
|
cp "$BACKUP_FILE" "$WEEKLY_FILE"
|
|
37
143
|
fi
|
|
38
144
|
|
|
39
|
-
#
|
|
40
|
-
|
|
41
|
-
|
|
145
|
+
# Local memory backup: separate and aggressively rotated so the index cannot
|
|
146
|
+
# block core DB backups or fill the disk with duplicate multi-GB snapshots.
|
|
147
|
+
if [ -f "$LOCAL_CONTEXT_DB" ]; then
|
|
148
|
+
LOCAL_CONTEXT_BACKUP_FILE="$BACKUP_DIR/local-context-$TIMESTAMP.db"
|
|
149
|
+
LOCAL_CONTEXT_TMP_BACKUP="$LOCAL_CONTEXT_BACKUP_FILE.tmp.$$"
|
|
150
|
+
rm -f "$LOCAL_CONTEXT_TMP_BACKUP"
|
|
151
|
+
if [ -f "$LOCK_FILE" ] && find "$LOCK_FILE" -mmin -30 -print -quit | grep -q . && has_recent_local_context_backup; then
|
|
152
|
+
echo "NEXO local memory backup skipped: index is active and a recent local backup exists"
|
|
153
|
+
elif sqlite3 -cmd ".timeout $BUSY_TIMEOUT_MS" "$LOCAL_CONTEXT_DB" <<SQL
|
|
154
|
+
PRAGMA busy_timeout=$BUSY_TIMEOUT_MS;
|
|
155
|
+
.backup '$LOCAL_CONTEXT_TMP_BACKUP'
|
|
156
|
+
SQL
|
|
157
|
+
then
|
|
158
|
+
mv "$LOCAL_CONTEXT_TMP_BACKUP" "$LOCAL_CONTEXT_BACKUP_FILE"
|
|
159
|
+
else
|
|
160
|
+
rm -f "$LOCAL_CONTEXT_TMP_BACKUP"
|
|
161
|
+
echo "NEXO local memory backup warning: local-context database busy or unavailable" >&2
|
|
162
|
+
fi
|
|
163
|
+
fi
|
|
164
|
+
|
|
165
|
+
cleanup_backups
|
package/src/server.py
CHANGED
|
@@ -5,6 +5,7 @@ import os
|
|
|
5
5
|
import signal
|
|
6
6
|
import sys
|
|
7
7
|
import json
|
|
8
|
+
import time
|
|
8
9
|
|
|
9
10
|
from fastmcp import FastMCP
|
|
10
11
|
from core_prompts import render_core_prompt
|
|
@@ -94,10 +95,15 @@ from tools_automation_sessions import (
|
|
|
94
95
|
from plugins.cortex import handle_cortex_check
|
|
95
96
|
from plugins.guard import handle_guard_check
|
|
96
97
|
from plugins.protocol import (
|
|
98
|
+
handle_confidence_check,
|
|
99
|
+
handle_protocol_debt_resolve,
|
|
97
100
|
handle_task_acknowledge_guard,
|
|
98
101
|
handle_task_close,
|
|
99
102
|
handle_task_open,
|
|
100
103
|
)
|
|
104
|
+
from plugins.episodic_memory import handle_session_diary_read
|
|
105
|
+
from plugins.cards import handle_card_match
|
|
106
|
+
from plugins.skills import handle_skill_match
|
|
101
107
|
from plugins.workflow import (
|
|
102
108
|
handle_workflow_open,
|
|
103
109
|
handle_workflow_update,
|
|
@@ -112,10 +118,12 @@ from runtime_versioning import (
|
|
|
112
118
|
prime_process_version,
|
|
113
119
|
)
|
|
114
120
|
from local_context import api as local_context_api
|
|
121
|
+
from local_context.db import close_local_context_db
|
|
115
122
|
|
|
116
123
|
|
|
117
124
|
# ── Graceful shutdown: close DB on any termination signal ──────────
|
|
118
125
|
def _shutdown_handler(signum, frame):
|
|
126
|
+
close_local_context_db()
|
|
119
127
|
close_db()
|
|
120
128
|
sys.exit(0)
|
|
121
129
|
|
|
@@ -184,11 +192,23 @@ def _restore_valid_db_backup() -> bool:
|
|
|
184
192
|
def _init_db_or_exit() -> None:
|
|
185
193
|
import sqlite3
|
|
186
194
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
195
|
+
for attempt in range(3):
|
|
196
|
+
try:
|
|
197
|
+
init_db()
|
|
198
|
+
return
|
|
199
|
+
except sqlite3.OperationalError as exc:
|
|
200
|
+
message = str(exc).lower()
|
|
201
|
+
if "locked" in message or "busy" in message:
|
|
202
|
+
if attempt < 2:
|
|
203
|
+
time.sleep(0.25 * (attempt + 1))
|
|
204
|
+
continue
|
|
205
|
+
print(f"[NEXO] DB init temporarily busy: {exc}", file=sys.stderr)
|
|
206
|
+
raise SystemExit(75)
|
|
207
|
+
print(f"[NEXO] DB init failed: {exc}", file=sys.stderr)
|
|
208
|
+
break
|
|
209
|
+
except sqlite3.DatabaseError as exc:
|
|
210
|
+
print(f"[NEXO] DB init failed: {exc}", file=sys.stderr)
|
|
211
|
+
break
|
|
192
212
|
|
|
193
213
|
restored = False
|
|
194
214
|
try:
|
|
@@ -439,6 +459,76 @@ def nexo_heartbeat(sid: str, task: str, context_hint: str = '') -> str:
|
|
|
439
459
|
return handle_heartbeat(sid, task, context_hint)
|
|
440
460
|
|
|
441
461
|
|
|
462
|
+
@mcp.tool
|
|
463
|
+
def nexo_session_diary_read(
|
|
464
|
+
session_id: str = "",
|
|
465
|
+
last_n: int = 3,
|
|
466
|
+
last_day: bool = False,
|
|
467
|
+
domain: str = "",
|
|
468
|
+
brief: bool = False,
|
|
469
|
+
) -> str:
|
|
470
|
+
"""Read recent session diaries for context continuity."""
|
|
471
|
+
return handle_session_diary_read(session_id, last_n, last_day, domain, brief)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
@mcp.tool
|
|
475
|
+
def nexo_confidence_check(
|
|
476
|
+
goal: str,
|
|
477
|
+
task_type: str = "answer",
|
|
478
|
+
area: str = "",
|
|
479
|
+
context_hint: str = "",
|
|
480
|
+
constraints: str = "[]",
|
|
481
|
+
evidence_refs: str = "[]",
|
|
482
|
+
unknowns: str = "[]",
|
|
483
|
+
verification_step: str = "",
|
|
484
|
+
stakes: str = "",
|
|
485
|
+
) -> str:
|
|
486
|
+
"""Decide whether an answer should proceed directly or be verified first."""
|
|
487
|
+
return handle_confidence_check(
|
|
488
|
+
goal,
|
|
489
|
+
task_type,
|
|
490
|
+
area,
|
|
491
|
+
context_hint,
|
|
492
|
+
constraints,
|
|
493
|
+
evidence_refs,
|
|
494
|
+
unknowns,
|
|
495
|
+
verification_step,
|
|
496
|
+
stakes,
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
@mcp.tool
|
|
501
|
+
def nexo_protocol_debt_resolve(
|
|
502
|
+
debt_ids: str = "",
|
|
503
|
+
task_id: str = "",
|
|
504
|
+
session_id: str = "",
|
|
505
|
+
debt_types: str = "",
|
|
506
|
+
resolution: str = "",
|
|
507
|
+
debt_id: str = "",
|
|
508
|
+
) -> str:
|
|
509
|
+
"""Resolve protocol debt records by id or filters."""
|
|
510
|
+
return handle_protocol_debt_resolve(debt_ids, task_id, session_id, debt_types, resolution, debt_id)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
@mcp.tool
|
|
514
|
+
def nexo_card_match(
|
|
515
|
+
query: str,
|
|
516
|
+
limit: int = 5,
|
|
517
|
+
include_protocol: bool = True,
|
|
518
|
+
locale: str = "es",
|
|
519
|
+
category: str = "",
|
|
520
|
+
business_type: str = "",
|
|
521
|
+
) -> str:
|
|
522
|
+
"""Find official NEXO protocol cards for a user request."""
|
|
523
|
+
return handle_card_match(query, limit, include_protocol, locale, category, business_type)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
@mcp.tool
|
|
527
|
+
def nexo_skill_match(task: str, level: str = "") -> str:
|
|
528
|
+
"""Find reusable NEXO skills for the current task."""
|
|
529
|
+
return handle_skill_match(task, level)
|
|
530
|
+
|
|
531
|
+
|
|
442
532
|
@mcp.tool
|
|
443
533
|
def nexo_stop(sid: str) -> str:
|
|
444
534
|
"""Cleanly close a session. Removes it from active sessions immediately.
|
package/src/tools_reminders.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from db import get_reminders, get_followups
|
|
4
4
|
from datetime import date
|
|
5
|
+
from interactive_db import interactive_db_timeout, is_db_busy
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
def _is_due(date_str: str) -> bool:
|
|
@@ -22,21 +23,49 @@ def handle_reminders(filter_type: str = "due") -> str:
|
|
|
22
23
|
filter_type: 'due', 'all', 'followups', 'completed', 'deleted', 'history', 'any'
|
|
23
24
|
"""
|
|
24
25
|
parts = []
|
|
26
|
+
warnings: list[str] = []
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
with interactive_db_timeout():
|
|
29
|
+
if filter_type in ("due", "all", "completed", "deleted", "history", "any"):
|
|
30
|
+
r = _format_reminders_safe(filter_type, warnings)
|
|
31
|
+
if r:
|
|
32
|
+
parts.append(r)
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
34
|
+
if filter_type in ("due", "all", "followups", "completed", "deleted", "history", "any"):
|
|
35
|
+
f = _format_followups_safe(filter_type, warnings)
|
|
36
|
+
if f:
|
|
37
|
+
parts.append(f)
|
|
35
38
|
|
|
36
39
|
result = "\n\n".join(parts)
|
|
40
|
+
if warnings:
|
|
41
|
+
prefix = "REMINDERS DEGRADED:\n " + "\n ".join(warnings)
|
|
42
|
+
prefix += "\n Continue with the user request; reminders will catch up shortly."
|
|
43
|
+
return f"{prefix}\n\n{result}" if result else prefix
|
|
37
44
|
return result if result else "No pending reminders."
|
|
38
45
|
|
|
39
46
|
|
|
47
|
+
def _format_reminders_safe(filter_type: str, warnings: list[str]) -> str:
|
|
48
|
+
try:
|
|
49
|
+
return _format_reminders(filter_type)
|
|
50
|
+
except Exception as exc:
|
|
51
|
+
if is_db_busy(exc):
|
|
52
|
+
warnings.append("reminders skipped because the local brain database is busy")
|
|
53
|
+
else:
|
|
54
|
+
warnings.append(f"reminders skipped ({type(exc).__name__})")
|
|
55
|
+
return ""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _format_followups_safe(filter_type: str, warnings: list[str]) -> str:
|
|
59
|
+
try:
|
|
60
|
+
return _format_followups(filter_type)
|
|
61
|
+
except Exception as exc:
|
|
62
|
+
if is_db_busy(exc):
|
|
63
|
+
warnings.append("followups skipped because the local brain database is busy")
|
|
64
|
+
else:
|
|
65
|
+
warnings.append(f"followups skipped ({type(exc).__name__})")
|
|
66
|
+
return ""
|
|
67
|
+
|
|
68
|
+
|
|
40
69
|
def _format_reminders(filter_type: str) -> str:
|
|
41
70
|
"""Format reminders from database."""
|
|
42
71
|
rows = get_reminders(filter_type)
|
package/src/tools_sessions.py
CHANGED
|
@@ -425,17 +425,6 @@ def handle_startup(
|
|
|
425
425
|
sid = _generate_sid()
|
|
426
426
|
startup_warnings: list[str] = []
|
|
427
427
|
cleaned = _safe_interactive("stale-session cleanup", clean_stale_sessions, 0, startup_warnings)
|
|
428
|
-
if startup_warnings:
|
|
429
|
-
lines = [f"SID: {sid}"]
|
|
430
|
-
conversation = str(conversation_id or "").strip()
|
|
431
|
-
if conversation:
|
|
432
|
-
lines.append(f"CONVERSATION_ID: {conversation}")
|
|
433
|
-
lines.append("")
|
|
434
|
-
lines.append("STARTUP DEGRADED:")
|
|
435
|
-
for warning in startup_warnings[:4]:
|
|
436
|
-
lines.append(f" {warning}")
|
|
437
|
-
lines.append(" Continue responding; retry nexo_heartbeat shortly for full context.")
|
|
438
|
-
return "\n".join(lines)
|
|
439
428
|
linked_session_id = (session_token or claude_session_id or "").strip()
|
|
440
429
|
# v6.0.7 hotfix: when the caller did not pass an explicit UUID, fall back to
|
|
441
430
|
# the Claude Code SessionStart UUID written by the SessionStart hook to
|