nexo-brain 7.20.20 → 7.20.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/auto_update.py +44 -14
- package/src/cli.py +10 -4
- package/src/db_guard.py +7 -5
- package/src/interactive_db.py +59 -0
- package/src/local_context/api.py +3 -6
- package/src/local_context/db.py +318 -0
- package/src/local_context/logging.py +3 -4
- package/src/mcp_required_tools.py +31 -0
- package/src/plugins/episodic_memory.py +18 -0
- package/src/plugins/skills.py +14 -3
- package/src/plugins/update.py +29 -10
- package/src/scripts/nexo-backup.sh +131 -7
- package/src/server.py +95 -5
- package/src/tools_reminders.py +37 -8
- package/src/tools_sessions.py +121 -42
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.20.
|
|
3
|
+
"version": "7.20.22",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/auto_update.py
CHANGED
|
@@ -89,6 +89,7 @@ LOCAL_CONTEXT_BACKUP_TABLES = (
|
|
|
89
89
|
"local_index_checkpoints",
|
|
90
90
|
"local_index_state",
|
|
91
91
|
"local_index_errors",
|
|
92
|
+
"local_index_logs",
|
|
92
93
|
"local_assets",
|
|
93
94
|
"local_asset_versions",
|
|
94
95
|
"local_chunks",
|
|
@@ -98,7 +99,17 @@ LOCAL_CONTEXT_BACKUP_TABLES = (
|
|
|
98
99
|
"local_context_queries",
|
|
99
100
|
"local_index_dirs",
|
|
100
101
|
)
|
|
101
|
-
PROTECTED_BACKUP_TABLES = CRITICAL_BACKUP_TABLES
|
|
102
|
+
PROTECTED_BACKUP_TABLES = CRITICAL_BACKUP_TABLES
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _backup_validation_tables(db_file: Path) -> tuple[str, ...]:
|
|
106
|
+
if db_file.name == "nexo.db":
|
|
107
|
+
return PROTECTED_BACKUP_TABLES
|
|
108
|
+
if db_file.name == "local-context.db":
|
|
109
|
+
return LOCAL_CONTEXT_BACKUP_TABLES
|
|
110
|
+
return ()
|
|
111
|
+
|
|
112
|
+
|
|
102
113
|
CLASSIFIER_INSTALL_TIMEOUT_SECONDS = 1800
|
|
103
114
|
CLASSIFIER_INSTALL_JOIN_SECONDS = 1500
|
|
104
115
|
CLASSIFIER_INSTALL_LOG = paths.logs_dir() / "classifier-install.log"
|
|
@@ -338,7 +349,7 @@ def _validate_db_backup(source_db: Path, backup_db: Path) -> dict:
|
|
|
338
349
|
report["errors"].append(f"backup db missing: {backup_db}")
|
|
339
350
|
return report
|
|
340
351
|
|
|
341
|
-
for table in
|
|
352
|
+
for table in _backup_validation_tables(source_db):
|
|
342
353
|
source_count = _critical_table_count(source_db, table)
|
|
343
354
|
backup_count = _critical_table_count(backup_db, table)
|
|
344
355
|
report["source_counts"][table] = source_count
|
|
@@ -373,18 +384,29 @@ def _create_validated_db_backup() -> tuple[str | None, dict | None]:
|
|
|
373
384
|
if not backup_dir:
|
|
374
385
|
return None, None
|
|
375
386
|
|
|
376
|
-
|
|
377
|
-
|
|
387
|
+
source_dbs: list[Path] = []
|
|
388
|
+
primary_db = _find_primary_db_path()
|
|
389
|
+
if primary_db is not None:
|
|
390
|
+
source_dbs.append(primary_db)
|
|
391
|
+
local_context_db = paths.memory_dir() / "local-context.db"
|
|
392
|
+
if local_context_db.is_file():
|
|
393
|
+
source_dbs.append(local_context_db)
|
|
394
|
+
if not source_dbs:
|
|
378
395
|
return backup_dir, None
|
|
379
396
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
397
|
+
reports = []
|
|
398
|
+
ok = True
|
|
399
|
+
for source_db in source_dbs:
|
|
400
|
+
report = _validate_db_backup(source_db, Path(backup_dir) / source_db.name)
|
|
401
|
+
reports.append(report)
|
|
402
|
+
if not report["ok"]:
|
|
403
|
+
ok = False
|
|
404
|
+
details = ", ".join(
|
|
405
|
+
f"{item['table']} {item['source']}->{item['backup']}"
|
|
406
|
+
for item in report["regressions"]
|
|
407
|
+
) or "; ".join(report["errors"])
|
|
408
|
+
_log(f"DB backup validation failed ({source_db.name}): {details}")
|
|
409
|
+
return backup_dir, {"ok": ok, "reports": reports}
|
|
388
410
|
|
|
389
411
|
|
|
390
412
|
def _read_last_check() -> dict:
|
|
@@ -2047,6 +2069,9 @@ def _backup_dbs() -> str | None:
|
|
|
2047
2069
|
backup_dir = paths.backups_dir() / f"pre-autoupdate-{timestamp}"
|
|
2048
2070
|
|
|
2049
2071
|
db_files = list(DATA_DIR.glob("*.db")) if DATA_DIR.is_dir() else []
|
|
2072
|
+
local_context_db = paths.memory_dir() / "local-context.db"
|
|
2073
|
+
if local_context_db.is_file():
|
|
2074
|
+
db_files.append(local_context_db)
|
|
2050
2075
|
db_files += [f for f in NEXO_HOME.glob("*.db") if f.is_file()]
|
|
2051
2076
|
src_db = SRC_DIR / "nexo.db"
|
|
2052
2077
|
if src_db.is_file() and src_db not in db_files:
|
|
@@ -2090,11 +2115,16 @@ def _restore_dbs(backup_dir: str):
|
|
|
2090
2115
|
if not bdir.is_dir():
|
|
2091
2116
|
return
|
|
2092
2117
|
for db_backup in bdir.glob("*.db"):
|
|
2093
|
-
|
|
2094
|
-
|
|
2118
|
+
if db_backup.name == "local-context.db":
|
|
2119
|
+
candidates = [paths.memory_dir() / db_backup.name]
|
|
2120
|
+
else:
|
|
2121
|
+
candidates = [DATA_DIR / db_backup.name, NEXO_HOME / db_backup.name, SRC_DIR / db_backup.name]
|
|
2122
|
+
for candidate in candidates:
|
|
2123
|
+
if candidate.is_file() or db_backup.name == "local-context.db":
|
|
2095
2124
|
src_conn = None
|
|
2096
2125
|
dst_conn = None
|
|
2097
2126
|
try:
|
|
2127
|
+
candidate.parent.mkdir(parents=True, exist_ok=True)
|
|
2098
2128
|
src_conn = sqlite3.connect(str(db_backup))
|
|
2099
2129
|
dst_conn = sqlite3.connect(str(candidate))
|
|
2100
2130
|
src_conn.backup(dst_conn)
|
package/src/cli.py
CHANGED
|
@@ -52,6 +52,7 @@ from pathlib import Path
|
|
|
52
52
|
|
|
53
53
|
from runtime_home import export_resolved_nexo_home
|
|
54
54
|
from runtime_versioning import build_mcp_status, clear_restart_required_marker
|
|
55
|
+
from mcp_required_tools import BOOTSTRAP_REQUIRED_MCP_TOOLS, missing_required_tools
|
|
55
56
|
|
|
56
57
|
NEXO_HOME = export_resolved_nexo_home()
|
|
57
58
|
NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parent)))
|
|
@@ -200,8 +201,11 @@ def _mcp_probe(args) -> int:
|
|
|
200
201
|
server_path = NEXO_CODE / "server.py"
|
|
201
202
|
env = os.environ.copy()
|
|
202
203
|
env["NEXO_MCP_PROBE"] = "1"
|
|
203
|
-
env
|
|
204
|
-
|
|
204
|
+
# Probe must be deterministic: inherited env from an older Desktop/runtime
|
|
205
|
+
# must not force plugin loading or startup preflight and turn this into a
|
|
206
|
+
# slow/false-negative health check.
|
|
207
|
+
env["NEXO_MCP_PLUGIN_MODE"] = getattr(args, "plugin_mode", None) or "none"
|
|
208
|
+
env["NEXO_MCP_RUN_STARTUP_PREFLIGHT"] = "0"
|
|
205
209
|
client = str(getattr(args, "client", "") or "").strip()
|
|
206
210
|
if client:
|
|
207
211
|
env["NEXO_MCP_CLIENT"] = client
|
|
@@ -256,8 +260,8 @@ def _mcp_probe(args) -> int:
|
|
|
256
260
|
for tool in tools
|
|
257
261
|
if isinstance(tool, dict) and tool.get("name")
|
|
258
262
|
]
|
|
259
|
-
required =
|
|
260
|
-
missing =
|
|
263
|
+
required = list(BOOTSTRAP_REQUIRED_MCP_TOOLS)
|
|
264
|
+
missing = missing_required_tools(tool_names)
|
|
261
265
|
ok = not missing and len(tool_names) > 0
|
|
262
266
|
payload = {
|
|
263
267
|
"ok": ok,
|
|
@@ -265,6 +269,8 @@ def _mcp_probe(args) -> int:
|
|
|
265
269
|
"probe_ok": ok,
|
|
266
270
|
"tools_available": len(tool_names) > 0,
|
|
267
271
|
"tool_count": len(tool_names),
|
|
272
|
+
"required_tools": required,
|
|
273
|
+
"required_tool_count": len(required),
|
|
268
274
|
"required_tools_present": not missing,
|
|
269
275
|
"missing_required_tools": missing,
|
|
270
276
|
"client": client,
|
package/src/db_guard.py
CHANGED
|
@@ -75,6 +75,7 @@ LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
|
75
75
|
"local_index_checkpoints",
|
|
76
76
|
"local_index_state",
|
|
77
77
|
"local_index_errors",
|
|
78
|
+
"local_index_logs",
|
|
78
79
|
"local_assets",
|
|
79
80
|
"local_asset_versions",
|
|
80
81
|
"local_chunks",
|
|
@@ -85,10 +86,11 @@ LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
|
85
86
|
"local_index_dirs",
|
|
86
87
|
)
|
|
87
88
|
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
|
|
89
|
+
# Tables protected inside the operational Brain DB. Local memory now lives in
|
|
90
|
+
# runtime/memory/local-context.db and is validated with LOCAL_CONTEXT_TABLES
|
|
91
|
+
# separately; mixing both here makes old local-index backups look like the
|
|
92
|
+
# source of truth for nexo.db and can block every update.
|
|
93
|
+
PROTECTED_TABLES: tuple[str, ...] = CRITICAL_TABLES
|
|
92
94
|
|
|
93
95
|
# A reference backup must contain at least this many rows (summed across
|
|
94
96
|
# CRITICAL_TABLES) before we will treat it as "proof the user has real data".
|
|
@@ -478,7 +480,7 @@ def _backup_drift_is_safe(source_count: int, backup_count: int) -> bool:
|
|
|
478
480
|
|
|
479
481
|
|
|
480
482
|
def _quote_identifier(identifier: str) -> str:
|
|
481
|
-
if identifier not in PROTECTED_TABLES:
|
|
483
|
+
if identifier not in PROTECTED_TABLES and identifier not in LOCAL_CONTEXT_TABLES:
|
|
482
484
|
raise ValueError(f"refusing unsafe table identifier: {identifier!r}")
|
|
483
485
|
return '"' + identifier.replace('"', '""') + '"'
|
|
484
486
|
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Helpers for interactive MCP tools that must not block on SQLite locks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sqlite3
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
|
|
9
|
+
from db import get_db
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def interactive_db_timeout_ms() -> int:
|
|
13
|
+
"""Return the max SQLite busy wait for user-facing MCP calls."""
|
|
14
|
+
try:
|
|
15
|
+
return max(50, min(int(os.environ.get("NEXO_MCP_DB_BUSY_TIMEOUT_MS", "250")), 10000))
|
|
16
|
+
except Exception:
|
|
17
|
+
return 250
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def set_interactive_db_timeout() -> None:
|
|
21
|
+
"""Make the shared connection fail fast when a background writer owns the DB."""
|
|
22
|
+
try:
|
|
23
|
+
get_db().execute(f"PRAGMA busy_timeout={interactive_db_timeout_ms()}")
|
|
24
|
+
except Exception:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@contextmanager
|
|
29
|
+
def interactive_db_timeout():
|
|
30
|
+
"""Temporarily reduce SQLite busy wait for a user-facing MCP call."""
|
|
31
|
+
conn = None
|
|
32
|
+
previous = None
|
|
33
|
+
try:
|
|
34
|
+
conn = get_db()
|
|
35
|
+
row = conn.execute("PRAGMA busy_timeout").fetchone()
|
|
36
|
+
previous = int(row[0]) if row and row[0] is not None else None
|
|
37
|
+
conn.execute(f"PRAGMA busy_timeout={interactive_db_timeout_ms()}")
|
|
38
|
+
except Exception:
|
|
39
|
+
conn = None
|
|
40
|
+
try:
|
|
41
|
+
yield
|
|
42
|
+
finally:
|
|
43
|
+
if conn is not None and previous is not None:
|
|
44
|
+
try:
|
|
45
|
+
conn.execute(f"PRAGMA busy_timeout={previous}")
|
|
46
|
+
except Exception:
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def is_db_busy(exc: BaseException) -> bool:
|
|
51
|
+
msg = str(exc).lower()
|
|
52
|
+
return (
|
|
53
|
+
isinstance(exc, sqlite3.OperationalError)
|
|
54
|
+
and (
|
|
55
|
+
"database is locked" in msg
|
|
56
|
+
or "database is busy" in msg
|
|
57
|
+
or "database table is locked" in msg
|
|
58
|
+
)
|
|
59
|
+
) or "database is locked" in msg or "database is busy" in msg
|
package/src/local_context/api.py
CHANGED
|
@@ -11,10 +11,8 @@ import sys
|
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
from typing import Any
|
|
13
13
|
|
|
14
|
-
from db import get_db, init_db
|
|
15
|
-
from db._schema import run_migrations
|
|
16
|
-
|
|
17
14
|
from . import embeddings
|
|
15
|
+
from .db import ensure_local_context_db, get_local_context_db
|
|
18
16
|
from .extractors import chunk_text, contains_secret, entities, extract_text, summarize
|
|
19
17
|
from .logging import log_event, tail
|
|
20
18
|
from .privacy import classify_path, is_local_email_tree, is_queryable_path, should_extract, should_skip_file, should_skip_tree
|
|
@@ -88,13 +86,12 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
88
86
|
|
|
89
87
|
|
|
90
88
|
def ensure_ready() -> None:
|
|
91
|
-
|
|
92
|
-
run_migrations()
|
|
89
|
+
ensure_local_context_db()
|
|
93
90
|
|
|
94
91
|
|
|
95
92
|
def _conn():
|
|
96
93
|
ensure_ready()
|
|
97
|
-
return
|
|
94
|
+
return get_local_context_db()
|
|
98
95
|
|
|
99
96
|
|
|
100
97
|
def add_root(path: str, *, mode: str = "normal", depth: int | None = None) -> dict:
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import sqlite3
|
|
5
|
+
import time
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Iterable
|
|
8
|
+
|
|
9
|
+
import paths
|
|
10
|
+
from db._schema import _m63_local_context_layer, _m64_local_context_live_dirs
|
|
11
|
+
|
|
12
|
+
LOCAL_CONTEXT_DB_NAME = "local-context.db"
|
|
13
|
+
MIGRATION_STATE_KEY = "local_context_db_migrated_from_main"
|
|
14
|
+
MIGRATION_SKIPPED_KEY = "local_context_db_migration_skipped"
|
|
15
|
+
MAIN_CLEANUP_STATE_KEY = "local_context_main_tables_drained"
|
|
16
|
+
|
|
17
|
+
LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
18
|
+
"local_index_roots",
|
|
19
|
+
"local_index_exclusions",
|
|
20
|
+
"local_index_jobs",
|
|
21
|
+
"local_index_checkpoints",
|
|
22
|
+
"local_index_state",
|
|
23
|
+
"local_index_errors",
|
|
24
|
+
"local_index_logs",
|
|
25
|
+
"local_assets",
|
|
26
|
+
"local_asset_versions",
|
|
27
|
+
"local_chunks",
|
|
28
|
+
"local_entities",
|
|
29
|
+
"local_relations",
|
|
30
|
+
"local_embeddings",
|
|
31
|
+
"local_context_queries",
|
|
32
|
+
"local_index_dirs",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
_CONN: sqlite3.Connection | None = None
|
|
36
|
+
_CONN_PATH: Path | None = None
|
|
37
|
+
_READY = False
|
|
38
|
+
_LAST_MIGRATION_ATTEMPT = 0.0
|
|
39
|
+
_MIGRATION_RETRY_INTERVAL_SECONDS = 300.0
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def local_context_db_path() -> Path:
|
|
43
|
+
override = os.environ.get("NEXO_LOCAL_CONTEXT_DB", "").strip()
|
|
44
|
+
if override:
|
|
45
|
+
return Path(override).expanduser()
|
|
46
|
+
test_db = os.environ.get("NEXO_TEST_DB", "").strip()
|
|
47
|
+
if test_db:
|
|
48
|
+
return Path(test_db).expanduser().with_name("test_local_context.db")
|
|
49
|
+
return paths.memory_dir() / LOCAL_CONTEXT_DB_NAME
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _main_db_path_for_migration() -> Path:
|
|
53
|
+
override = os.environ.get("NEXO_LOCAL_CONTEXT_MAIN_DB", "").strip()
|
|
54
|
+
if override:
|
|
55
|
+
return Path(override).expanduser()
|
|
56
|
+
test_db = os.environ.get("NEXO_TEST_DB", "").strip()
|
|
57
|
+
if test_db:
|
|
58
|
+
return Path(test_db).expanduser()
|
|
59
|
+
return paths.db_path()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _busy_timeout_ms() -> int:
|
|
63
|
+
raw = os.environ.get("NEXO_LOCAL_CONTEXT_DB_BUSY_TIMEOUT_MS", "15000")
|
|
64
|
+
try:
|
|
65
|
+
value = int(raw)
|
|
66
|
+
except Exception:
|
|
67
|
+
value = 15000
|
|
68
|
+
return max(1000, min(value, 60000))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _connect(db_path: Path) -> sqlite3.Connection:
|
|
72
|
+
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
73
|
+
conn = sqlite3.connect(str(db_path), timeout=max(_busy_timeout_ms() / 1000.0, 1.0), check_same_thread=False)
|
|
74
|
+
conn.row_factory = sqlite3.Row
|
|
75
|
+
conn.execute(f"PRAGMA busy_timeout={_busy_timeout_ms()}")
|
|
76
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
77
|
+
conn.execute("PRAGMA synchronous=NORMAL")
|
|
78
|
+
conn.execute("PRAGMA temp_store=MEMORY")
|
|
79
|
+
return conn
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
|
83
|
+
_m63_local_context_layer(conn)
|
|
84
|
+
_m64_local_context_live_dirs(conn)
|
|
85
|
+
conn.execute("PRAGMA user_version=64")
|
|
86
|
+
conn.commit()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _table_exists(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> bool:
|
|
90
|
+
row = conn.execute(
|
|
91
|
+
f"SELECT 1 FROM {schema}.sqlite_master WHERE type='table' AND name=? LIMIT 1",
|
|
92
|
+
(table,),
|
|
93
|
+
).fetchone()
|
|
94
|
+
return bool(row)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _table_count(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> int:
|
|
98
|
+
if not _table_exists(conn, table, schema=schema):
|
|
99
|
+
return 0
|
|
100
|
+
row = conn.execute(f"SELECT COUNT(*) AS total FROM {schema}.{_quoted(table)}").fetchone()
|
|
101
|
+
return int(row["total"] or 0)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _state(conn: sqlite3.Connection, key: str) -> str:
|
|
105
|
+
row = conn.execute("SELECT value FROM local_index_state WHERE key=?", (key,)).fetchone()
|
|
106
|
+
return str(row["value"] or "") if row else ""
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _set_state(conn: sqlite3.Connection, key: str, value: str) -> None:
|
|
110
|
+
conn.execute(
|
|
111
|
+
"""
|
|
112
|
+
INSERT INTO local_index_state(key, value, updated_at)
|
|
113
|
+
VALUES (?, ?, strftime('%s','now'))
|
|
114
|
+
ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at
|
|
115
|
+
""",
|
|
116
|
+
(key, value),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _quoted(name: str) -> str:
|
|
121
|
+
return '"' + name.replace('"', '""') + '"'
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _table_columns(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> list[str]:
|
|
125
|
+
try:
|
|
126
|
+
rows = conn.execute(f"PRAGMA {schema}.table_info({_quoted(table)})").fetchall()
|
|
127
|
+
except sqlite3.OperationalError:
|
|
128
|
+
return []
|
|
129
|
+
return [str(row["name"]) for row in rows if row["name"]]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _primary_key_columns(conn: sqlite3.Connection, table: str, *, schema: str = "main") -> list[str]:
|
|
133
|
+
try:
|
|
134
|
+
rows = conn.execute(f"PRAGMA {schema}.table_info({_quoted(table)})").fetchall()
|
|
135
|
+
except sqlite3.OperationalError:
|
|
136
|
+
return []
|
|
137
|
+
ordered = sorted(
|
|
138
|
+
(
|
|
139
|
+
(int(row["pk"] or 0), str(row["name"]))
|
|
140
|
+
for row in rows
|
|
141
|
+
if int(row["pk"] or 0) > 0 and row["name"]
|
|
142
|
+
),
|
|
143
|
+
key=lambda item: item[0],
|
|
144
|
+
)
|
|
145
|
+
return [name for _order, name in ordered]
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _source_rows_missing_in_target(conn: sqlite3.Connection, table: str) -> int:
|
|
149
|
+
target_pk = _primary_key_columns(conn, table)
|
|
150
|
+
source_columns = set(_table_columns(conn, table, schema="source"))
|
|
151
|
+
pk_columns = [column for column in target_pk if column in source_columns]
|
|
152
|
+
if not pk_columns:
|
|
153
|
+
raise RuntimeError(f"cannot verify local context migration for {table}: missing primary key")
|
|
154
|
+
join_sql = " AND ".join(f"s.{_quoted(column)} = t.{_quoted(column)}" for column in pk_columns)
|
|
155
|
+
null_check = f"t.{_quoted(pk_columns[0])} IS NULL"
|
|
156
|
+
row = conn.execute(
|
|
157
|
+
f"SELECT COUNT(*) AS total "
|
|
158
|
+
f"FROM source.{_quoted(table)} s "
|
|
159
|
+
f"LEFT JOIN {_quoted(table)} t ON {join_sql} "
|
|
160
|
+
f"WHERE {null_check}"
|
|
161
|
+
).fetchone()
|
|
162
|
+
return int(row["total"] or 0)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _copy_local_tables(conn: sqlite3.Connection, tables: Iterable[str]) -> dict[str, int]:
|
|
166
|
+
copied: dict[str, int] = {}
|
|
167
|
+
for table in tables:
|
|
168
|
+
if not _table_exists(conn, table, schema="source"):
|
|
169
|
+
continue
|
|
170
|
+
target_columns = _table_columns(conn, table)
|
|
171
|
+
source_columns = set(_table_columns(conn, table, schema="source"))
|
|
172
|
+
columns = [column for column in target_columns if column in source_columns]
|
|
173
|
+
if not columns:
|
|
174
|
+
continue
|
|
175
|
+
before = _table_count(conn, table)
|
|
176
|
+
column_sql = ", ".join(_quoted(column) for column in columns)
|
|
177
|
+
conn.execute(
|
|
178
|
+
f"INSERT OR IGNORE INTO {_quoted(table)} ({column_sql}) "
|
|
179
|
+
f"SELECT {column_sql} FROM source.{_quoted(table)}"
|
|
180
|
+
)
|
|
181
|
+
after = _table_count(conn, table)
|
|
182
|
+
copied[table] = max(0, after - before)
|
|
183
|
+
return copied
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _source_table_counts(conn: sqlite3.Connection, tables: Iterable[str]) -> dict[str, int]:
|
|
187
|
+
counts: dict[str, int] = {}
|
|
188
|
+
for table in tables:
|
|
189
|
+
if _table_exists(conn, table, schema="source"):
|
|
190
|
+
counts[table] = _table_count(conn, table, schema="source")
|
|
191
|
+
return counts
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _drain_source_local_tables_if_verified(
|
|
195
|
+
conn: sqlite3.Connection,
|
|
196
|
+
source_counts: dict[str, int],
|
|
197
|
+
) -> dict[str, int]:
|
|
198
|
+
"""Clear old local-memory rows from main DB after verifying the copy.
|
|
199
|
+
|
|
200
|
+
We keep the table schema in ``nexo.db`` for backward compatibility with
|
|
201
|
+
older binaries, but empty the rows after the new DB has at least the same
|
|
202
|
+
number of records per table. This prevents two sources of truth and stops
|
|
203
|
+
the main DB from carrying the huge local-memory payload.
|
|
204
|
+
"""
|
|
205
|
+
drained: dict[str, int] = {}
|
|
206
|
+
if not source_counts:
|
|
207
|
+
return drained
|
|
208
|
+
|
|
209
|
+
for table, source_count in source_counts.items():
|
|
210
|
+
local_count = _table_count(conn, table)
|
|
211
|
+
missing = _source_rows_missing_in_target(conn, table)
|
|
212
|
+
if local_count < source_count or missing:
|
|
213
|
+
raise RuntimeError(
|
|
214
|
+
f"local context migration verification failed for {table}: "
|
|
215
|
+
f"local={local_count} source={source_count} missing={missing}"
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
for table, source_count in source_counts.items():
|
|
219
|
+
if source_count <= 0:
|
|
220
|
+
continue
|
|
221
|
+
conn.execute(f"DELETE FROM source.{_quoted(table)}")
|
|
222
|
+
drained[table] = source_count
|
|
223
|
+
|
|
224
|
+
if drained:
|
|
225
|
+
_set_state(conn, MAIN_CLEANUP_STATE_KEY, ",".join(sorted(drained)))
|
|
226
|
+
return drained
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def migrate_from_main_if_needed(conn: sqlite3.Connection) -> dict:
|
|
230
|
+
if os.environ.get("NEXO_LOCAL_CONTEXT_DISABLE_MAIN_MIGRATION", "").strip().lower() in {"1", "true", "yes"}:
|
|
231
|
+
return {"ok": True, "skipped": "disabled"}
|
|
232
|
+
|
|
233
|
+
main_db = _main_db_path_for_migration()
|
|
234
|
+
if not main_db.is_file():
|
|
235
|
+
_set_state(conn, MIGRATION_SKIPPED_KEY, "main_db_missing")
|
|
236
|
+
conn.commit()
|
|
237
|
+
return {"ok": True, "skipped": "main_db_missing"}
|
|
238
|
+
|
|
239
|
+
source = str(main_db).replace("'", "''")
|
|
240
|
+
try:
|
|
241
|
+
conn.execute("PRAGMA busy_timeout=1000")
|
|
242
|
+
conn.execute(f"ATTACH DATABASE '{source}' AS source")
|
|
243
|
+
if not _table_exists(conn, "local_assets", schema="source") and not _table_exists(conn, "local_index_roots", schema="source"):
|
|
244
|
+
_set_state(conn, MIGRATION_SKIPPED_KEY, "main_local_tables_missing")
|
|
245
|
+
if _table_count(conn, "local_assets") > 0 or _table_count(conn, "local_index_jobs") > 0:
|
|
246
|
+
_set_state(conn, MIGRATION_STATE_KEY, "already_has_local_data")
|
|
247
|
+
conn.commit()
|
|
248
|
+
return {"ok": True, "skipped": "main_local_tables_missing"}
|
|
249
|
+
source_counts = _source_table_counts(conn, LOCAL_CONTEXT_TABLES)
|
|
250
|
+
if not any(source_counts.values()):
|
|
251
|
+
_set_state(conn, MIGRATION_STATE_KEY, "main_local_tables_empty")
|
|
252
|
+
_set_state(conn, MAIN_CLEANUP_STATE_KEY, "empty")
|
|
253
|
+
conn.commit()
|
|
254
|
+
return {"ok": True, "skipped": "main_local_tables_empty"}
|
|
255
|
+
copied = _copy_local_tables(conn, LOCAL_CONTEXT_TABLES)
|
|
256
|
+
drained = _drain_source_local_tables_if_verified(conn, source_counts)
|
|
257
|
+
_set_state(conn, MIGRATION_STATE_KEY, str(main_db))
|
|
258
|
+
_set_state(conn, "local_context_db_migrated_rows", str(sum(copied.values())))
|
|
259
|
+
if drained:
|
|
260
|
+
_set_state(conn, "local_context_main_tables_drained_rows", str(sum(drained.values())))
|
|
261
|
+
conn.commit()
|
|
262
|
+
return {"ok": True, "migrated_from": str(main_db), "copied": copied, "drained": drained}
|
|
263
|
+
except sqlite3.OperationalError as exc:
|
|
264
|
+
message = str(exc)
|
|
265
|
+
_set_state(conn, MIGRATION_SKIPPED_KEY, message[:240])
|
|
266
|
+
_set_state(conn, "local_context_db_drain_pending", "main_db_busy_or_unavailable")
|
|
267
|
+
conn.commit()
|
|
268
|
+
return {"ok": True, "skipped": "main_db_busy_or_unavailable", "error": message, "retry_pending": True}
|
|
269
|
+
finally:
|
|
270
|
+
try:
|
|
271
|
+
conn.execute("DETACH DATABASE source")
|
|
272
|
+
except Exception:
|
|
273
|
+
pass
|
|
274
|
+
conn.execute(f"PRAGMA busy_timeout={_busy_timeout_ms()}")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def ensure_local_context_db() -> None:
|
|
278
|
+
global _CONN, _CONN_PATH, _READY, _LAST_MIGRATION_ATTEMPT
|
|
279
|
+
db_path = local_context_db_path()
|
|
280
|
+
if _CONN is not None and _CONN_PATH != db_path:
|
|
281
|
+
close_local_context_db()
|
|
282
|
+
if _CONN is None:
|
|
283
|
+
_CONN = _connect(db_path)
|
|
284
|
+
_CONN_PATH = db_path
|
|
285
|
+
now = time.monotonic()
|
|
286
|
+
if _READY:
|
|
287
|
+
if now - _LAST_MIGRATION_ATTEMPT >= _MIGRATION_RETRY_INTERVAL_SECONDS:
|
|
288
|
+
_LAST_MIGRATION_ATTEMPT = now
|
|
289
|
+
try:
|
|
290
|
+
migrate_from_main_if_needed(_CONN)
|
|
291
|
+
except Exception:
|
|
292
|
+
# Sidecar data is still usable; cleanup from the old main DB is
|
|
293
|
+
# best-effort and will retry on later service cycles.
|
|
294
|
+
pass
|
|
295
|
+
return
|
|
296
|
+
_ensure_schema(_CONN)
|
|
297
|
+
_LAST_MIGRATION_ATTEMPT = now
|
|
298
|
+
migration = migrate_from_main_if_needed(_CONN)
|
|
299
|
+
_READY = True
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def get_local_context_db() -> sqlite3.Connection:
|
|
303
|
+
ensure_local_context_db()
|
|
304
|
+
assert _CONN is not None
|
|
305
|
+
return _CONN
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def close_local_context_db() -> None:
|
|
309
|
+
global _CONN, _CONN_PATH, _READY, _LAST_MIGRATION_ATTEMPT
|
|
310
|
+
if _CONN is not None:
|
|
311
|
+
try:
|
|
312
|
+
_CONN.close()
|
|
313
|
+
except Exception:
|
|
314
|
+
pass
|
|
315
|
+
_CONN = None
|
|
316
|
+
_CONN_PATH = None
|
|
317
|
+
_READY = False
|
|
318
|
+
_LAST_MIGRATION_ATTEMPT = 0.0
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from db import
|
|
4
|
-
|
|
3
|
+
from .db import get_local_context_db
|
|
5
4
|
from .util import json_dumps, now
|
|
6
5
|
|
|
7
6
|
|
|
8
7
|
def log_event(level: str, event: str, message: str, **metadata) -> None:
|
|
9
|
-
conn =
|
|
8
|
+
conn = get_local_context_db()
|
|
10
9
|
conn.execute(
|
|
11
10
|
"""
|
|
12
11
|
INSERT INTO local_index_logs(created_at, level, event, message, metadata_json)
|
|
@@ -18,7 +17,7 @@ def log_event(level: str, event: str, message: str, **metadata) -> None:
|
|
|
18
17
|
|
|
19
18
|
|
|
20
19
|
def tail(limit: int = 100) -> list[dict]:
|
|
21
|
-
conn =
|
|
20
|
+
conn = get_local_context_db()
|
|
22
21
|
rows = conn.execute(
|
|
23
22
|
"""
|
|
24
23
|
SELECT created_at, level, event, message, metadata_json
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
"""Required MCP tool contract shared by Brain and Desktop probes.
|
|
4
|
+
|
|
5
|
+
These tools are the minimum bootstrap surface NEXO Desktop needs before a
|
|
6
|
+
conversation can be considered healthy. Dynamic plugin loading may still add
|
|
7
|
+
more tools, but these names must always be present.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
BOOTSTRAP_REQUIRED_MCP_TOOLS: tuple[str, ...] = (
|
|
11
|
+
"nexo_startup",
|
|
12
|
+
"nexo_heartbeat",
|
|
13
|
+
"nexo_session_diary_read",
|
|
14
|
+
"nexo_reminders",
|
|
15
|
+
"nexo_smart_startup",
|
|
16
|
+
"nexo_task_open",
|
|
17
|
+
"nexo_task_close",
|
|
18
|
+
"nexo_task_acknowledge_guard",
|
|
19
|
+
"nexo_guard_check",
|
|
20
|
+
"nexo_learning_add",
|
|
21
|
+
"nexo_confidence_check",
|
|
22
|
+
"nexo_followup_create",
|
|
23
|
+
"nexo_protocol_debt_resolve",
|
|
24
|
+
"nexo_card_match",
|
|
25
|
+
"nexo_skill_match",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def missing_required_tools(tool_names: list[str] | tuple[str, ...] | set[str]) -> list[str]:
|
|
30
|
+
available = {str(name) for name in tool_names}
|
|
31
|
+
return [name for name in BOOTSTRAP_REQUIRED_MCP_TOOLS if name not in available]
|