nexo-brain 7.20.22 → 7.20.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/auto_update.py +10 -10
- package/src/cli.py +12 -2
- package/src/db_guard.py +24 -8
- package/src/doctor/providers/boot.py +91 -2
- package/src/local_context/api.py +272 -76
- package/src/local_context/db.py +18 -0
- package/src/plugins/recover.py +7 -4
- package/src/plugins/update.py +8 -2
- package/src/server.py +2 -2
- package/src/tools_sessions.py +11 -8
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.20.
|
|
3
|
+
"version": "7.20.23",
|
|
4
4
|
"description": "Local cognitive runtime for Claude Code \u2014 persistent memory, overnight learning, doctor diagnostics, personal scripts, recovery-aware jobs, startup preflight, and optional dashboard/power helper.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "NEXO Brain",
|
package/README.md
CHANGED
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
|
|
19
19
|
[Watch the overview video](https://nexo-brain.com/watch/) · [Watch on YouTube](https://www.youtube.com/watch?v=i2lkGhKyVqI) · [Open the infographic](https://nexo-brain.com/assets/nexo-brain-infographic-v5.png)
|
|
20
20
|
|
|
21
|
-
Version `7.20.
|
|
21
|
+
Version `7.20.23` is the current packaged-runtime line. Patch release over v7.20.22 — Local Memory status reads the real split sidecar database read-only, reports retryable keyed failures without false zeroes, and keeps Desktop Spanish/English copy localized.
|
|
22
|
+
|
|
23
|
+
Previously in `7.20.22`: patch release over v7.20.19 — Local Memory moved out of the main Brain database, MCP readiness verifies required tools, and split-aware Desktop backups validate the main DB and Local Memory sidecar separately.
|
|
22
24
|
|
|
23
25
|
Previously in `7.20.18`: patch release over v7.20.17 — Desktop-managed setup now preserves a completed onboarding flag when Brain is later invoked with the non-interactive `--skip` bootstrap path.
|
|
24
26
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.20.
|
|
3
|
+
"version": "7.20.23",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/auto_update.py
CHANGED
|
@@ -99,15 +99,13 @@ LOCAL_CONTEXT_BACKUP_TABLES = (
|
|
|
99
99
|
"local_context_queries",
|
|
100
100
|
"local_index_dirs",
|
|
101
101
|
)
|
|
102
|
-
PROTECTED_BACKUP_TABLES = CRITICAL_BACKUP_TABLES
|
|
102
|
+
PROTECTED_BACKUP_TABLES = CRITICAL_BACKUP_TABLES + LOCAL_CONTEXT_BACKUP_TABLES
|
|
103
103
|
|
|
104
104
|
|
|
105
105
|
def _backup_validation_tables(db_file: Path) -> tuple[str, ...]:
|
|
106
|
-
if db_file.name == "nexo.db":
|
|
107
|
-
return PROTECTED_BACKUP_TABLES
|
|
108
106
|
if db_file.name == "local-context.db":
|
|
109
107
|
return LOCAL_CONTEXT_BACKUP_TABLES
|
|
110
|
-
return
|
|
108
|
+
return PROTECTED_BACKUP_TABLES
|
|
111
109
|
|
|
112
110
|
|
|
113
111
|
CLASSIFIER_INSTALL_TIMEOUT_SECONDS = 1800
|
|
@@ -1447,6 +1445,7 @@ def _self_heal_if_wiped() -> dict | None:
|
|
|
1447
1445
|
from db_guard import (
|
|
1448
1446
|
CRITICAL_TABLES,
|
|
1449
1447
|
HOURLY_BACKUP_MAX_AGE,
|
|
1448
|
+
LOCAL_CONTEXT_TABLES,
|
|
1450
1449
|
MIN_REFERENCE_ROWS,
|
|
1451
1450
|
PROTECTED_TABLES,
|
|
1452
1451
|
db_looks_wiped,
|
|
@@ -1467,14 +1466,15 @@ def _self_heal_if_wiped() -> dict | None:
|
|
|
1467
1466
|
return None
|
|
1468
1467
|
if not db_looks_wiped(primary, PROTECTED_TABLES):
|
|
1469
1468
|
return None
|
|
1469
|
+
recovery_tables = PROTECTED_TABLES + LOCAL_CONTEXT_TABLES
|
|
1470
1470
|
reference = find_best_hourly_backup(
|
|
1471
1471
|
paths.backups_dir(),
|
|
1472
1472
|
max_age_seconds=HOURLY_BACKUP_MAX_AGE,
|
|
1473
|
-
tables=
|
|
1473
|
+
tables=recovery_tables,
|
|
1474
1474
|
) or find_latest_hourly_backup(
|
|
1475
1475
|
paths.backups_dir(),
|
|
1476
1476
|
max_age_seconds=HOURLY_BACKUP_MAX_AGE,
|
|
1477
|
-
tables=
|
|
1477
|
+
tables=recovery_tables,
|
|
1478
1478
|
)
|
|
1479
1479
|
if reference is None:
|
|
1480
1480
|
_log("self-heal: nexo.db looks wiped but no usable hourly backup found — skipping.")
|
|
@@ -1483,7 +1483,7 @@ def _self_heal_if_wiped() -> dict | None:
|
|
|
1483
1483
|
"reason": "no_usable_hourly_backup",
|
|
1484
1484
|
"primary_db": str(primary),
|
|
1485
1485
|
}
|
|
1486
|
-
ref_counts = db_row_counts(reference,
|
|
1486
|
+
ref_counts = db_row_counts(reference, recovery_tables)
|
|
1487
1487
|
ref_total = sum(v for v in ref_counts.values() if isinstance(v, int))
|
|
1488
1488
|
if ref_total < MIN_REFERENCE_ROWS:
|
|
1489
1489
|
_log(f"self-heal: reference backup {reference.name} has {ref_total} rows, below floor {MIN_REFERENCE_ROWS}")
|
|
@@ -1572,7 +1572,7 @@ def _self_heal_if_wiped() -> dict | None:
|
|
|
1572
1572
|
"quiesce": quiesce_report,
|
|
1573
1573
|
"resume": resume_report,
|
|
1574
1574
|
}
|
|
1575
|
-
valid, valid_err = validate_backup_matches_source(reference, primary,
|
|
1575
|
+
valid, valid_err = validate_backup_matches_source(reference, primary, recovery_tables)
|
|
1576
1576
|
if not valid:
|
|
1577
1577
|
_log(f"self-heal: post-restore validation failed: {valid_err}")
|
|
1578
1578
|
resume_report = _resume_quiesced()
|
|
@@ -1586,7 +1586,7 @@ def _self_heal_if_wiped() -> dict | None:
|
|
|
1586
1586
|
"resume": resume_report,
|
|
1587
1587
|
}
|
|
1588
1588
|
|
|
1589
|
-
final_counts = db_row_counts(primary,
|
|
1589
|
+
final_counts = db_row_counts(primary, recovery_tables)
|
|
1590
1590
|
final_total = sum(v for v in final_counts.values() if isinstance(v, int))
|
|
1591
1591
|
_log(f"self-heal: restored {final_total} critical rows from {reference.name}.")
|
|
1592
1592
|
resume_report = _resume_quiesced()
|
|
@@ -3988,7 +3988,7 @@ def _auto_update_check_locked() -> dict:
|
|
|
3988
3988
|
|
|
3989
3989
|
# Backfill runtime CLI modules for existing installs
|
|
3990
3990
|
try:
|
|
3991
|
-
for fname in ("cli.py", "script_registry.py", "skills_runtime.py", "cron_recovery.py", "client_preferences.py", "claude_cli.py", "agent_runner.py", "bootstrap_docs.py"):
|
|
3991
|
+
for fname in ("cli.py", "script_registry.py", "skills_runtime.py", "cron_recovery.py", "client_preferences.py", "claude_cli.py", "agent_runner.py", "bootstrap_docs.py", "mcp_required_tools.py"):
|
|
3992
3992
|
src_file = SRC_DIR / fname
|
|
3993
3993
|
dest_file = NEXO_HOME / fname
|
|
3994
3994
|
if src_file.is_file() and (not dest_file.exists() or src_file.stat().st_mtime > dest_file.stat().st_mtime):
|
package/src/cli.py
CHANGED
|
@@ -52,7 +52,17 @@ from pathlib import Path
|
|
|
52
52
|
|
|
53
53
|
from runtime_home import export_resolved_nexo_home
|
|
54
54
|
from runtime_versioning import build_mcp_status, clear_restart_required_marker
|
|
55
|
-
|
|
55
|
+
try:
|
|
56
|
+
from mcp_required_tools import BOOTSTRAP_REQUIRED_MCP_TOOLS, missing_required_tools
|
|
57
|
+
except ModuleNotFoundError as exc:
|
|
58
|
+
if getattr(exc, "name", "") != "mcp_required_tools":
|
|
59
|
+
raise
|
|
60
|
+
# Older installed runtimes can be missing modules added after their CLI was
|
|
61
|
+
# copied. Keep `nexo update` bootable so the sync can repair the runtime.
|
|
62
|
+
BOOTSTRAP_REQUIRED_MCP_TOOLS: tuple[str, ...] = ()
|
|
63
|
+
|
|
64
|
+
def missing_required_tools(_tool_names):
|
|
65
|
+
return []
|
|
56
66
|
|
|
57
67
|
NEXO_HOME = export_resolved_nexo_home()
|
|
58
68
|
NEXO_CODE = Path(os.environ.get("NEXO_CODE", str(Path(__file__).resolve().parent)))
|
|
@@ -3389,7 +3399,7 @@ def main():
|
|
|
3389
3399
|
|
|
3390
3400
|
local_context_query_p = local_context_sub.add_parser("query", help="Query local memory evidence")
|
|
3391
3401
|
local_context_query_p.add_argument("query", help="Question or search phrase")
|
|
3392
|
-
local_context_query_p.add_argument("--intent", default="answer", help="Intent label
|
|
3402
|
+
local_context_query_p.add_argument("--intent", default="answer", help="Intent label included with the query result")
|
|
3393
3403
|
local_context_query_p.add_argument("--limit", type=int, default=12, help="Maximum evidence rows")
|
|
3394
3404
|
local_context_query_p.add_argument("--current-context", default="", help="Optional current conversation/task context")
|
|
3395
3405
|
local_context_query_p.add_argument("--mode", choices=["compact", "full"], default="compact", help="Payload shape. Compact is safe for chat clients; full is for debugging.")
|
package/src/db_guard.py
CHANGED
|
@@ -86,12 +86,17 @@ LOCAL_CONTEXT_TABLES: tuple[str, ...] = (
|
|
|
86
86
|
"local_index_dirs",
|
|
87
87
|
)
|
|
88
88
|
|
|
89
|
-
# Tables protected inside the operational Brain DB.
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
89
|
+
# Tables protected inside the operational Brain DB. Keep this core-only:
|
|
90
|
+
# callers that need local memory checks must request LOCAL_CONTEXT_TABLES or
|
|
91
|
+
# RECOVERY_TABLES explicitly so normal Brain wipe detection is not skewed by
|
|
92
|
+
# split local-memory databases.
|
|
93
93
|
PROTECTED_TABLES: tuple[str, ...] = CRITICAL_TABLES
|
|
94
94
|
|
|
95
|
+
# Recovery must still inspect legacy backups that carried local memory tables
|
|
96
|
+
# inside nexo.db. This wider set is safe for row counts and restore validation,
|
|
97
|
+
# but should not replace PROTECTED_TABLES in core wipe-diff callers.
|
|
98
|
+
RECOVERY_TABLES: tuple[str, ...] = CRITICAL_TABLES + LOCAL_CONTEXT_TABLES
|
|
99
|
+
|
|
95
100
|
# A reference backup must contain at least this many rows (summed across
|
|
96
101
|
# CRITICAL_TABLES) before we will treat it as "proof the user has real data".
|
|
97
102
|
# Otherwise we cannot distinguish a fresh install from a wipe.
|
|
@@ -211,7 +216,7 @@ def _table_count(conn: sqlite3.Connection, table: str) -> int | None:
|
|
|
211
216
|
return int(result[0]) if result is not None else 0
|
|
212
217
|
|
|
213
218
|
|
|
214
|
-
def db_row_counts(path: str | Path, tables: tuple[str, ...] =
|
|
219
|
+
def db_row_counts(path: str | Path, tables: tuple[str, ...] = RECOVERY_TABLES) -> dict[str, int | None]:
|
|
215
220
|
"""Return {table: count} for a SQLite DB. Missing DB / missing tables map to None."""
|
|
216
221
|
p = Path(path)
|
|
217
222
|
counts: dict[str, int | None] = {t: None for t in tables}
|
|
@@ -253,11 +258,22 @@ def db_looks_wiped(
|
|
|
253
258
|
size = p.stat().st_size
|
|
254
259
|
except OSError:
|
|
255
260
|
return False
|
|
261
|
+
counts = db_row_counts(p, tables)
|
|
256
262
|
if size <= EMPTY_DB_SIZE_BYTES:
|
|
257
263
|
# Small but not necessarily wiped — confirm via row counts.
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
264
|
+
return _counts_look_wiped(counts)
|
|
265
|
+
return _counts_look_wiped(counts)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _counts_look_wiped(counts: dict[str, int | None]) -> bool:
|
|
269
|
+
"""Treat Brain table loss as a wipe even if legacy local tables remain."""
|
|
270
|
+
critical_present = {
|
|
271
|
+
table: counts.get(table)
|
|
272
|
+
for table in CRITICAL_TABLES
|
|
273
|
+
if counts.get(table) is not None
|
|
274
|
+
}
|
|
275
|
+
if critical_present:
|
|
276
|
+
return _all_tables_empty_or_missing(critical_present)
|
|
261
277
|
return _all_tables_empty_or_missing(counts)
|
|
262
278
|
|
|
263
279
|
|
|
@@ -44,6 +44,7 @@ def check_db_integrity(fix: bool = False) -> DoctorCheck:
|
|
|
44
44
|
from db_guard import (
|
|
45
45
|
CRITICAL_TABLES,
|
|
46
46
|
EMPTY_DB_SIZE_BYTES,
|
|
47
|
+
LOCAL_CONTEXT_TABLES,
|
|
47
48
|
MIN_REFERENCE_ROWS,
|
|
48
49
|
PROTECTED_TABLES,
|
|
49
50
|
db_looks_wiped,
|
|
@@ -98,6 +99,47 @@ def check_db_integrity(fix: bool = False) -> DoctorCheck:
|
|
|
98
99
|
reference_counts = db_row_counts(reference, PROTECTED_TABLES) if reference else {}
|
|
99
100
|
reference_rows = sum(v for v in reference_counts.values() if isinstance(v, int))
|
|
100
101
|
protected_regression = diff_row_counts(db_path, reference, PROTECTED_TABLES) if reference else None
|
|
102
|
+
local_context_db = paths.memory_dir() / "local-context.db"
|
|
103
|
+
local_needs_reference = (
|
|
104
|
+
not local_context_db.is_file()
|
|
105
|
+
or db_looks_wiped(local_context_db, LOCAL_CONTEXT_TABLES)
|
|
106
|
+
)
|
|
107
|
+
local_reference = None
|
|
108
|
+
if local_needs_reference:
|
|
109
|
+
local_reference = (
|
|
110
|
+
find_best_hourly_backup(
|
|
111
|
+
paths.backups_dir(),
|
|
112
|
+
glob="local-context-*.db",
|
|
113
|
+
min_critical_rows=MIN_REFERENCE_ROWS,
|
|
114
|
+
tables=LOCAL_CONTEXT_TABLES,
|
|
115
|
+
)
|
|
116
|
+
or find_latest_hourly_backup(
|
|
117
|
+
paths.backups_dir(),
|
|
118
|
+
glob="local-context-*.db",
|
|
119
|
+
min_critical_rows=MIN_REFERENCE_ROWS,
|
|
120
|
+
tables=LOCAL_CONTEXT_TABLES,
|
|
121
|
+
)
|
|
122
|
+
or find_best_hourly_backup(
|
|
123
|
+
paths.backups_dir(),
|
|
124
|
+
glob="nexo-*.db",
|
|
125
|
+
min_critical_rows=MIN_REFERENCE_ROWS,
|
|
126
|
+
tables=LOCAL_CONTEXT_TABLES,
|
|
127
|
+
)
|
|
128
|
+
or find_latest_hourly_backup(
|
|
129
|
+
paths.backups_dir(),
|
|
130
|
+
glob="nexo-*.db",
|
|
131
|
+
min_critical_rows=MIN_REFERENCE_ROWS,
|
|
132
|
+
tables=LOCAL_CONTEXT_TABLES,
|
|
133
|
+
)
|
|
134
|
+
)
|
|
135
|
+
local_reference_counts = db_row_counts(local_reference, LOCAL_CONTEXT_TABLES) if local_reference else {}
|
|
136
|
+
local_reference_rows = sum(v for v in local_reference_counts.values() if isinstance(v, int))
|
|
137
|
+
local_regression = diff_row_counts(local_context_db, local_reference, LOCAL_CONTEXT_TABLES) if local_reference else None
|
|
138
|
+
recoverable_local_regression = bool(
|
|
139
|
+
local_reference
|
|
140
|
+
and local_regression
|
|
141
|
+
and local_regression.is_wipe()
|
|
142
|
+
)
|
|
101
143
|
recoverable_regression = bool(
|
|
102
144
|
reference
|
|
103
145
|
and protected_regression
|
|
@@ -121,7 +163,7 @@ def check_db_integrity(fix: bool = False) -> DoctorCheck:
|
|
|
121
163
|
)
|
|
122
164
|
)
|
|
123
165
|
|
|
124
|
-
if quick_ok and not recoverable_wipe and not recoverable_regression:
|
|
166
|
+
if quick_ok and not recoverable_wipe and not recoverable_regression and not recoverable_local_regression:
|
|
125
167
|
if looks_wiped and not reference:
|
|
126
168
|
return DoctorCheck(
|
|
127
169
|
id="boot.db_integrity",
|
|
@@ -150,6 +192,11 @@ def check_db_integrity(fix: bool = False) -> DoctorCheck:
|
|
|
150
192
|
evidence.append(f"Reference backup: {reference} ({reference_rows} protected rows)")
|
|
151
193
|
if protected_regression:
|
|
152
194
|
evidence.extend(protected_regression.summary_lines())
|
|
195
|
+
if local_reference:
|
|
196
|
+
evidence.append(f"Local memory DB: {local_context_db}")
|
|
197
|
+
evidence.append(f"Local memory reference: {local_reference} ({local_reference_rows} rows)")
|
|
198
|
+
if local_regression:
|
|
199
|
+
evidence.extend(["Local memory regression:", *local_regression.summary_lines()])
|
|
153
200
|
|
|
154
201
|
if fix and recoverable_regression:
|
|
155
202
|
report = restore_tables_from_backup(reference, db_path)
|
|
@@ -205,7 +252,49 @@ def check_db_integrity(fix: bool = False) -> DoctorCheck:
|
|
|
205
252
|
repair_plan=["Run nexo recover --force --yes, then restart Desktop"],
|
|
206
253
|
)
|
|
207
254
|
|
|
208
|
-
if
|
|
255
|
+
if fix and recoverable_local_regression:
|
|
256
|
+
try:
|
|
257
|
+
local_context_db.parent.mkdir(parents=True, exist_ok=True)
|
|
258
|
+
if not local_context_db.exists():
|
|
259
|
+
sqlite3.connect(str(local_context_db)).close()
|
|
260
|
+
except Exception as exc:
|
|
261
|
+
return DoctorCheck(
|
|
262
|
+
id="boot.db_integrity",
|
|
263
|
+
tier="boot",
|
|
264
|
+
status="critical",
|
|
265
|
+
severity="error",
|
|
266
|
+
summary="Local memory sidecar repair failed",
|
|
267
|
+
evidence=evidence + [f"Cannot create local memory DB: {type(exc).__name__}: {exc}"],
|
|
268
|
+
repair_plan=["Close NEXO Desktop and run nexo doctor --tier boot --plane database_real --fix"],
|
|
269
|
+
)
|
|
270
|
+
report = restore_tables_from_backup(local_reference, local_context_db, tables=LOCAL_CONTEXT_TABLES)
|
|
271
|
+
if report.get("ok"):
|
|
272
|
+
restored = {
|
|
273
|
+
table: payload
|
|
274
|
+
for table, payload in (report.get("tables") or {}).items()
|
|
275
|
+
if isinstance(payload, dict) and payload.get("status") == "restored"
|
|
276
|
+
}
|
|
277
|
+
restored_rows = sum(int(payload.get("after") or 0) for payload in restored.values())
|
|
278
|
+
return DoctorCheck(
|
|
279
|
+
id="boot.db_integrity",
|
|
280
|
+
tier="boot",
|
|
281
|
+
status="healthy",
|
|
282
|
+
severity="info",
|
|
283
|
+
summary=f"Local memory sidecar restored from backup ({restored_rows} rows)",
|
|
284
|
+
evidence=evidence + [f"Restored local-memory sidecar tables: {len(restored)}"],
|
|
285
|
+
fixed=True,
|
|
286
|
+
)
|
|
287
|
+
return DoctorCheck(
|
|
288
|
+
id="boot.db_integrity",
|
|
289
|
+
tier="boot",
|
|
290
|
+
status="critical",
|
|
291
|
+
severity="error",
|
|
292
|
+
summary="Local memory sidecar repair failed",
|
|
293
|
+
evidence=evidence + [f"Restore errors: {report.get('errors') or []}"],
|
|
294
|
+
repair_plan=["Close NEXO Desktop and run nexo doctor --tier boot --plane database_real --fix"],
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
if recoverable_wipe or recoverable_regression or recoverable_local_regression:
|
|
209
298
|
return DoctorCheck(
|
|
210
299
|
id="boot.db_integrity",
|
|
211
300
|
tier="boot",
|
package/src/local_context/api.py
CHANGED
|
@@ -4,6 +4,7 @@ import json
|
|
|
4
4
|
import os
|
|
5
5
|
import re
|
|
6
6
|
import shutil
|
|
7
|
+
import sqlite3
|
|
7
8
|
import stat
|
|
8
9
|
import hashlib
|
|
9
10
|
import subprocess
|
|
@@ -12,7 +13,7 @@ from pathlib import Path
|
|
|
12
13
|
from typing import Any
|
|
13
14
|
|
|
14
15
|
from . import embeddings
|
|
15
|
-
from .db import ensure_local_context_db, get_local_context_db
|
|
16
|
+
from .db import LOCAL_CONTEXT_TABLES, connect_local_context_db_readonly, ensure_local_context_db, get_local_context_db
|
|
16
17
|
from .extractors import chunk_text, contains_secret, entities, extract_text, summarize
|
|
17
18
|
from .logging import log_event, tail
|
|
18
19
|
from .privacy import classify_path, is_local_email_tree, is_queryable_path, should_extract, should_skip_file, should_skip_tree
|
|
@@ -40,7 +41,7 @@ VALID_CONTEXT_MODES = {"compact", "full"}
|
|
|
40
41
|
PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
41
42
|
"low": {
|
|
42
43
|
"profile": "low",
|
|
43
|
-
"
|
|
44
|
+
"label_key": "local_context.performance.low",
|
|
44
45
|
"scan_limit": 250,
|
|
45
46
|
"process_limit": 50,
|
|
46
47
|
"live_asset_limit": 500,
|
|
@@ -51,7 +52,7 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
51
52
|
},
|
|
52
53
|
"medium": {
|
|
53
54
|
"profile": "medium",
|
|
54
|
-
"
|
|
55
|
+
"label_key": "local_context.performance.medium",
|
|
55
56
|
"scan_limit": 1000,
|
|
56
57
|
"process_limit": 200,
|
|
57
58
|
"live_asset_limit": DEFAULT_LIVE_ASSET_LIMIT,
|
|
@@ -62,7 +63,7 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
62
63
|
},
|
|
63
64
|
"high": {
|
|
64
65
|
"profile": "high",
|
|
65
|
-
"
|
|
66
|
+
"label_key": "local_context.performance.high",
|
|
66
67
|
"scan_limit": 3000,
|
|
67
68
|
"process_limit": 600,
|
|
68
69
|
"live_asset_limit": 5000,
|
|
@@ -73,7 +74,7 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
73
74
|
},
|
|
74
75
|
"extreme": {
|
|
75
76
|
"profile": "extreme",
|
|
76
|
-
"
|
|
77
|
+
"label_key": "local_context.performance.extreme",
|
|
77
78
|
"scan_limit": 8000,
|
|
78
79
|
"process_limit": 1500,
|
|
79
80
|
"live_asset_limit": 10000,
|
|
@@ -94,6 +95,19 @@ def _conn():
|
|
|
94
95
|
return get_local_context_db()
|
|
95
96
|
|
|
96
97
|
|
|
98
|
+
def _read_conn():
|
|
99
|
+
conn = connect_local_context_db_readonly(timeout_ms=1200)
|
|
100
|
+
_validate_status_schema(conn)
|
|
101
|
+
return conn
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _close_read_conn(conn) -> None:
|
|
105
|
+
try:
|
|
106
|
+
conn.close()
|
|
107
|
+
except Exception:
|
|
108
|
+
pass
|
|
109
|
+
|
|
110
|
+
|
|
97
111
|
def add_root(path: str, *, mode: str = "normal", depth: int | None = None) -> dict:
|
|
98
112
|
conn = _conn()
|
|
99
113
|
root_path = norm_path(path)
|
|
@@ -136,8 +150,18 @@ def remove_root(path: str) -> dict:
|
|
|
136
150
|
return {"ok": True, "root_path": root_path, "cleanup": cleanup}
|
|
137
151
|
|
|
138
152
|
|
|
139
|
-
def list_roots() -> list[dict]:
|
|
140
|
-
|
|
153
|
+
def list_roots(*, readonly: bool = True) -> list[dict]:
|
|
154
|
+
if not readonly:
|
|
155
|
+
conn = _conn()
|
|
156
|
+
return _list_roots_conn(conn)
|
|
157
|
+
conn = _read_conn()
|
|
158
|
+
try:
|
|
159
|
+
return _list_roots_conn(conn)
|
|
160
|
+
finally:
|
|
161
|
+
_close_read_conn(conn)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _list_roots_conn(conn) -> list[dict]:
|
|
141
165
|
rows = conn.execute("SELECT * FROM local_index_roots WHERE status != 'removed' ORDER BY root_path").fetchall()
|
|
142
166
|
return [dict(row) for row in rows]
|
|
143
167
|
|
|
@@ -265,7 +289,7 @@ def default_root_specs() -> list[tuple[str, int]]:
|
|
|
265
289
|
|
|
266
290
|
|
|
267
291
|
def ensure_default_roots() -> dict:
|
|
268
|
-
existing = {row["root_path"]: row for row in list_roots()}
|
|
292
|
+
existing = {row["root_path"]: row for row in list_roots(readonly=False)}
|
|
269
293
|
created = []
|
|
270
294
|
updated = []
|
|
271
295
|
for root, depth in default_root_specs():
|
|
@@ -285,7 +309,7 @@ def ensure_default_roots() -> dict:
|
|
|
285
309
|
updated.append({"root_path": existing_row["root_path"], "depth": depth})
|
|
286
310
|
continue
|
|
287
311
|
created.append(add_root(str(candidate), mode="normal", depth=depth))
|
|
288
|
-
return {"ok": True, "created": len(created), "updated": len(updated), "roots": list_roots()}
|
|
312
|
+
return {"ok": True, "created": len(created), "updated": len(updated), "roots": list_roots(readonly=False)}
|
|
289
313
|
|
|
290
314
|
|
|
291
315
|
def _should_skip_mounted_root(candidate: Path) -> bool:
|
|
@@ -557,8 +581,18 @@ def remove_exclusion(path: str) -> dict:
|
|
|
557
581
|
return {"ok": True, "path": excluded_path}
|
|
558
582
|
|
|
559
583
|
|
|
560
|
-
def list_exclusions() -> list[dict]:
|
|
561
|
-
|
|
584
|
+
def list_exclusions(*, readonly: bool = True) -> list[dict]:
|
|
585
|
+
if not readonly:
|
|
586
|
+
conn = _conn()
|
|
587
|
+
return _list_exclusions_conn(conn)
|
|
588
|
+
conn = _read_conn()
|
|
589
|
+
try:
|
|
590
|
+
return _list_exclusions_conn(conn)
|
|
591
|
+
finally:
|
|
592
|
+
_close_read_conn(conn)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _list_exclusions_conn(conn) -> list[dict]:
|
|
562
596
|
rows = conn.execute("SELECT * FROM local_index_exclusions ORDER BY path").fetchall()
|
|
563
597
|
return [dict(row) for row in rows]
|
|
564
598
|
|
|
@@ -700,6 +734,15 @@ def _ensure_initial_index_started_at(conn) -> float:
|
|
|
700
734
|
return value
|
|
701
735
|
|
|
702
736
|
|
|
737
|
+
def _initial_index_started_at_readonly(conn) -> float:
|
|
738
|
+
raw = _get_state_conn(conn, INITIAL_INDEX_STARTED_AT_KEY, "")
|
|
739
|
+
try:
|
|
740
|
+
value = float(raw or 0)
|
|
741
|
+
except Exception:
|
|
742
|
+
value = 0.0
|
|
743
|
+
return value if value > 0 else (_earliest_index_activity(conn) or 0.0)
|
|
744
|
+
|
|
745
|
+
|
|
703
746
|
def _active_job_count(conn) -> int:
|
|
704
747
|
row = conn.execute(
|
|
705
748
|
"""
|
|
@@ -711,20 +754,20 @@ def _active_job_count(conn) -> int:
|
|
|
711
754
|
return int(row["total"] or 0)
|
|
712
755
|
|
|
713
756
|
|
|
714
|
-
def _refresh_initial_index_complete(conn, initial_scan: dict | None = None, active_jobs: int | None = None) -> bool:
|
|
757
|
+
def _refresh_initial_index_complete(conn, initial_scan: dict | None = None, active_jobs: int | None = None, *, readonly: bool = False) -> bool:
|
|
715
758
|
if _initial_index_complete(conn):
|
|
716
759
|
return True
|
|
717
760
|
scan_state = initial_scan if initial_scan is not None else _initial_scan_status(conn)
|
|
718
761
|
remaining = _active_job_count(conn) if active_jobs is None else int(active_jobs or 0)
|
|
719
762
|
complete = bool(scan_state.get("complete")) and remaining == 0
|
|
720
|
-
if complete:
|
|
763
|
+
if complete and not readonly:
|
|
721
764
|
_set_initial_index_complete(conn, True)
|
|
722
765
|
conn.commit()
|
|
723
766
|
return complete
|
|
724
767
|
|
|
725
768
|
|
|
726
769
|
def _initial_scan_status(conn, roots: list[dict] | None = None) -> dict:
|
|
727
|
-
rows = roots if roots is not None else
|
|
770
|
+
rows = roots if roots is not None else _list_roots_conn(conn)
|
|
728
771
|
tracked = _effective_scan_roots([dict(row) for row in rows if str(row.get("status") or "active") not in {"removed", "offline"}])
|
|
729
772
|
pending = [row for row in tracked if not _root_initial_scan_complete(conn, row)]
|
|
730
773
|
checkpoints = conn.execute(
|
|
@@ -753,7 +796,12 @@ def resume() -> dict:
|
|
|
753
796
|
|
|
754
797
|
|
|
755
798
|
def _is_paused() -> bool:
|
|
756
|
-
|
|
799
|
+
conn = _conn()
|
|
800
|
+
return _is_paused_conn(conn)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _is_paused_conn(conn) -> bool:
|
|
804
|
+
return _get_state_conn(conn, "paused", "0") == "1"
|
|
757
805
|
|
|
758
806
|
|
|
759
807
|
def _allow_explicit_blocked_root(path: str) -> bool:
|
|
@@ -1465,7 +1513,7 @@ def reconcile_live_changes(
|
|
|
1465
1513
|
conn = _conn()
|
|
1466
1514
|
if _is_paused():
|
|
1467
1515
|
return {"ok": True, "paused": True, "assets": {}, "dirs": {}}
|
|
1468
|
-
exclusions = [row["path"] for row in list_exclusions()]
|
|
1516
|
+
exclusions = [row["path"] for row in list_exclusions(readonly=False)]
|
|
1469
1517
|
asset_stats = _reconcile_known_assets(conn, exclusions, limit=int(asset_limit or 0))
|
|
1470
1518
|
dir_stats = _reconcile_known_dirs(conn, exclusions, dir_limit=int(dir_limit or 0), file_limit=int(file_limit or 0))
|
|
1471
1519
|
conn.commit()
|
|
@@ -1498,8 +1546,8 @@ def scan_once(*, limit: int | None = None) -> dict:
|
|
|
1498
1546
|
log_event("info", "scan_skipped_paused", "Local memory scan skipped because indexing is paused")
|
|
1499
1547
|
return {"ok": True, "paused": True, "roots": 0, "seen": 0, "changed": 0, "errors": 0, "partial": False}
|
|
1500
1548
|
started = now()
|
|
1501
|
-
roots = _effective_scan_roots(list_roots())
|
|
1502
|
-
exclusions = [row["path"] for row in list_exclusions()]
|
|
1549
|
+
roots = _effective_scan_roots(list_roots(readonly=False))
|
|
1550
|
+
exclusions = [row["path"] for row in list_exclusions(readonly=False)]
|
|
1503
1551
|
totals = {"roots": len(roots), "seen": 0, "changed": 0, "errors": 0, "partial": False}
|
|
1504
1552
|
log_event("info", "scan_started", "Local memory scan started", roots=len(roots))
|
|
1505
1553
|
for root in roots:
|
|
@@ -1793,7 +1841,7 @@ def run_once(
|
|
|
1793
1841
|
effective_live_dir_limit = int(live_dir_limit if live_dir_limit is not None else config["live_dir_limit"])
|
|
1794
1842
|
effective_live_file_limit = int(live_file_limit if live_file_limit is not None else config["live_file_limit"])
|
|
1795
1843
|
conn = _conn()
|
|
1796
|
-
initial_before = _initial_scan_status(conn, list_roots())
|
|
1844
|
+
initial_before = _initial_scan_status(conn, list_roots(readonly=False))
|
|
1797
1845
|
initial_index_before = _refresh_initial_index_complete(conn, initial_before)
|
|
1798
1846
|
if initial_index_before:
|
|
1799
1847
|
live_result = reconcile_live_changes(
|
|
@@ -1812,7 +1860,7 @@ def run_once(
|
|
|
1812
1860
|
scan_result = scan_once(limit=effective_scan_limit)
|
|
1813
1861
|
job_result = process_jobs(limit=effective_process_limit)
|
|
1814
1862
|
conn_after = _conn()
|
|
1815
|
-
initial_after = _initial_scan_status(conn_after, list_roots())
|
|
1863
|
+
initial_after = _initial_scan_status(conn_after, list_roots(readonly=False))
|
|
1816
1864
|
active_after = _active_job_count(conn_after)
|
|
1817
1865
|
initial_index_after = _refresh_initial_index_complete(conn_after, initial_after, active_after)
|
|
1818
1866
|
return {
|
|
@@ -1854,8 +1902,10 @@ def _problem_rows(conn) -> list[dict]:
|
|
|
1854
1902
|
).fetchall()
|
|
1855
1903
|
problems = [
|
|
1856
1904
|
{
|
|
1857
|
-
"user_message":
|
|
1858
|
-
"
|
|
1905
|
+
"user_message": "",
|
|
1906
|
+
"message_key": "local_context.problem.file_read_failed",
|
|
1907
|
+
"recommended_action": "",
|
|
1908
|
+
"recommended_action_key": "local_context.retry_later" if row["retryable"] else "local_context.review_permissions_or_file",
|
|
1859
1909
|
"technical_detail": row["technical_detail"],
|
|
1860
1910
|
"support_code": row["error_code"],
|
|
1861
1911
|
"severity": "warning",
|
|
@@ -1882,8 +1932,10 @@ def _problem_rows(conn) -> list[dict]:
|
|
|
1882
1932
|
).fetchall()
|
|
1883
1933
|
problems.extend(
|
|
1884
1934
|
{
|
|
1885
|
-
"user_message": "
|
|
1886
|
-
"
|
|
1935
|
+
"user_message": "",
|
|
1936
|
+
"message_key": "local_context.problem.service_temporary",
|
|
1937
|
+
"recommended_action": "",
|
|
1938
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
1887
1939
|
"technical_detail": f"{row['event']}: {row['message']} {row['metadata_json']}",
|
|
1888
1940
|
"support_code": row["event"],
|
|
1889
1941
|
"severity": "warning" if row["level"] == "warn" else "error",
|
|
@@ -2084,13 +2136,13 @@ def _service_cycle_observation(conn) -> dict:
|
|
|
2084
2136
|
return observation
|
|
2085
2137
|
|
|
2086
2138
|
|
|
2087
|
-
def _index_timing(conn, *, done: int, active_jobs: int, percent: int) -> dict:
|
|
2088
|
-
first_seen = _ensure_initial_index_started_at(conn)
|
|
2139
|
+
def _index_timing(conn, *, done: int, active_jobs: int, percent: int, readonly: bool = False) -> dict:
|
|
2140
|
+
first_seen = _initial_index_started_at_readonly(conn) if readonly else _ensure_initial_index_started_at(conn)
|
|
2089
2141
|
elapsed_seconds = max(0, int(now() - float(first_seen))) if first_seen else 0
|
|
2090
2142
|
eta_seconds = None
|
|
2091
2143
|
if elapsed_seconds > 0 and done > 0 and active_jobs > 0 and 0 < percent < 100:
|
|
2092
2144
|
eta_seconds = max(0, int((elapsed_seconds / max(done, 1)) * active_jobs))
|
|
2093
|
-
return {"elapsed_seconds": elapsed_seconds, "eta_seconds": eta_seconds}
|
|
2145
|
+
return {"started_at": first_seen, "elapsed_seconds": elapsed_seconds, "eta_seconds": eta_seconds}
|
|
2094
2146
|
|
|
2095
2147
|
|
|
2096
2148
|
def _service_scheduler_has_error(service: dict) -> bool:
|
|
@@ -2107,38 +2159,118 @@ def _service_problem(service: dict) -> dict | None:
|
|
|
2107
2159
|
if not service.get("installed"):
|
|
2108
2160
|
return {
|
|
2109
2161
|
"support_code": "local_index_service_not_installed",
|
|
2110
|
-
"user_message": "
|
|
2111
|
-
"
|
|
2162
|
+
"user_message": "",
|
|
2163
|
+
"message_key": "local_context.problem.service_not_installed",
|
|
2164
|
+
"recommended_action": "",
|
|
2165
|
+
"recommended_action_key": "local_context.reopen_or_update_desktop",
|
|
2112
2166
|
"technical_detail": f"manager={service.get('manager')} platform={service.get('platform')}",
|
|
2113
2167
|
}
|
|
2114
2168
|
if not service.get("running"):
|
|
2115
2169
|
return {
|
|
2116
2170
|
"support_code": "local_index_service_not_running",
|
|
2117
|
-
"user_message": "
|
|
2118
|
-
"
|
|
2171
|
+
"user_message": "",
|
|
2172
|
+
"message_key": "local_context.problem.service_not_running",
|
|
2173
|
+
"recommended_action": "",
|
|
2174
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2119
2175
|
"technical_detail": f"manager={service.get('manager')} platform={service.get('platform')}",
|
|
2120
2176
|
}
|
|
2121
2177
|
if _service_scheduler_has_error(service):
|
|
2122
2178
|
code = service.get("last_exit_code") or service.get("last_task_result") or ""
|
|
2123
2179
|
return {
|
|
2124
2180
|
"support_code": "local_index_service_last_run_failed",
|
|
2125
|
-
"user_message": "
|
|
2126
|
-
"
|
|
2181
|
+
"user_message": "",
|
|
2182
|
+
"message_key": "local_context.problem.service_last_run_failed",
|
|
2183
|
+
"recommended_action": "",
|
|
2184
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2127
2185
|
"technical_detail": f"last_result={code}",
|
|
2128
2186
|
}
|
|
2129
2187
|
if not service.get("healthy", True):
|
|
2130
2188
|
return {
|
|
2131
2189
|
"support_code": service.get("last_error_code") or "local_index_service_failed",
|
|
2132
|
-
"user_message": "
|
|
2133
|
-
"
|
|
2190
|
+
"user_message": "",
|
|
2191
|
+
"message_key": "local_context.problem.service_temporary",
|
|
2192
|
+
"recommended_action": "",
|
|
2193
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2134
2194
|
"technical_detail": service.get("last_error_detail") or "",
|
|
2135
2195
|
}
|
|
2136
2196
|
return None
|
|
2137
2197
|
|
|
2138
2198
|
|
|
2199
|
+
def _status_read_error(exc: Exception, *, code: str = "local_context_status_unavailable") -> dict:
|
|
2200
|
+
service = _local_index_service_status()
|
|
2201
|
+
service_problem = _service_problem(service)
|
|
2202
|
+
service["healthy"] = service_problem is None
|
|
2203
|
+
service["state"] = "attention" if service_problem else "unavailable"
|
|
2204
|
+
problems = []
|
|
2205
|
+
if service_problem:
|
|
2206
|
+
problems.append({
|
|
2207
|
+
"user_message": service_problem["user_message"],
|
|
2208
|
+
"message_key": service_problem.get("message_key", ""),
|
|
2209
|
+
"recommended_action": service_problem["recommended_action"],
|
|
2210
|
+
"recommended_action_key": service_problem.get("recommended_action_key", ""),
|
|
2211
|
+
"technical_detail": service_problem["technical_detail"],
|
|
2212
|
+
"support_code": service_problem["support_code"],
|
|
2213
|
+
"severity": "warning",
|
|
2214
|
+
"retryable": True,
|
|
2215
|
+
"path": "",
|
|
2216
|
+
"phase": "service",
|
|
2217
|
+
"created_at": now(),
|
|
2218
|
+
})
|
|
2219
|
+
problems.append({
|
|
2220
|
+
"user_message": "",
|
|
2221
|
+
"message_key": "local_context.status_unavailable",
|
|
2222
|
+
"recommended_action": "",
|
|
2223
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2224
|
+
"technical_detail": str(exc),
|
|
2225
|
+
"support_code": code,
|
|
2226
|
+
"severity": "warning",
|
|
2227
|
+
"retryable": True,
|
|
2228
|
+
"path": "",
|
|
2229
|
+
"phase": "status",
|
|
2230
|
+
"created_at": now(),
|
|
2231
|
+
})
|
|
2232
|
+
return {
|
|
2233
|
+
"ok": False,
|
|
2234
|
+
"error": code,
|
|
2235
|
+
"retryable": True,
|
|
2236
|
+
"global": None,
|
|
2237
|
+
"service": service,
|
|
2238
|
+
"problems": problems,
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
|
|
2242
|
+
def _status_db_error_code(exc: Exception) -> str:
|
|
2243
|
+
text = str(exc).lower()
|
|
2244
|
+
if "locked" in text or "busy" in text:
|
|
2245
|
+
return "local_context_db_busy"
|
|
2246
|
+
if "no such table" in text or "no such column" in text or "schema missing" in text or "missing tables" in text:
|
|
2247
|
+
return "local_context_db_schema_missing"
|
|
2248
|
+
if "file is not a database" in text or "database disk image is malformed" in text:
|
|
2249
|
+
return "local_context_db_invalid"
|
|
2250
|
+
return "local_context_db_unreadable"
|
|
2251
|
+
|
|
2252
|
+
|
|
2139
2253
|
def status() -> dict:
|
|
2140
|
-
|
|
2141
|
-
|
|
2254
|
+
try:
|
|
2255
|
+
conn = connect_local_context_db_readonly(timeout_ms=1200)
|
|
2256
|
+
except FileNotFoundError as exc:
|
|
2257
|
+
return _status_read_error(exc, code="local_context_db_missing")
|
|
2258
|
+
except sqlite3.DatabaseError as exc:
|
|
2259
|
+
return _status_read_error(exc, code=_status_db_error_code(exc))
|
|
2260
|
+
try:
|
|
2261
|
+
return _status_from_conn(conn, readonly=True)
|
|
2262
|
+
except sqlite3.DatabaseError as exc:
|
|
2263
|
+
return _status_read_error(exc, code=_status_db_error_code(exc))
|
|
2264
|
+
finally:
|
|
2265
|
+
try:
|
|
2266
|
+
conn.close()
|
|
2267
|
+
except Exception:
|
|
2268
|
+
pass
|
|
2269
|
+
|
|
2270
|
+
|
|
2271
|
+
def _status_from_conn(conn, *, readonly: bool = False) -> dict:
|
|
2272
|
+
_validate_status_schema(conn)
|
|
2273
|
+
paused = _is_paused_conn(conn)
|
|
2142
2274
|
assets = conn.execute(
|
|
2143
2275
|
"""
|
|
2144
2276
|
SELECT COUNT(*) AS total, SUM(CASE WHEN a.status='active' THEN 1 ELSE 0 END) AS active
|
|
@@ -2166,10 +2298,10 @@ def status() -> dict:
|
|
|
2166
2298
|
active_jobs = pending + running_jobs + failed_jobs
|
|
2167
2299
|
total_jobs = active_jobs + done
|
|
2168
2300
|
percent = 100 if total_jobs == 0 else int((done / max(total_jobs, 1)) * 100)
|
|
2169
|
-
timing = _index_timing(conn, done=done, active_jobs=active_jobs, percent=percent)
|
|
2170
|
-
roots =
|
|
2301
|
+
timing = _index_timing(conn, done=done, active_jobs=active_jobs, percent=percent, readonly=readonly)
|
|
2302
|
+
roots = _list_roots_conn(conn)
|
|
2171
2303
|
initial_scan = _initial_scan_status(conn, roots)
|
|
2172
|
-
initial_index_complete = _refresh_initial_index_complete(conn, initial_scan, active_jobs)
|
|
2304
|
+
initial_index_complete = _refresh_initial_index_complete(conn, initial_scan, active_jobs, readonly=readonly)
|
|
2173
2305
|
volumes = []
|
|
2174
2306
|
by_volume = conn.execute(
|
|
2175
2307
|
"""
|
|
@@ -2194,7 +2326,9 @@ def status() -> dict:
|
|
|
2194
2326
|
if problem:
|
|
2195
2327
|
problems.insert(0, {
|
|
2196
2328
|
"user_message": problem["user_message"],
|
|
2329
|
+
"message_key": problem.get("message_key", ""),
|
|
2197
2330
|
"recommended_action": problem["recommended_action"],
|
|
2331
|
+
"recommended_action_key": problem.get("recommended_action_key", ""),
|
|
2198
2332
|
"technical_detail": problem["technical_detail"],
|
|
2199
2333
|
"support_code": problem["support_code"],
|
|
2200
2334
|
"severity": "warning",
|
|
@@ -2213,6 +2347,9 @@ def status() -> dict:
|
|
|
2213
2347
|
phase = "idle"
|
|
2214
2348
|
else:
|
|
2215
2349
|
phase = "updating_changes"
|
|
2350
|
+
index_started_at = _get_state_conn(conn, INITIAL_INDEX_STARTED_AT_KEY, "")
|
|
2351
|
+
if not index_started_at and timing["started_at"]:
|
|
2352
|
+
index_started_at = str(float(timing["started_at"]))
|
|
2216
2353
|
return {
|
|
2217
2354
|
"ok": True,
|
|
2218
2355
|
"service": service,
|
|
@@ -2227,7 +2364,7 @@ def status() -> dict:
|
|
|
2227
2364
|
"jobs_failed": failed_jobs,
|
|
2228
2365
|
"elapsed_seconds": timing["elapsed_seconds"],
|
|
2229
2366
|
"eta_seconds": timing["eta_seconds"],
|
|
2230
|
-
"index_started_at":
|
|
2367
|
+
"index_started_at": index_started_at,
|
|
2231
2368
|
"initial_scan_complete": bool(initial_index_complete),
|
|
2232
2369
|
"initial_discovery_complete": bool(initial_scan["complete"]),
|
|
2233
2370
|
"initial_index_complete": bool(initial_index_complete),
|
|
@@ -2239,7 +2376,7 @@ def status() -> dict:
|
|
|
2239
2376
|
"initial_index_complete": bool(initial_index_complete),
|
|
2240
2377
|
"volumes": volumes,
|
|
2241
2378
|
"roots": roots,
|
|
2242
|
-
"exclusions":
|
|
2379
|
+
"exclusions": _list_exclusions_conn(conn),
|
|
2243
2380
|
"problems": problems,
|
|
2244
2381
|
"permissions": [],
|
|
2245
2382
|
"models": model_status()["models"],
|
|
@@ -2247,6 +2384,18 @@ def status() -> dict:
|
|
|
2247
2384
|
}
|
|
2248
2385
|
|
|
2249
2386
|
|
|
2387
|
+
def _validate_status_schema(conn) -> None:
|
|
2388
|
+
placeholders = ",".join("?" for _ in LOCAL_CONTEXT_TABLES)
|
|
2389
|
+
rows = conn.execute(
|
|
2390
|
+
f"SELECT name FROM sqlite_master WHERE type='table' AND name IN ({placeholders})",
|
|
2391
|
+
tuple(LOCAL_CONTEXT_TABLES),
|
|
2392
|
+
).fetchall()
|
|
2393
|
+
found = {str(row["name"] if isinstance(row, sqlite3.Row) else row[0]) for row in rows}
|
|
2394
|
+
missing = [table for table in LOCAL_CONTEXT_TABLES if table not in found]
|
|
2395
|
+
if missing:
|
|
2396
|
+
raise sqlite3.OperationalError("local context schema missing tables: " + ", ".join(missing[:8]))
|
|
2397
|
+
|
|
2398
|
+
|
|
2250
2399
|
def diagnostics_tail(limit: int = 100) -> dict:
|
|
2251
2400
|
return {"ok": True, "logs": tail(limit)}
|
|
2252
2401
|
|
|
@@ -2768,8 +2917,46 @@ def context_query(
|
|
|
2768
2917
|
include_entities: bool = True,
|
|
2769
2918
|
include_relations: bool = True,
|
|
2770
2919
|
snippet_chars: int = 1200,
|
|
2920
|
+
readonly: bool = True,
|
|
2921
|
+
record_query: bool = False,
|
|
2922
|
+
) -> dict:
|
|
2923
|
+
conn = _read_conn() if readonly else _conn()
|
|
2924
|
+
close_conn = bool(readonly)
|
|
2925
|
+
try:
|
|
2926
|
+
return _context_query_conn(
|
|
2927
|
+
conn,
|
|
2928
|
+
query,
|
|
2929
|
+
intent=intent,
|
|
2930
|
+
limit=limit,
|
|
2931
|
+
evidence_required=evidence_required,
|
|
2932
|
+
current_context=current_context,
|
|
2933
|
+
mode=mode,
|
|
2934
|
+
max_chars=max_chars,
|
|
2935
|
+
include_entities=include_entities,
|
|
2936
|
+
include_relations=include_relations,
|
|
2937
|
+
snippet_chars=snippet_chars,
|
|
2938
|
+
record_query=bool(record_query and not readonly),
|
|
2939
|
+
)
|
|
2940
|
+
finally:
|
|
2941
|
+
if close_conn:
|
|
2942
|
+
_close_read_conn(conn)
|
|
2943
|
+
|
|
2944
|
+
|
|
2945
|
+
def _context_query_conn(
|
|
2946
|
+
conn,
|
|
2947
|
+
query: str,
|
|
2948
|
+
*,
|
|
2949
|
+
intent: str,
|
|
2950
|
+
limit: int,
|
|
2951
|
+
evidence_required: bool,
|
|
2952
|
+
current_context: str,
|
|
2953
|
+
mode: str,
|
|
2954
|
+
max_chars: int,
|
|
2955
|
+
include_entities: bool,
|
|
2956
|
+
include_relations: bool,
|
|
2957
|
+
snippet_chars: int,
|
|
2958
|
+
record_query: bool,
|
|
2771
2959
|
) -> dict:
|
|
2772
|
-
conn = _conn()
|
|
2773
2960
|
clean_query = str(query or "").strip()
|
|
2774
2961
|
normalized_mode, mode_warnings = _normalize_context_mode(mode)
|
|
2775
2962
|
context_tail = _compact_text(current_context or "", max_chars=1000)
|
|
@@ -2843,21 +3030,22 @@ def context_query(
|
|
|
2843
3030
|
summary = ""
|
|
2844
3031
|
if assets:
|
|
2845
3032
|
summary = f"Found {len(assets)} local asset(s) related to '{clean_query}'."
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
3033
|
+
if record_query:
|
|
3034
|
+
conn.execute(
|
|
3035
|
+
"""
|
|
3036
|
+
INSERT INTO local_context_queries(query_hash, intent, result_count, confidence, warnings_json, created_at)
|
|
3037
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
3038
|
+
""",
|
|
3039
|
+
(
|
|
3040
|
+
hashlib.sha256(clean_query.encode("utf-8", errors="ignore")).hexdigest(),
|
|
3041
|
+
intent,
|
|
3042
|
+
len(assets),
|
|
3043
|
+
0.75 if evidence_refs else 0.0,
|
|
3044
|
+
json_dumps(warnings),
|
|
3045
|
+
now(),
|
|
3046
|
+
),
|
|
3047
|
+
)
|
|
3048
|
+
conn.commit()
|
|
2861
3049
|
payload = {
|
|
2862
3050
|
"ok": True,
|
|
2863
3051
|
"query": clean_query,
|
|
@@ -2881,26 +3069,34 @@ def context_query(
|
|
|
2881
3069
|
)
|
|
2882
3070
|
|
|
2883
3071
|
|
|
2884
|
-
def get_asset(asset_id: str) -> dict:
|
|
2885
|
-
conn = _conn()
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
3072
|
+
def get_asset(asset_id: str, *, readonly: bool = True) -> dict:
|
|
3073
|
+
conn = _read_conn() if readonly else _conn()
|
|
3074
|
+
try:
|
|
3075
|
+
row = conn.execute("SELECT * FROM local_assets WHERE asset_id=?", (asset_id,)).fetchone()
|
|
3076
|
+
if not row:
|
|
3077
|
+
return {"ok": False, "error": "asset_not_found"}
|
|
3078
|
+
return {"ok": True, "asset": dict(row)}
|
|
3079
|
+
finally:
|
|
3080
|
+
if readonly:
|
|
3081
|
+
_close_read_conn(conn)
|
|
2890
3082
|
|
|
2891
3083
|
|
|
2892
|
-
def get_neighbors(asset_id: str, *, limit: int = 30) -> dict:
|
|
2893
|
-
conn = _conn()
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
3084
|
+
def get_neighbors(asset_id: str, *, limit: int = 30, readonly: bool = True) -> dict:
|
|
3085
|
+
conn = _read_conn() if readonly else _conn()
|
|
3086
|
+
try:
|
|
3087
|
+
rows = conn.execute(
|
|
3088
|
+
"""
|
|
3089
|
+
SELECT * FROM local_relations
|
|
3090
|
+
WHERE source_asset_id=? AND active=1
|
|
3091
|
+
ORDER BY confidence DESC
|
|
3092
|
+
LIMIT ?
|
|
3093
|
+
""",
|
|
3094
|
+
(asset_id, int(limit)),
|
|
3095
|
+
).fetchall()
|
|
3096
|
+
return {"ok": True, "relations": [dict(row) for row in rows]}
|
|
3097
|
+
finally:
|
|
3098
|
+
if readonly:
|
|
3099
|
+
_close_read_conn(conn)
|
|
2904
3100
|
|
|
2905
3101
|
|
|
2906
3102
|
def purge_asset(asset_id: str) -> dict:
|
package/src/local_context/db.py
CHANGED
|
@@ -5,6 +5,7 @@ import sqlite3
|
|
|
5
5
|
import time
|
|
6
6
|
from pathlib import Path
|
|
7
7
|
from typing import Iterable
|
|
8
|
+
from urllib.parse import quote
|
|
8
9
|
|
|
9
10
|
import paths
|
|
10
11
|
from db._schema import _m63_local_context_layer, _m64_local_context_live_dirs
|
|
@@ -79,6 +80,23 @@ def _connect(db_path: Path) -> sqlite3.Connection:
|
|
|
79
80
|
return conn
|
|
80
81
|
|
|
81
82
|
|
|
83
|
+
def connect_local_context_db_readonly(*, timeout_ms: int = 1200) -> sqlite3.Connection:
|
|
84
|
+
db_path = local_context_db_path()
|
|
85
|
+
if not db_path.is_file():
|
|
86
|
+
raise FileNotFoundError(str(db_path))
|
|
87
|
+
timeout = max(float(timeout_ms) / 1000.0, 0.1)
|
|
88
|
+
uri_path = quote(db_path.resolve().as_posix(), safe="/:")
|
|
89
|
+
uri_params = "mode=ro"
|
|
90
|
+
if not db_path.with_name(db_path.name + "-wal").exists() and not db_path.with_name(db_path.name + "-shm").exists():
|
|
91
|
+
uri_params += "&immutable=1"
|
|
92
|
+
uri = f"file:{uri_path}?{uri_params}"
|
|
93
|
+
conn = sqlite3.connect(uri, uri=True, timeout=timeout, check_same_thread=False)
|
|
94
|
+
conn.row_factory = sqlite3.Row
|
|
95
|
+
conn.execute(f"PRAGMA busy_timeout={max(100, int(timeout_ms))}")
|
|
96
|
+
conn.execute("PRAGMA query_only=ON")
|
|
97
|
+
return conn
|
|
98
|
+
|
|
99
|
+
|
|
82
100
|
def _ensure_schema(conn: sqlite3.Connection) -> None:
|
|
83
101
|
_m63_local_context_layer(conn)
|
|
84
102
|
_m64_local_context_live_dirs(conn)
|
package/src/plugins/recover.py
CHANGED
|
@@ -35,6 +35,7 @@ from db_guard import (
|
|
|
35
35
|
CRITICAL_TABLES,
|
|
36
36
|
EMPTY_DB_SIZE_BYTES,
|
|
37
37
|
HOURLY_BACKUP_GLOB,
|
|
38
|
+
LOCAL_CONTEXT_TABLES,
|
|
38
39
|
MIN_REFERENCE_ROWS,
|
|
39
40
|
PROTECTED_TABLES,
|
|
40
41
|
WIPE_THRESHOLD_PCT,
|
|
@@ -48,6 +49,8 @@ from db_guard import (
|
|
|
48
49
|
validate_backup_matches_source,
|
|
49
50
|
)
|
|
50
51
|
|
|
52
|
+
RECOVERY_TABLES = PROTECTED_TABLES + LOCAL_CONTEXT_TABLES
|
|
53
|
+
|
|
51
54
|
# Path resolution moved to lazy helpers (AUDITOR-V700-PASS2 §11, B10 item 3)
|
|
52
55
|
# to keep monkeypatched NEXO_HOME / paths.* fixtures honoured. PEP 562
|
|
53
56
|
# ``__getattr__`` below preserves the legacy constant names for any caller
|
|
@@ -139,7 +142,7 @@ def _describe_backup(path: Path, kind: str) -> dict:
|
|
|
139
142
|
counts: dict[str, int | None] = {}
|
|
140
143
|
critical_rows = 0
|
|
141
144
|
if size > EMPTY_DB_SIZE_BYTES:
|
|
142
|
-
counts = db_row_counts(path,
|
|
145
|
+
counts = db_row_counts(path, RECOVERY_TABLES)
|
|
143
146
|
critical_rows = sum(v for v in counts.values() if isinstance(v, int))
|
|
144
147
|
return {
|
|
145
148
|
"path": str(path),
|
|
@@ -240,7 +243,7 @@ def recover(
|
|
|
240
243
|
result["source"] = str(chosen)
|
|
241
244
|
result["steps"].append(f"chose source: {chosen}")
|
|
242
245
|
|
|
243
|
-
source_counts = db_row_counts(chosen,
|
|
246
|
+
source_counts = db_row_counts(chosen, RECOVERY_TABLES)
|
|
244
247
|
result["source_row_counts"] = {k: v for k, v in source_counts.items() if v is not None}
|
|
245
248
|
source_total = sum(v for v in source_counts.values() if isinstance(v, int))
|
|
246
249
|
if source_total < MIN_REFERENCE_ROWS:
|
|
@@ -317,7 +320,7 @@ def recover(
|
|
|
317
320
|
return result
|
|
318
321
|
result["steps"].append(f"restored {chosen.name} -> {target_path}")
|
|
319
322
|
|
|
320
|
-
valid, valid_err = validate_backup_matches_source(chosen, target_path,
|
|
323
|
+
valid, valid_err = validate_backup_matches_source(chosen, target_path, RECOVERY_TABLES)
|
|
321
324
|
if not valid:
|
|
322
325
|
result["errors"].append(f"post-restore validation failed: {valid_err}")
|
|
323
326
|
if stopped_launchagents:
|
|
@@ -325,7 +328,7 @@ def recover(
|
|
|
325
328
|
return result
|
|
326
329
|
result["steps"].append("validated post-restore row counts")
|
|
327
330
|
|
|
328
|
-
final_counts = db_row_counts(target_path,
|
|
331
|
+
final_counts = db_row_counts(target_path, RECOVERY_TABLES)
|
|
329
332
|
result["final_row_counts"] = {k: v for k, v in final_counts.items() if v is not None}
|
|
330
333
|
if stopped_launchagents:
|
|
331
334
|
result["resume"] = resume_nexo_launchagents(stopped_launchagents)
|
package/src/plugins/update.py
CHANGED
|
@@ -485,8 +485,14 @@ def _row_count_regression(pre: dict[str, int | None], post: dict[str, int | None
|
|
|
485
485
|
drop >= WIPE_THRESHOLD_PCT across CRITICAL_TABLES, is treated as a wipe.
|
|
486
486
|
"""
|
|
487
487
|
regressions: list[str] = []
|
|
488
|
-
pre_total = sum(
|
|
489
|
-
|
|
488
|
+
pre_total = sum(
|
|
489
|
+
pre.get(table) for table in PROTECTED_TABLES
|
|
490
|
+
if isinstance(pre.get(table), int)
|
|
491
|
+
)
|
|
492
|
+
post_total = sum(
|
|
493
|
+
post.get(table) for table in PROTECTED_TABLES
|
|
494
|
+
if isinstance(post.get(table), int)
|
|
495
|
+
)
|
|
490
496
|
for table in PROTECTED_TABLES:
|
|
491
497
|
pre_v = pre.get(table)
|
|
492
498
|
post_v = post.get(table)
|
package/src/server.py
CHANGED
|
@@ -810,7 +810,7 @@ def nexo_local_index_roots(action: str = "list", path: str = "", mode: str = "no
|
|
|
810
810
|
"""List, add or remove local memory roots."""
|
|
811
811
|
normalized = str(action or "list").strip().lower()
|
|
812
812
|
if normalized == "list":
|
|
813
|
-
result = {"ok": True, "roots": local_context_api.list_roots()}
|
|
813
|
+
result = {"ok": True, "roots": local_context_api.list_roots(readonly=True)}
|
|
814
814
|
elif normalized == "add":
|
|
815
815
|
result = local_context_api.add_root(path, mode=mode, depth=depth)
|
|
816
816
|
elif normalized == "remove":
|
|
@@ -825,7 +825,7 @@ def nexo_local_index_exclusions(action: str = "list", path: str = "", reason: st
|
|
|
825
825
|
"""List, add or remove local memory exclusions."""
|
|
826
826
|
normalized = str(action or "list").strip().lower()
|
|
827
827
|
if normalized == "list":
|
|
828
|
-
result = {"ok": True, "exclusions": local_context_api.list_exclusions()}
|
|
828
|
+
result = {"ok": True, "exclusions": local_context_api.list_exclusions(readonly=True)}
|
|
829
829
|
elif normalized == "add":
|
|
830
830
|
result = local_context_api.add_exclusion(path, reason=reason)
|
|
831
831
|
elif normalized == "remove":
|
package/src/tools_sessions.py
CHANGED
|
@@ -474,9 +474,12 @@ def handle_startup(
|
|
|
474
474
|
startup_warnings,
|
|
475
475
|
)
|
|
476
476
|
memory_maintenance = None
|
|
477
|
-
|
|
477
|
+
try:
|
|
478
|
+
backfill_limit = int(os.environ.get("NEXO_MEMORY_STARTUP_BACKFILL_LIMIT", "0") or "0")
|
|
479
|
+
except Exception:
|
|
480
|
+
backfill_limit = 0
|
|
481
|
+
if _env_flag("NEXO_MEMORY_MAINTENANCE_IN_STARTUP", default=False) or backfill_limit > 0:
|
|
478
482
|
try:
|
|
479
|
-
backfill_limit = int(os.environ.get("NEXO_MEMORY_STARTUP_BACKFILL_LIMIT", "0") or "0")
|
|
480
483
|
memory_maintenance = maintain_memory_observations(
|
|
481
484
|
process_limit=int(os.environ.get("NEXO_MEMORY_STARTUP_PROCESS_LIMIT", "20") or "20"),
|
|
482
485
|
retry_failed=True,
|
|
@@ -783,7 +786,7 @@ def _handle_heartbeat_inner(sid: str, task: str, context_hint: str = '') -> str:
|
|
|
783
786
|
if bundle.get("has_matches"):
|
|
784
787
|
parts.append("")
|
|
785
788
|
parts.append(format_pre_action_context_bundle(bundle, compact=True))
|
|
786
|
-
if _env_flag("NEXO_HEARTBEAT_LOCAL_CONTEXT", default=
|
|
789
|
+
if _env_flag("NEXO_HEARTBEAT_LOCAL_CONTEXT", default=True) and append_local_context_evidence is not None:
|
|
787
790
|
local_rendered = append_local_context_evidence("", recent_query, limit=4).strip()
|
|
788
791
|
if local_rendered:
|
|
789
792
|
parts.append("")
|
|
@@ -884,7 +887,7 @@ def _handle_heartbeat_inner(sid: str, task: str, context_hint: str = '') -> str:
|
|
|
884
887
|
|
|
885
888
|
# ── Drive/Curiosity: detect signals from context_hint (best-effort) ──
|
|
886
889
|
try:
|
|
887
|
-
if _env_flag("NEXO_DRIVE_IN_HEARTBEAT", default=
|
|
890
|
+
if _env_flag("NEXO_DRIVE_IN_HEARTBEAT", default=True) and context_hint and len(context_hint.strip()) >= 15:
|
|
888
891
|
from tools_drive import detect_drive_signal as _detect_drive
|
|
889
892
|
_drive_allow_llm = _env_flag("NEXO_DRIVE_LLM_IN_HEARTBEAT", default=False)
|
|
890
893
|
_drive_result = _detect_drive(
|
|
@@ -990,7 +993,7 @@ def _handle_heartbeat_inner(sid: str, task: str, context_hint: str = '') -> str:
|
|
|
990
993
|
# adaptive_log row per heartbeat. Wrapped in best-effort try/except so
|
|
991
994
|
# a failure here cannot block the heartbeat itself.
|
|
992
995
|
try:
|
|
993
|
-
if _env_flag("NEXO_HEARTBEAT_ADAPTIVE_MODE", default=
|
|
996
|
+
if _env_flag("NEXO_HEARTBEAT_ADAPTIVE_MODE", default=True) and context_hint and len(context_hint.strip()) >= 5:
|
|
994
997
|
from plugins.adaptive_mode import compute_mode
|
|
995
998
|
from cognitive._trust import detect_sentiment
|
|
996
999
|
sentiment = detect_sentiment(context_hint)
|
|
@@ -1334,9 +1337,7 @@ def _hint_suggests_correction(hint: str, *, correction_detector=None) -> bool:
|
|
|
1334
1337
|
text = (hint or "").strip()
|
|
1335
1338
|
if not text:
|
|
1336
1339
|
return False
|
|
1337
|
-
detector = correction_detector if correction_detector is not None else
|
|
1338
|
-
_detect_correction_semantic if _env_flag("NEXO_HEARTBEAT_SEMANTIC_DETECTORS", default=False) else None
|
|
1339
|
-
)
|
|
1340
|
+
detector = correction_detector if correction_detector is not None else _detect_correction_semantic
|
|
1340
1341
|
if detector is not None:
|
|
1341
1342
|
try:
|
|
1342
1343
|
if bool(detector(text)):
|
|
@@ -1375,6 +1376,8 @@ def _hint_suggests_correction(hint: str, *, correction_detector=None) -> bool:
|
|
|
1375
1376
|
|
|
1376
1377
|
def _recent_learning_capture_exists(conn, sid: str, window_seconds: int = 300) -> bool:
|
|
1377
1378
|
"""Check whether a recent learning was captured manually or via protocol task close."""
|
|
1379
|
+
if conn is None:
|
|
1380
|
+
conn = get_db()
|
|
1378
1381
|
cutoff_epoch = time.time() - window_seconds
|
|
1379
1382
|
|
|
1380
1383
|
row = conn.execute(
|