nexo-brain 7.20.21 → 7.20.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/auto_update.py +49 -19
- package/src/cli.py +21 -5
- package/src/db_guard.py +27 -9
- package/src/doctor/providers/boot.py +91 -2
- package/src/interactive_db.py +59 -0
- package/src/local_context/api.py +274 -81
- package/src/local_context/db.py +336 -0
- package/src/local_context/logging.py +3 -4
- package/src/mcp_required_tools.py +31 -0
- package/src/plugins/episodic_memory.py +18 -0
- package/src/plugins/recover.py +7 -4
- package/src/plugins/skills.py +14 -3
- package/src/plugins/update.py +37 -12
- package/src/scripts/nexo-backup.sh +131 -7
- package/src/server.py +97 -7
- package/src/tools_reminders.py +37 -8
- package/src/tools_sessions.py +11 -19
package/src/local_context/api.py
CHANGED
|
@@ -4,6 +4,7 @@ import json
|
|
|
4
4
|
import os
|
|
5
5
|
import re
|
|
6
6
|
import shutil
|
|
7
|
+
import sqlite3
|
|
7
8
|
import stat
|
|
8
9
|
import hashlib
|
|
9
10
|
import subprocess
|
|
@@ -11,10 +12,8 @@ import sys
|
|
|
11
12
|
from pathlib import Path
|
|
12
13
|
from typing import Any
|
|
13
14
|
|
|
14
|
-
from db import get_db, init_db
|
|
15
|
-
from db._schema import run_migrations
|
|
16
|
-
|
|
17
15
|
from . import embeddings
|
|
16
|
+
from .db import LOCAL_CONTEXT_TABLES, connect_local_context_db_readonly, ensure_local_context_db, get_local_context_db
|
|
18
17
|
from .extractors import chunk_text, contains_secret, entities, extract_text, summarize
|
|
19
18
|
from .logging import log_event, tail
|
|
20
19
|
from .privacy import classify_path, is_local_email_tree, is_queryable_path, should_extract, should_skip_file, should_skip_tree
|
|
@@ -42,7 +41,7 @@ VALID_CONTEXT_MODES = {"compact", "full"}
|
|
|
42
41
|
PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
43
42
|
"low": {
|
|
44
43
|
"profile": "low",
|
|
45
|
-
"
|
|
44
|
+
"label_key": "local_context.performance.low",
|
|
46
45
|
"scan_limit": 250,
|
|
47
46
|
"process_limit": 50,
|
|
48
47
|
"live_asset_limit": 500,
|
|
@@ -53,7 +52,7 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
53
52
|
},
|
|
54
53
|
"medium": {
|
|
55
54
|
"profile": "medium",
|
|
56
|
-
"
|
|
55
|
+
"label_key": "local_context.performance.medium",
|
|
57
56
|
"scan_limit": 1000,
|
|
58
57
|
"process_limit": 200,
|
|
59
58
|
"live_asset_limit": DEFAULT_LIVE_ASSET_LIMIT,
|
|
@@ -64,7 +63,7 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
64
63
|
},
|
|
65
64
|
"high": {
|
|
66
65
|
"profile": "high",
|
|
67
|
-
"
|
|
66
|
+
"label_key": "local_context.performance.high",
|
|
68
67
|
"scan_limit": 3000,
|
|
69
68
|
"process_limit": 600,
|
|
70
69
|
"live_asset_limit": 5000,
|
|
@@ -75,7 +74,7 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
75
74
|
},
|
|
76
75
|
"extreme": {
|
|
77
76
|
"profile": "extreme",
|
|
78
|
-
"
|
|
77
|
+
"label_key": "local_context.performance.extreme",
|
|
79
78
|
"scan_limit": 8000,
|
|
80
79
|
"process_limit": 1500,
|
|
81
80
|
"live_asset_limit": 10000,
|
|
@@ -88,13 +87,25 @@ PERFORMANCE_PROFILES: dict[str, dict[str, Any]] = {
|
|
|
88
87
|
|
|
89
88
|
|
|
90
89
|
def ensure_ready() -> None:
|
|
91
|
-
|
|
92
|
-
run_migrations()
|
|
90
|
+
ensure_local_context_db()
|
|
93
91
|
|
|
94
92
|
|
|
95
93
|
def _conn():
|
|
96
94
|
ensure_ready()
|
|
97
|
-
return
|
|
95
|
+
return get_local_context_db()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _read_conn():
|
|
99
|
+
conn = connect_local_context_db_readonly(timeout_ms=1200)
|
|
100
|
+
_validate_status_schema(conn)
|
|
101
|
+
return conn
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _close_read_conn(conn) -> None:
|
|
105
|
+
try:
|
|
106
|
+
conn.close()
|
|
107
|
+
except Exception:
|
|
108
|
+
pass
|
|
98
109
|
|
|
99
110
|
|
|
100
111
|
def add_root(path: str, *, mode: str = "normal", depth: int | None = None) -> dict:
|
|
@@ -139,8 +150,18 @@ def remove_root(path: str) -> dict:
|
|
|
139
150
|
return {"ok": True, "root_path": root_path, "cleanup": cleanup}
|
|
140
151
|
|
|
141
152
|
|
|
142
|
-
def list_roots() -> list[dict]:
|
|
143
|
-
|
|
153
|
+
def list_roots(*, readonly: bool = True) -> list[dict]:
|
|
154
|
+
if not readonly:
|
|
155
|
+
conn = _conn()
|
|
156
|
+
return _list_roots_conn(conn)
|
|
157
|
+
conn = _read_conn()
|
|
158
|
+
try:
|
|
159
|
+
return _list_roots_conn(conn)
|
|
160
|
+
finally:
|
|
161
|
+
_close_read_conn(conn)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _list_roots_conn(conn) -> list[dict]:
|
|
144
165
|
rows = conn.execute("SELECT * FROM local_index_roots WHERE status != 'removed' ORDER BY root_path").fetchall()
|
|
145
166
|
return [dict(row) for row in rows]
|
|
146
167
|
|
|
@@ -268,7 +289,7 @@ def default_root_specs() -> list[tuple[str, int]]:
|
|
|
268
289
|
|
|
269
290
|
|
|
270
291
|
def ensure_default_roots() -> dict:
|
|
271
|
-
existing = {row["root_path"]: row for row in list_roots()}
|
|
292
|
+
existing = {row["root_path"]: row for row in list_roots(readonly=False)}
|
|
272
293
|
created = []
|
|
273
294
|
updated = []
|
|
274
295
|
for root, depth in default_root_specs():
|
|
@@ -288,7 +309,7 @@ def ensure_default_roots() -> dict:
|
|
|
288
309
|
updated.append({"root_path": existing_row["root_path"], "depth": depth})
|
|
289
310
|
continue
|
|
290
311
|
created.append(add_root(str(candidate), mode="normal", depth=depth))
|
|
291
|
-
return {"ok": True, "created": len(created), "updated": len(updated), "roots": list_roots()}
|
|
312
|
+
return {"ok": True, "created": len(created), "updated": len(updated), "roots": list_roots(readonly=False)}
|
|
292
313
|
|
|
293
314
|
|
|
294
315
|
def _should_skip_mounted_root(candidate: Path) -> bool:
|
|
@@ -560,8 +581,18 @@ def remove_exclusion(path: str) -> dict:
|
|
|
560
581
|
return {"ok": True, "path": excluded_path}
|
|
561
582
|
|
|
562
583
|
|
|
563
|
-
def list_exclusions() -> list[dict]:
|
|
564
|
-
|
|
584
|
+
def list_exclusions(*, readonly: bool = True) -> list[dict]:
|
|
585
|
+
if not readonly:
|
|
586
|
+
conn = _conn()
|
|
587
|
+
return _list_exclusions_conn(conn)
|
|
588
|
+
conn = _read_conn()
|
|
589
|
+
try:
|
|
590
|
+
return _list_exclusions_conn(conn)
|
|
591
|
+
finally:
|
|
592
|
+
_close_read_conn(conn)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _list_exclusions_conn(conn) -> list[dict]:
|
|
565
596
|
rows = conn.execute("SELECT * FROM local_index_exclusions ORDER BY path").fetchall()
|
|
566
597
|
return [dict(row) for row in rows]
|
|
567
598
|
|
|
@@ -703,6 +734,15 @@ def _ensure_initial_index_started_at(conn) -> float:
|
|
|
703
734
|
return value
|
|
704
735
|
|
|
705
736
|
|
|
737
|
+
def _initial_index_started_at_readonly(conn) -> float:
|
|
738
|
+
raw = _get_state_conn(conn, INITIAL_INDEX_STARTED_AT_KEY, "")
|
|
739
|
+
try:
|
|
740
|
+
value = float(raw or 0)
|
|
741
|
+
except Exception:
|
|
742
|
+
value = 0.0
|
|
743
|
+
return value if value > 0 else (_earliest_index_activity(conn) or 0.0)
|
|
744
|
+
|
|
745
|
+
|
|
706
746
|
def _active_job_count(conn) -> int:
|
|
707
747
|
row = conn.execute(
|
|
708
748
|
"""
|
|
@@ -714,20 +754,20 @@ def _active_job_count(conn) -> int:
|
|
|
714
754
|
return int(row["total"] or 0)
|
|
715
755
|
|
|
716
756
|
|
|
717
|
-
def _refresh_initial_index_complete(conn, initial_scan: dict | None = None, active_jobs: int | None = None) -> bool:
|
|
757
|
+
def _refresh_initial_index_complete(conn, initial_scan: dict | None = None, active_jobs: int | None = None, *, readonly: bool = False) -> bool:
|
|
718
758
|
if _initial_index_complete(conn):
|
|
719
759
|
return True
|
|
720
760
|
scan_state = initial_scan if initial_scan is not None else _initial_scan_status(conn)
|
|
721
761
|
remaining = _active_job_count(conn) if active_jobs is None else int(active_jobs or 0)
|
|
722
762
|
complete = bool(scan_state.get("complete")) and remaining == 0
|
|
723
|
-
if complete:
|
|
763
|
+
if complete and not readonly:
|
|
724
764
|
_set_initial_index_complete(conn, True)
|
|
725
765
|
conn.commit()
|
|
726
766
|
return complete
|
|
727
767
|
|
|
728
768
|
|
|
729
769
|
def _initial_scan_status(conn, roots: list[dict] | None = None) -> dict:
|
|
730
|
-
rows = roots if roots is not None else
|
|
770
|
+
rows = roots if roots is not None else _list_roots_conn(conn)
|
|
731
771
|
tracked = _effective_scan_roots([dict(row) for row in rows if str(row.get("status") or "active") not in {"removed", "offline"}])
|
|
732
772
|
pending = [row for row in tracked if not _root_initial_scan_complete(conn, row)]
|
|
733
773
|
checkpoints = conn.execute(
|
|
@@ -756,7 +796,12 @@ def resume() -> dict:
|
|
|
756
796
|
|
|
757
797
|
|
|
758
798
|
def _is_paused() -> bool:
|
|
759
|
-
|
|
799
|
+
conn = _conn()
|
|
800
|
+
return _is_paused_conn(conn)
|
|
801
|
+
|
|
802
|
+
|
|
803
|
+
def _is_paused_conn(conn) -> bool:
|
|
804
|
+
return _get_state_conn(conn, "paused", "0") == "1"
|
|
760
805
|
|
|
761
806
|
|
|
762
807
|
def _allow_explicit_blocked_root(path: str) -> bool:
|
|
@@ -1468,7 +1513,7 @@ def reconcile_live_changes(
|
|
|
1468
1513
|
conn = _conn()
|
|
1469
1514
|
if _is_paused():
|
|
1470
1515
|
return {"ok": True, "paused": True, "assets": {}, "dirs": {}}
|
|
1471
|
-
exclusions = [row["path"] for row in list_exclusions()]
|
|
1516
|
+
exclusions = [row["path"] for row in list_exclusions(readonly=False)]
|
|
1472
1517
|
asset_stats = _reconcile_known_assets(conn, exclusions, limit=int(asset_limit or 0))
|
|
1473
1518
|
dir_stats = _reconcile_known_dirs(conn, exclusions, dir_limit=int(dir_limit or 0), file_limit=int(file_limit or 0))
|
|
1474
1519
|
conn.commit()
|
|
@@ -1501,8 +1546,8 @@ def scan_once(*, limit: int | None = None) -> dict:
|
|
|
1501
1546
|
log_event("info", "scan_skipped_paused", "Local memory scan skipped because indexing is paused")
|
|
1502
1547
|
return {"ok": True, "paused": True, "roots": 0, "seen": 0, "changed": 0, "errors": 0, "partial": False}
|
|
1503
1548
|
started = now()
|
|
1504
|
-
roots = _effective_scan_roots(list_roots())
|
|
1505
|
-
exclusions = [row["path"] for row in list_exclusions()]
|
|
1549
|
+
roots = _effective_scan_roots(list_roots(readonly=False))
|
|
1550
|
+
exclusions = [row["path"] for row in list_exclusions(readonly=False)]
|
|
1506
1551
|
totals = {"roots": len(roots), "seen": 0, "changed": 0, "errors": 0, "partial": False}
|
|
1507
1552
|
log_event("info", "scan_started", "Local memory scan started", roots=len(roots))
|
|
1508
1553
|
for root in roots:
|
|
@@ -1796,7 +1841,7 @@ def run_once(
|
|
|
1796
1841
|
effective_live_dir_limit = int(live_dir_limit if live_dir_limit is not None else config["live_dir_limit"])
|
|
1797
1842
|
effective_live_file_limit = int(live_file_limit if live_file_limit is not None else config["live_file_limit"])
|
|
1798
1843
|
conn = _conn()
|
|
1799
|
-
initial_before = _initial_scan_status(conn, list_roots())
|
|
1844
|
+
initial_before = _initial_scan_status(conn, list_roots(readonly=False))
|
|
1800
1845
|
initial_index_before = _refresh_initial_index_complete(conn, initial_before)
|
|
1801
1846
|
if initial_index_before:
|
|
1802
1847
|
live_result = reconcile_live_changes(
|
|
@@ -1815,7 +1860,7 @@ def run_once(
|
|
|
1815
1860
|
scan_result = scan_once(limit=effective_scan_limit)
|
|
1816
1861
|
job_result = process_jobs(limit=effective_process_limit)
|
|
1817
1862
|
conn_after = _conn()
|
|
1818
|
-
initial_after = _initial_scan_status(conn_after, list_roots())
|
|
1863
|
+
initial_after = _initial_scan_status(conn_after, list_roots(readonly=False))
|
|
1819
1864
|
active_after = _active_job_count(conn_after)
|
|
1820
1865
|
initial_index_after = _refresh_initial_index_complete(conn_after, initial_after, active_after)
|
|
1821
1866
|
return {
|
|
@@ -1857,8 +1902,10 @@ def _problem_rows(conn) -> list[dict]:
|
|
|
1857
1902
|
).fetchall()
|
|
1858
1903
|
problems = [
|
|
1859
1904
|
{
|
|
1860
|
-
"user_message":
|
|
1861
|
-
"
|
|
1905
|
+
"user_message": "",
|
|
1906
|
+
"message_key": "local_context.problem.file_read_failed",
|
|
1907
|
+
"recommended_action": "",
|
|
1908
|
+
"recommended_action_key": "local_context.retry_later" if row["retryable"] else "local_context.review_permissions_or_file",
|
|
1862
1909
|
"technical_detail": row["technical_detail"],
|
|
1863
1910
|
"support_code": row["error_code"],
|
|
1864
1911
|
"severity": "warning",
|
|
@@ -1885,8 +1932,10 @@ def _problem_rows(conn) -> list[dict]:
|
|
|
1885
1932
|
).fetchall()
|
|
1886
1933
|
problems.extend(
|
|
1887
1934
|
{
|
|
1888
|
-
"user_message": "
|
|
1889
|
-
"
|
|
1935
|
+
"user_message": "",
|
|
1936
|
+
"message_key": "local_context.problem.service_temporary",
|
|
1937
|
+
"recommended_action": "",
|
|
1938
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
1890
1939
|
"technical_detail": f"{row['event']}: {row['message']} {row['metadata_json']}",
|
|
1891
1940
|
"support_code": row["event"],
|
|
1892
1941
|
"severity": "warning" if row["level"] == "warn" else "error",
|
|
@@ -2087,13 +2136,13 @@ def _service_cycle_observation(conn) -> dict:
|
|
|
2087
2136
|
return observation
|
|
2088
2137
|
|
|
2089
2138
|
|
|
2090
|
-
def _index_timing(conn, *, done: int, active_jobs: int, percent: int) -> dict:
|
|
2091
|
-
first_seen = _ensure_initial_index_started_at(conn)
|
|
2139
|
+
def _index_timing(conn, *, done: int, active_jobs: int, percent: int, readonly: bool = False) -> dict:
|
|
2140
|
+
first_seen = _initial_index_started_at_readonly(conn) if readonly else _ensure_initial_index_started_at(conn)
|
|
2092
2141
|
elapsed_seconds = max(0, int(now() - float(first_seen))) if first_seen else 0
|
|
2093
2142
|
eta_seconds = None
|
|
2094
2143
|
if elapsed_seconds > 0 and done > 0 and active_jobs > 0 and 0 < percent < 100:
|
|
2095
2144
|
eta_seconds = max(0, int((elapsed_seconds / max(done, 1)) * active_jobs))
|
|
2096
|
-
return {"elapsed_seconds": elapsed_seconds, "eta_seconds": eta_seconds}
|
|
2145
|
+
return {"started_at": first_seen, "elapsed_seconds": elapsed_seconds, "eta_seconds": eta_seconds}
|
|
2097
2146
|
|
|
2098
2147
|
|
|
2099
2148
|
def _service_scheduler_has_error(service: dict) -> bool:
|
|
@@ -2110,38 +2159,118 @@ def _service_problem(service: dict) -> dict | None:
|
|
|
2110
2159
|
if not service.get("installed"):
|
|
2111
2160
|
return {
|
|
2112
2161
|
"support_code": "local_index_service_not_installed",
|
|
2113
|
-
"user_message": "
|
|
2114
|
-
"
|
|
2162
|
+
"user_message": "",
|
|
2163
|
+
"message_key": "local_context.problem.service_not_installed",
|
|
2164
|
+
"recommended_action": "",
|
|
2165
|
+
"recommended_action_key": "local_context.reopen_or_update_desktop",
|
|
2115
2166
|
"technical_detail": f"manager={service.get('manager')} platform={service.get('platform')}",
|
|
2116
2167
|
}
|
|
2117
2168
|
if not service.get("running"):
|
|
2118
2169
|
return {
|
|
2119
2170
|
"support_code": "local_index_service_not_running",
|
|
2120
|
-
"user_message": "
|
|
2121
|
-
"
|
|
2171
|
+
"user_message": "",
|
|
2172
|
+
"message_key": "local_context.problem.service_not_running",
|
|
2173
|
+
"recommended_action": "",
|
|
2174
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2122
2175
|
"technical_detail": f"manager={service.get('manager')} platform={service.get('platform')}",
|
|
2123
2176
|
}
|
|
2124
2177
|
if _service_scheduler_has_error(service):
|
|
2125
2178
|
code = service.get("last_exit_code") or service.get("last_task_result") or ""
|
|
2126
2179
|
return {
|
|
2127
2180
|
"support_code": "local_index_service_last_run_failed",
|
|
2128
|
-
"user_message": "
|
|
2129
|
-
"
|
|
2181
|
+
"user_message": "",
|
|
2182
|
+
"message_key": "local_context.problem.service_last_run_failed",
|
|
2183
|
+
"recommended_action": "",
|
|
2184
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2130
2185
|
"technical_detail": f"last_result={code}",
|
|
2131
2186
|
}
|
|
2132
2187
|
if not service.get("healthy", True):
|
|
2133
2188
|
return {
|
|
2134
2189
|
"support_code": service.get("last_error_code") or "local_index_service_failed",
|
|
2135
|
-
"user_message": "
|
|
2136
|
-
"
|
|
2190
|
+
"user_message": "",
|
|
2191
|
+
"message_key": "local_context.problem.service_temporary",
|
|
2192
|
+
"recommended_action": "",
|
|
2193
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2137
2194
|
"technical_detail": service.get("last_error_detail") or "",
|
|
2138
2195
|
}
|
|
2139
2196
|
return None
|
|
2140
2197
|
|
|
2141
2198
|
|
|
2199
|
+
def _status_read_error(exc: Exception, *, code: str = "local_context_status_unavailable") -> dict:
|
|
2200
|
+
service = _local_index_service_status()
|
|
2201
|
+
service_problem = _service_problem(service)
|
|
2202
|
+
service["healthy"] = service_problem is None
|
|
2203
|
+
service["state"] = "attention" if service_problem else "unavailable"
|
|
2204
|
+
problems = []
|
|
2205
|
+
if service_problem:
|
|
2206
|
+
problems.append({
|
|
2207
|
+
"user_message": service_problem["user_message"],
|
|
2208
|
+
"message_key": service_problem.get("message_key", ""),
|
|
2209
|
+
"recommended_action": service_problem["recommended_action"],
|
|
2210
|
+
"recommended_action_key": service_problem.get("recommended_action_key", ""),
|
|
2211
|
+
"technical_detail": service_problem["technical_detail"],
|
|
2212
|
+
"support_code": service_problem["support_code"],
|
|
2213
|
+
"severity": "warning",
|
|
2214
|
+
"retryable": True,
|
|
2215
|
+
"path": "",
|
|
2216
|
+
"phase": "service",
|
|
2217
|
+
"created_at": now(),
|
|
2218
|
+
})
|
|
2219
|
+
problems.append({
|
|
2220
|
+
"user_message": "",
|
|
2221
|
+
"message_key": "local_context.status_unavailable",
|
|
2222
|
+
"recommended_action": "",
|
|
2223
|
+
"recommended_action_key": "local_context.retry_automatic",
|
|
2224
|
+
"technical_detail": str(exc),
|
|
2225
|
+
"support_code": code,
|
|
2226
|
+
"severity": "warning",
|
|
2227
|
+
"retryable": True,
|
|
2228
|
+
"path": "",
|
|
2229
|
+
"phase": "status",
|
|
2230
|
+
"created_at": now(),
|
|
2231
|
+
})
|
|
2232
|
+
return {
|
|
2233
|
+
"ok": False,
|
|
2234
|
+
"error": code,
|
|
2235
|
+
"retryable": True,
|
|
2236
|
+
"global": None,
|
|
2237
|
+
"service": service,
|
|
2238
|
+
"problems": problems,
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
|
|
2242
|
+
def _status_db_error_code(exc: Exception) -> str:
|
|
2243
|
+
text = str(exc).lower()
|
|
2244
|
+
if "locked" in text or "busy" in text:
|
|
2245
|
+
return "local_context_db_busy"
|
|
2246
|
+
if "no such table" in text or "no such column" in text or "schema missing" in text or "missing tables" in text:
|
|
2247
|
+
return "local_context_db_schema_missing"
|
|
2248
|
+
if "file is not a database" in text or "database disk image is malformed" in text:
|
|
2249
|
+
return "local_context_db_invalid"
|
|
2250
|
+
return "local_context_db_unreadable"
|
|
2251
|
+
|
|
2252
|
+
|
|
2142
2253
|
def status() -> dict:
|
|
2143
|
-
|
|
2144
|
-
|
|
2254
|
+
try:
|
|
2255
|
+
conn = connect_local_context_db_readonly(timeout_ms=1200)
|
|
2256
|
+
except FileNotFoundError as exc:
|
|
2257
|
+
return _status_read_error(exc, code="local_context_db_missing")
|
|
2258
|
+
except sqlite3.DatabaseError as exc:
|
|
2259
|
+
return _status_read_error(exc, code=_status_db_error_code(exc))
|
|
2260
|
+
try:
|
|
2261
|
+
return _status_from_conn(conn, readonly=True)
|
|
2262
|
+
except sqlite3.DatabaseError as exc:
|
|
2263
|
+
return _status_read_error(exc, code=_status_db_error_code(exc))
|
|
2264
|
+
finally:
|
|
2265
|
+
try:
|
|
2266
|
+
conn.close()
|
|
2267
|
+
except Exception:
|
|
2268
|
+
pass
|
|
2269
|
+
|
|
2270
|
+
|
|
2271
|
+
def _status_from_conn(conn, *, readonly: bool = False) -> dict:
|
|
2272
|
+
_validate_status_schema(conn)
|
|
2273
|
+
paused = _is_paused_conn(conn)
|
|
2145
2274
|
assets = conn.execute(
|
|
2146
2275
|
"""
|
|
2147
2276
|
SELECT COUNT(*) AS total, SUM(CASE WHEN a.status='active' THEN 1 ELSE 0 END) AS active
|
|
@@ -2169,10 +2298,10 @@ def status() -> dict:
|
|
|
2169
2298
|
active_jobs = pending + running_jobs + failed_jobs
|
|
2170
2299
|
total_jobs = active_jobs + done
|
|
2171
2300
|
percent = 100 if total_jobs == 0 else int((done / max(total_jobs, 1)) * 100)
|
|
2172
|
-
timing = _index_timing(conn, done=done, active_jobs=active_jobs, percent=percent)
|
|
2173
|
-
roots =
|
|
2301
|
+
timing = _index_timing(conn, done=done, active_jobs=active_jobs, percent=percent, readonly=readonly)
|
|
2302
|
+
roots = _list_roots_conn(conn)
|
|
2174
2303
|
initial_scan = _initial_scan_status(conn, roots)
|
|
2175
|
-
initial_index_complete = _refresh_initial_index_complete(conn, initial_scan, active_jobs)
|
|
2304
|
+
initial_index_complete = _refresh_initial_index_complete(conn, initial_scan, active_jobs, readonly=readonly)
|
|
2176
2305
|
volumes = []
|
|
2177
2306
|
by_volume = conn.execute(
|
|
2178
2307
|
"""
|
|
@@ -2197,7 +2326,9 @@ def status() -> dict:
|
|
|
2197
2326
|
if problem:
|
|
2198
2327
|
problems.insert(0, {
|
|
2199
2328
|
"user_message": problem["user_message"],
|
|
2329
|
+
"message_key": problem.get("message_key", ""),
|
|
2200
2330
|
"recommended_action": problem["recommended_action"],
|
|
2331
|
+
"recommended_action_key": problem.get("recommended_action_key", ""),
|
|
2201
2332
|
"technical_detail": problem["technical_detail"],
|
|
2202
2333
|
"support_code": problem["support_code"],
|
|
2203
2334
|
"severity": "warning",
|
|
@@ -2216,6 +2347,9 @@ def status() -> dict:
|
|
|
2216
2347
|
phase = "idle"
|
|
2217
2348
|
else:
|
|
2218
2349
|
phase = "updating_changes"
|
|
2350
|
+
index_started_at = _get_state_conn(conn, INITIAL_INDEX_STARTED_AT_KEY, "")
|
|
2351
|
+
if not index_started_at and timing["started_at"]:
|
|
2352
|
+
index_started_at = str(float(timing["started_at"]))
|
|
2219
2353
|
return {
|
|
2220
2354
|
"ok": True,
|
|
2221
2355
|
"service": service,
|
|
@@ -2230,7 +2364,7 @@ def status() -> dict:
|
|
|
2230
2364
|
"jobs_failed": failed_jobs,
|
|
2231
2365
|
"elapsed_seconds": timing["elapsed_seconds"],
|
|
2232
2366
|
"eta_seconds": timing["eta_seconds"],
|
|
2233
|
-
"index_started_at":
|
|
2367
|
+
"index_started_at": index_started_at,
|
|
2234
2368
|
"initial_scan_complete": bool(initial_index_complete),
|
|
2235
2369
|
"initial_discovery_complete": bool(initial_scan["complete"]),
|
|
2236
2370
|
"initial_index_complete": bool(initial_index_complete),
|
|
@@ -2242,7 +2376,7 @@ def status() -> dict:
|
|
|
2242
2376
|
"initial_index_complete": bool(initial_index_complete),
|
|
2243
2377
|
"volumes": volumes,
|
|
2244
2378
|
"roots": roots,
|
|
2245
|
-
"exclusions":
|
|
2379
|
+
"exclusions": _list_exclusions_conn(conn),
|
|
2246
2380
|
"problems": problems,
|
|
2247
2381
|
"permissions": [],
|
|
2248
2382
|
"models": model_status()["models"],
|
|
@@ -2250,6 +2384,18 @@ def status() -> dict:
|
|
|
2250
2384
|
}
|
|
2251
2385
|
|
|
2252
2386
|
|
|
2387
|
+
def _validate_status_schema(conn) -> None:
|
|
2388
|
+
placeholders = ",".join("?" for _ in LOCAL_CONTEXT_TABLES)
|
|
2389
|
+
rows = conn.execute(
|
|
2390
|
+
f"SELECT name FROM sqlite_master WHERE type='table' AND name IN ({placeholders})",
|
|
2391
|
+
tuple(LOCAL_CONTEXT_TABLES),
|
|
2392
|
+
).fetchall()
|
|
2393
|
+
found = {str(row["name"] if isinstance(row, sqlite3.Row) else row[0]) for row in rows}
|
|
2394
|
+
missing = [table for table in LOCAL_CONTEXT_TABLES if table not in found]
|
|
2395
|
+
if missing:
|
|
2396
|
+
raise sqlite3.OperationalError("local context schema missing tables: " + ", ".join(missing[:8]))
|
|
2397
|
+
|
|
2398
|
+
|
|
2253
2399
|
def diagnostics_tail(limit: int = 100) -> dict:
|
|
2254
2400
|
return {"ok": True, "logs": tail(limit)}
|
|
2255
2401
|
|
|
@@ -2771,8 +2917,46 @@ def context_query(
|
|
|
2771
2917
|
include_entities: bool = True,
|
|
2772
2918
|
include_relations: bool = True,
|
|
2773
2919
|
snippet_chars: int = 1200,
|
|
2920
|
+
readonly: bool = True,
|
|
2921
|
+
record_query: bool = False,
|
|
2922
|
+
) -> dict:
|
|
2923
|
+
conn = _read_conn() if readonly else _conn()
|
|
2924
|
+
close_conn = bool(readonly)
|
|
2925
|
+
try:
|
|
2926
|
+
return _context_query_conn(
|
|
2927
|
+
conn,
|
|
2928
|
+
query,
|
|
2929
|
+
intent=intent,
|
|
2930
|
+
limit=limit,
|
|
2931
|
+
evidence_required=evidence_required,
|
|
2932
|
+
current_context=current_context,
|
|
2933
|
+
mode=mode,
|
|
2934
|
+
max_chars=max_chars,
|
|
2935
|
+
include_entities=include_entities,
|
|
2936
|
+
include_relations=include_relations,
|
|
2937
|
+
snippet_chars=snippet_chars,
|
|
2938
|
+
record_query=bool(record_query and not readonly),
|
|
2939
|
+
)
|
|
2940
|
+
finally:
|
|
2941
|
+
if close_conn:
|
|
2942
|
+
_close_read_conn(conn)
|
|
2943
|
+
|
|
2944
|
+
|
|
2945
|
+
def _context_query_conn(
|
|
2946
|
+
conn,
|
|
2947
|
+
query: str,
|
|
2948
|
+
*,
|
|
2949
|
+
intent: str,
|
|
2950
|
+
limit: int,
|
|
2951
|
+
evidence_required: bool,
|
|
2952
|
+
current_context: str,
|
|
2953
|
+
mode: str,
|
|
2954
|
+
max_chars: int,
|
|
2955
|
+
include_entities: bool,
|
|
2956
|
+
include_relations: bool,
|
|
2957
|
+
snippet_chars: int,
|
|
2958
|
+
record_query: bool,
|
|
2774
2959
|
) -> dict:
|
|
2775
|
-
conn = _conn()
|
|
2776
2960
|
clean_query = str(query or "").strip()
|
|
2777
2961
|
normalized_mode, mode_warnings = _normalize_context_mode(mode)
|
|
2778
2962
|
context_tail = _compact_text(current_context or "", max_chars=1000)
|
|
@@ -2846,21 +3030,22 @@ def context_query(
|
|
|
2846
3030
|
summary = ""
|
|
2847
3031
|
if assets:
|
|
2848
3032
|
summary = f"Found {len(assets)} local asset(s) related to '{clean_query}'."
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
3033
|
+
if record_query:
|
|
3034
|
+
conn.execute(
|
|
3035
|
+
"""
|
|
3036
|
+
INSERT INTO local_context_queries(query_hash, intent, result_count, confidence, warnings_json, created_at)
|
|
3037
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
3038
|
+
""",
|
|
3039
|
+
(
|
|
3040
|
+
hashlib.sha256(clean_query.encode("utf-8", errors="ignore")).hexdigest(),
|
|
3041
|
+
intent,
|
|
3042
|
+
len(assets),
|
|
3043
|
+
0.75 if evidence_refs else 0.0,
|
|
3044
|
+
json_dumps(warnings),
|
|
3045
|
+
now(),
|
|
3046
|
+
),
|
|
3047
|
+
)
|
|
3048
|
+
conn.commit()
|
|
2864
3049
|
payload = {
|
|
2865
3050
|
"ok": True,
|
|
2866
3051
|
"query": clean_query,
|
|
@@ -2884,26 +3069,34 @@ def context_query(
|
|
|
2884
3069
|
)
|
|
2885
3070
|
|
|
2886
3071
|
|
|
2887
|
-
def get_asset(asset_id: str) -> dict:
|
|
2888
|
-
conn = _conn()
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
3072
|
+
def get_asset(asset_id: str, *, readonly: bool = True) -> dict:
|
|
3073
|
+
conn = _read_conn() if readonly else _conn()
|
|
3074
|
+
try:
|
|
3075
|
+
row = conn.execute("SELECT * FROM local_assets WHERE asset_id=?", (asset_id,)).fetchone()
|
|
3076
|
+
if not row:
|
|
3077
|
+
return {"ok": False, "error": "asset_not_found"}
|
|
3078
|
+
return {"ok": True, "asset": dict(row)}
|
|
3079
|
+
finally:
|
|
3080
|
+
if readonly:
|
|
3081
|
+
_close_read_conn(conn)
|
|
2893
3082
|
|
|
2894
3083
|
|
|
2895
|
-
def get_neighbors(asset_id: str, *, limit: int = 30) -> dict:
|
|
2896
|
-
conn = _conn()
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
2900
|
-
|
|
2901
|
-
|
|
2902
|
-
|
|
2903
|
-
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
|
|
3084
|
+
def get_neighbors(asset_id: str, *, limit: int = 30, readonly: bool = True) -> dict:
|
|
3085
|
+
conn = _read_conn() if readonly else _conn()
|
|
3086
|
+
try:
|
|
3087
|
+
rows = conn.execute(
|
|
3088
|
+
"""
|
|
3089
|
+
SELECT * FROM local_relations
|
|
3090
|
+
WHERE source_asset_id=? AND active=1
|
|
3091
|
+
ORDER BY confidence DESC
|
|
3092
|
+
LIMIT ?
|
|
3093
|
+
""",
|
|
3094
|
+
(asset_id, int(limit)),
|
|
3095
|
+
).fetchall()
|
|
3096
|
+
return {"ok": True, "relations": [dict(row) for row in rows]}
|
|
3097
|
+
finally:
|
|
3098
|
+
if readonly:
|
|
3099
|
+
_close_read_conn(conn)
|
|
2907
3100
|
|
|
2908
3101
|
|
|
2909
3102
|
def purge_asset(asset_id: str) -> dict:
|