nexo-brain 7.19.0 → 7.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/cli.py +21 -1
- package/src/db/_schema.py +31 -0
- package/src/local_context/__init__.py +2 -0
- package/src/local_context/api.py +393 -5
- package/src/runtime_power.py +14 -1
- package/src/scripts/nexo-local-index.py +10 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.20.0",
|
|
4
4
|
"description": "Local cognitive runtime for Claude Code \u2014 persistent memory, overnight learning, doctor diagnostics, personal scripts, recovery-aware jobs, startup preflight, and optional dashboard/power helper.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "NEXO Brain",
|
package/README.md
CHANGED
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
|
|
19
19
|
[Watch the overview video](https://nexo-brain.com/watch/) · [Watch on YouTube](https://www.youtube.com/watch?v=i2lkGhKyVqI) · [Open the infographic](https://nexo-brain.com/assets/nexo-brain-infographic-v5.png)
|
|
20
20
|
|
|
21
|
-
Version `7.
|
|
21
|
+
Version `7.20.0` is the current packaged-runtime line. Minor release over v7.19.0 — the Local Context index now reconciles known files and folders on every service cycle, so created, modified, deleted and newly excluded local files are reflected automatically between full scans.
|
|
22
|
+
|
|
23
|
+
Previously in `7.19.0`: minor release over v7.18.1 - bundle-managed installations (NEXO Desktop `brain-bundle/`) can now pin Brain to the host application release cycle via `NEXO_BRAIN_AUTO_UPDATE=false`, and the server auto-exits with code 75 on fingerprint mismatch so MCP clients respawn the server with the new code instead of leaving stale `server.py` processes alive.
|
|
22
24
|
|
|
23
25
|
Previously in `7.18.1`: patch release over v7.18.0 - packaged Brain runtimes now include the `local_context` package, so Desktop Local Memory and `nexo local-context` do not get stuck behind `ModuleNotFoundError` or zero-file status; the local-index service also keeps detecting newly mounted volumes automatically.
|
|
24
26
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.20.0",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/cli.py
CHANGED
|
@@ -20,7 +20,7 @@ Entry points:
|
|
|
20
20
|
nexo scripts run NAME_OR_PATH [-- args...]
|
|
21
21
|
nexo scripts doctor [NAME_OR_PATH] [--json]
|
|
22
22
|
nexo scripts call TOOL --input JSON [--json-output]
|
|
23
|
-
nexo local-context status|run-once|pause|resume|roots|exclusions|query|diagnostics|models [--json]
|
|
23
|
+
nexo local-context status|run-once|reconcile|pause|resume|roots|exclusions|query|diagnostics|models [--json]
|
|
24
24
|
nexo automations reactivate NAME [--test-run] [--json]
|
|
25
25
|
nexo skills list [--level ...] [--source-kind ...] [--json]
|
|
26
26
|
nexo skills get ID [--json]
|
|
@@ -1291,6 +1291,18 @@ def _local_context_run_once(args) -> int:
|
|
|
1291
1291
|
)
|
|
1292
1292
|
|
|
1293
1293
|
|
|
1294
|
+
def _local_context_reconcile(args) -> int:
|
|
1295
|
+
import local_context
|
|
1296
|
+
return _local_context_emit(
|
|
1297
|
+
local_context.reconcile_live_changes(
|
|
1298
|
+
asset_limit=int(getattr(args, "asset_limit", 2000) or 0),
|
|
1299
|
+
dir_limit=int(getattr(args, "dir_limit", 300) or 0),
|
|
1300
|
+
file_limit=int(getattr(args, "file_limit", 1000) or 0),
|
|
1301
|
+
),
|
|
1302
|
+
args,
|
|
1303
|
+
)
|
|
1304
|
+
|
|
1305
|
+
|
|
1294
1306
|
def _local_context_control(args) -> int:
|
|
1295
1307
|
import local_context
|
|
1296
1308
|
command = str(getattr(args, "local_context_command", "") or "")
|
|
@@ -3154,6 +3166,12 @@ def main():
|
|
|
3154
3166
|
local_context_run_p.add_argument("--process-limit", type=int, default=100, help="Maximum extraction/graph jobs to process")
|
|
3155
3167
|
local_context_run_p.add_argument("--json", action="store_true", help="JSON output")
|
|
3156
3168
|
|
|
3169
|
+
local_context_reconcile_p = local_context_sub.add_parser("reconcile", help="Reconcile changed, deleted and new local files")
|
|
3170
|
+
local_context_reconcile_p.add_argument("--asset-limit", type=int, default=2000, help="Maximum known files to verify")
|
|
3171
|
+
local_context_reconcile_p.add_argument("--dir-limit", type=int, default=300, help="Maximum known folders to verify")
|
|
3172
|
+
local_context_reconcile_p.add_argument("--file-limit", type=int, default=1000, help="Maximum files to scan inside changed folders")
|
|
3173
|
+
local_context_reconcile_p.add_argument("--json", action="store_true", help="JSON output")
|
|
3174
|
+
|
|
3157
3175
|
local_context_pause_p = local_context_sub.add_parser("pause", help="Pause local memory indexing")
|
|
3158
3176
|
local_context_pause_p.add_argument("--json", action="store_true", help="JSON output")
|
|
3159
3177
|
|
|
@@ -3734,6 +3752,8 @@ def main():
|
|
|
3734
3752
|
return _local_context_status(args)
|
|
3735
3753
|
if args.local_context_command == "run-once":
|
|
3736
3754
|
return _local_context_run_once(args)
|
|
3755
|
+
if args.local_context_command == "reconcile":
|
|
3756
|
+
return _local_context_reconcile(args)
|
|
3737
3757
|
if args.local_context_command in {"pause", "resume", "clear-index"}:
|
|
3738
3758
|
return _local_context_control(args)
|
|
3739
3759
|
if args.local_context_command == "roots":
|
package/src/db/_schema.py
CHANGED
|
@@ -1972,6 +1972,36 @@ def _m63_local_context_layer(conn):
|
|
|
1972
1972
|
)
|
|
1973
1973
|
|
|
1974
1974
|
|
|
1975
|
+
def _m64_local_context_live_dirs(conn):
|
|
1976
|
+
"""Track known folders so local context can detect new/deleted/changed files quickly."""
|
|
1977
|
+
conn.executescript(
|
|
1978
|
+
"""
|
|
1979
|
+
CREATE TABLE IF NOT EXISTS local_index_dirs (
|
|
1980
|
+
dir_id TEXT PRIMARY KEY,
|
|
1981
|
+
root_id INTEGER,
|
|
1982
|
+
path TEXT NOT NULL UNIQUE,
|
|
1983
|
+
display_path TEXT NOT NULL,
|
|
1984
|
+
parent_path TEXT NOT NULL DEFAULT '',
|
|
1985
|
+
quick_fingerprint TEXT NOT NULL DEFAULT '',
|
|
1986
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1987
|
+
first_seen_at REAL NOT NULL,
|
|
1988
|
+
last_seen_at REAL NOT NULL,
|
|
1989
|
+
updated_at REAL NOT NULL,
|
|
1990
|
+
deleted_at REAL
|
|
1991
|
+
);
|
|
1992
|
+
|
|
1993
|
+
CREATE INDEX IF NOT EXISTS idx_local_index_dirs_root_status
|
|
1994
|
+
ON local_index_dirs(root_id, status);
|
|
1995
|
+
CREATE INDEX IF NOT EXISTS idx_local_index_dirs_path
|
|
1996
|
+
ON local_index_dirs(path);
|
|
1997
|
+
CREATE INDEX IF NOT EXISTS idx_local_index_dirs_parent
|
|
1998
|
+
ON local_index_dirs(parent_path);
|
|
1999
|
+
CREATE INDEX IF NOT EXISTS idx_local_assets_updated
|
|
2000
|
+
ON local_assets(updated_at);
|
|
2001
|
+
"""
|
|
2002
|
+
)
|
|
2003
|
+
|
|
2004
|
+
|
|
1975
2005
|
MIGRATIONS = [
|
|
1976
2006
|
(1, "learnings_columns", _m1_learnings_columns),
|
|
1977
2007
|
(2, "followups_reasoning", _m2_followups_reasoning),
|
|
@@ -2036,6 +2066,7 @@ MIGRATIONS = [
|
|
|
2036
2066
|
(61, "memory_observations_fts", _m61_memory_observations_fts),
|
|
2037
2067
|
(62, "memory_observations_fts_trigger_fix", _m62_memory_observations_fts_trigger_fix),
|
|
2038
2068
|
(63, "local_context_layer", _m63_local_context_layer),
|
|
2069
|
+
(64, "local_context_live_dirs", _m64_local_context_live_dirs),
|
|
2039
2070
|
]
|
|
2040
2071
|
|
|
2041
2072
|
|
|
@@ -19,6 +19,7 @@ from .api import (
|
|
|
19
19
|
model_status,
|
|
20
20
|
pause,
|
|
21
21
|
purge_asset,
|
|
22
|
+
reconcile_live_changes,
|
|
22
23
|
remove_exclusion,
|
|
23
24
|
remove_root,
|
|
24
25
|
resume,
|
|
@@ -41,6 +42,7 @@ __all__ = [
|
|
|
41
42
|
"model_status",
|
|
42
43
|
"pause",
|
|
43
44
|
"purge_asset",
|
|
45
|
+
"reconcile_live_changes",
|
|
44
46
|
"remove_exclusion",
|
|
45
47
|
"remove_root",
|
|
46
48
|
"resume",
|
package/src/local_context/api.py
CHANGED
|
@@ -23,6 +23,9 @@ LOCAL_INDEX_SERVICE_LABEL = "com.nexo.local-index"
|
|
|
23
23
|
LOCAL_INDEX_SCRIPT_NAME = "nexo-local-index.py"
|
|
24
24
|
LOCAL_INDEX_WINDOWS_TASK = "NEXO Local Memory"
|
|
25
25
|
LOCAL_INDEX_LINUX_UNIT = "nexo-local-index.service"
|
|
26
|
+
DEFAULT_LIVE_ASSET_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_ASSET_LIMIT", "2000") or "2000")
|
|
27
|
+
DEFAULT_LIVE_DIR_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_DIR_LIMIT", "300") or "300")
|
|
28
|
+
DEFAULT_LIVE_FILE_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_FILE_LIMIT", "1000") or "1000")
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
def ensure_ready() -> None:
|
|
@@ -205,6 +208,11 @@ def _is_excluded(path: str, exclusions: list[str]) -> bool:
|
|
|
205
208
|
return any(value == item or value.startswith(item + os.sep) for item in exclusions)
|
|
206
209
|
|
|
207
210
|
|
|
211
|
+
def _path_prefix(path: str) -> str:
|
|
212
|
+
normalized = norm_path(path)
|
|
213
|
+
return normalized + os.sep if normalized else os.sep
|
|
214
|
+
|
|
215
|
+
|
|
208
216
|
def _file_type(path: Path) -> str:
|
|
209
217
|
if path.is_dir():
|
|
210
218
|
return "folder"
|
|
@@ -232,6 +240,65 @@ def _permission_state(path: Path) -> str:
|
|
|
232
240
|
return "granted"
|
|
233
241
|
|
|
234
242
|
|
|
243
|
+
def _dir_fingerprint(path: Path, stat_result: os.stat_result | None = None) -> str:
|
|
244
|
+
st = stat_result or path.stat()
|
|
245
|
+
ctime_ns = getattr(st, "st_ctime_ns", int(float(st.st_ctime) * 1_000_000_000))
|
|
246
|
+
return f"{int(st.st_mtime_ns)}:{int(ctime_ns)}"
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _upsert_dir(
|
|
250
|
+
conn,
|
|
251
|
+
root_id: int,
|
|
252
|
+
path: Path,
|
|
253
|
+
seen_at: float,
|
|
254
|
+
stat_result: os.stat_result | None = None,
|
|
255
|
+
) -> tuple[bool, str]:
|
|
256
|
+
raw_path = str(path)
|
|
257
|
+
normalized = norm_path(raw_path)
|
|
258
|
+
dir_id = stable_id("dir", normalized)
|
|
259
|
+
parent = norm_path(path.parent)
|
|
260
|
+
try:
|
|
261
|
+
fingerprint = _dir_fingerprint(path, stat_result)
|
|
262
|
+
except Exception:
|
|
263
|
+
return False, "error"
|
|
264
|
+
row = conn.execute(
|
|
265
|
+
"SELECT quick_fingerprint, status FROM local_index_dirs WHERE dir_id=?",
|
|
266
|
+
(dir_id,),
|
|
267
|
+
).fetchone()
|
|
268
|
+
changed = not row or row["quick_fingerprint"] != fingerprint or row["status"] == "deleted"
|
|
269
|
+
conn.execute(
|
|
270
|
+
"""
|
|
271
|
+
INSERT INTO local_index_dirs(
|
|
272
|
+
dir_id, root_id, path, display_path, parent_path, quick_fingerprint,
|
|
273
|
+
status, first_seen_at, last_seen_at, updated_at, deleted_at
|
|
274
|
+
)
|
|
275
|
+
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, NULL)
|
|
276
|
+
ON CONFLICT(dir_id) DO UPDATE SET
|
|
277
|
+
root_id=excluded.root_id,
|
|
278
|
+
path=excluded.path,
|
|
279
|
+
display_path=excluded.display_path,
|
|
280
|
+
parent_path=excluded.parent_path,
|
|
281
|
+
quick_fingerprint=excluded.quick_fingerprint,
|
|
282
|
+
status='active',
|
|
283
|
+
last_seen_at=excluded.last_seen_at,
|
|
284
|
+
updated_at=excluded.updated_at,
|
|
285
|
+
deleted_at=NULL
|
|
286
|
+
""",
|
|
287
|
+
(
|
|
288
|
+
dir_id,
|
|
289
|
+
root_id,
|
|
290
|
+
normalized,
|
|
291
|
+
raw_path,
|
|
292
|
+
parent,
|
|
293
|
+
fingerprint,
|
|
294
|
+
seen_at,
|
|
295
|
+
seen_at,
|
|
296
|
+
seen_at,
|
|
297
|
+
),
|
|
298
|
+
)
|
|
299
|
+
return changed, fingerprint
|
|
300
|
+
|
|
301
|
+
|
|
235
302
|
def _upsert_asset(conn, root_id: int, path: Path, seen_at: float, root_depth: int) -> tuple[str, bool, str]:
|
|
236
303
|
raw_path = str(path)
|
|
237
304
|
normalized = norm_path(raw_path)
|
|
@@ -321,6 +388,43 @@ def _upsert_asset(conn, root_id: int, path: Path, seen_at: float, root_depth: in
|
|
|
321
388
|
return asset_id, changed, "ok"
|
|
322
389
|
|
|
323
390
|
|
|
391
|
+
def _mark_asset_deleted(conn, asset_id: str, deleted_at: float | None = None) -> None:
|
|
392
|
+
deleted_at = deleted_at or now()
|
|
393
|
+
conn.execute(
|
|
394
|
+
"UPDATE local_assets SET status='deleted', deleted_at=?, updated_at=? WHERE asset_id=? AND status!='deleted'",
|
|
395
|
+
(deleted_at, deleted_at, asset_id),
|
|
396
|
+
)
|
|
397
|
+
conn.execute(
|
|
398
|
+
"""
|
|
399
|
+
UPDATE local_index_jobs
|
|
400
|
+
SET status='done', last_error_code='asset_deleted', updated_at=?
|
|
401
|
+
WHERE asset_id=? AND status IN ('pending', 'running')
|
|
402
|
+
""",
|
|
403
|
+
(deleted_at, asset_id),
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _mark_dir_subtree_deleted(conn, dir_path: str, deleted_at: float | None = None) -> int:
|
|
408
|
+
deleted_at = deleted_at or now()
|
|
409
|
+
normalized = norm_path(dir_path)
|
|
410
|
+
prefix = _path_prefix(normalized)
|
|
411
|
+
conn.execute(
|
|
412
|
+
"""
|
|
413
|
+
UPDATE local_index_dirs
|
|
414
|
+
SET status='deleted', deleted_at=?, updated_at=?
|
|
415
|
+
WHERE status='active' AND (path=? OR path LIKE ?)
|
|
416
|
+
""",
|
|
417
|
+
(deleted_at, deleted_at, normalized, prefix + "%"),
|
|
418
|
+
)
|
|
419
|
+
rows = conn.execute(
|
|
420
|
+
"SELECT asset_id FROM local_assets WHERE status='active' AND (path=? OR path LIKE ?)",
|
|
421
|
+
(normalized, prefix + "%"),
|
|
422
|
+
).fetchall()
|
|
423
|
+
for row in rows:
|
|
424
|
+
_mark_asset_deleted(conn, row["asset_id"], deleted_at)
|
|
425
|
+
return len(rows)
|
|
426
|
+
|
|
427
|
+
|
|
324
428
|
def enqueue_job(conn, asset_id: str, job_type: str, *, priority: int = 50) -> str:
|
|
325
429
|
job_id = stable_id("job", f"{asset_id}:{job_type}")
|
|
326
430
|
conn.execute(
|
|
@@ -334,7 +438,17 @@ def enqueue_job(conn, asset_id: str, job_type: str, *, priority: int = 50) -> st
|
|
|
334
438
|
return job_id
|
|
335
439
|
|
|
336
440
|
|
|
337
|
-
def _iter_files(
|
|
441
|
+
def _iter_files(
|
|
442
|
+
conn,
|
|
443
|
+
root_id: int,
|
|
444
|
+
root: Path,
|
|
445
|
+
exclusions: list[str],
|
|
446
|
+
*,
|
|
447
|
+
limit: int | None = None,
|
|
448
|
+
start_after: str = "",
|
|
449
|
+
seen_at: float | None = None,
|
|
450
|
+
):
|
|
451
|
+
seen_at = seen_at or now()
|
|
338
452
|
seen_dirs: set[tuple[int, int]] = set()
|
|
339
453
|
count = 0
|
|
340
454
|
stack = [root]
|
|
@@ -353,6 +467,7 @@ def _iter_files(root: Path, exclusions: list[str], *, limit: int | None = None,
|
|
|
353
467
|
if key in seen_dirs:
|
|
354
468
|
continue
|
|
355
469
|
seen_dirs.add(key)
|
|
470
|
+
_upsert_dir(conn, root_id, current, seen_at, st)
|
|
356
471
|
try:
|
|
357
472
|
entries = sorted(current.iterdir(), key=lambda item: str(item).lower())
|
|
358
473
|
except Exception:
|
|
@@ -427,6 +542,242 @@ def _clear_checkpoint(conn, root_id: int) -> None:
|
|
|
427
542
|
conn.execute("DELETE FROM local_index_checkpoints WHERE root_id=? AND phase='quick_index'", (root_id,))
|
|
428
543
|
|
|
429
544
|
|
|
545
|
+
def _reconcile_known_assets(conn, exclusions: list[str], *, limit: int) -> dict:
|
|
546
|
+
stats = {"checked": 0, "modified": 0, "deleted": 0, "excluded": 0, "offline": 0, "errors": 0}
|
|
547
|
+
if limit <= 0:
|
|
548
|
+
return stats
|
|
549
|
+
rows = conn.execute(
|
|
550
|
+
"""
|
|
551
|
+
SELECT a.asset_id, a.path, a.root_id, a.quick_fingerprint, a.depth, r.root_path
|
|
552
|
+
FROM local_assets a
|
|
553
|
+
LEFT JOIN local_index_roots r ON r.id = a.root_id
|
|
554
|
+
WHERE a.status='active'
|
|
555
|
+
ORDER BY a.updated_at ASC
|
|
556
|
+
LIMIT ?
|
|
557
|
+
""",
|
|
558
|
+
(int(limit),),
|
|
559
|
+
).fetchall()
|
|
560
|
+
seen_at = now()
|
|
561
|
+
for row in rows:
|
|
562
|
+
stats["checked"] += 1
|
|
563
|
+
path = str(row["path"])
|
|
564
|
+
root_path = Path(row["root_path"]).expanduser() if row["root_path"] else None
|
|
565
|
+
if _is_excluded(path, exclusions):
|
|
566
|
+
_mark_asset_deleted(conn, row["asset_id"], seen_at)
|
|
567
|
+
stats["excluded"] += 1
|
|
568
|
+
continue
|
|
569
|
+
if root_path is not None and not root_path.exists():
|
|
570
|
+
stats["offline"] += 1
|
|
571
|
+
continue
|
|
572
|
+
file_path = Path(path)
|
|
573
|
+
try:
|
|
574
|
+
if not file_path.exists() or not file_path.is_file():
|
|
575
|
+
_mark_asset_deleted(conn, row["asset_id"], seen_at)
|
|
576
|
+
stats["deleted"] += 1
|
|
577
|
+
continue
|
|
578
|
+
st = file_path.stat()
|
|
579
|
+
fingerprint = quick_fingerprint(file_path, st)
|
|
580
|
+
except Exception:
|
|
581
|
+
stats["errors"] += 1
|
|
582
|
+
continue
|
|
583
|
+
if fingerprint != row["quick_fingerprint"]:
|
|
584
|
+
_, changed, state = _upsert_asset(conn, int(row["root_id"] or 0), file_path, seen_at, int(row["depth"] or 2))
|
|
585
|
+
if changed:
|
|
586
|
+
stats["modified"] += 1
|
|
587
|
+
if state != "ok":
|
|
588
|
+
stats["errors"] += 1
|
|
589
|
+
else:
|
|
590
|
+
conn.execute("UPDATE local_assets SET updated_at=? WHERE asset_id=?", (seen_at, row["asset_id"]))
|
|
591
|
+
return stats
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _prune_missing_children(
|
|
595
|
+
conn,
|
|
596
|
+
directory: Path,
|
|
597
|
+
seen_files: set[str],
|
|
598
|
+
seen_dirs: set[str],
|
|
599
|
+
seen_at: float,
|
|
600
|
+
) -> tuple[int, int]:
|
|
601
|
+
parent = norm_path(directory)
|
|
602
|
+
deleted_files = 0
|
|
603
|
+
deleted_dirs = 0
|
|
604
|
+
file_rows = conn.execute(
|
|
605
|
+
"SELECT asset_id, path FROM local_assets WHERE parent_path=? AND status='active'",
|
|
606
|
+
(parent,),
|
|
607
|
+
).fetchall()
|
|
608
|
+
for row in file_rows:
|
|
609
|
+
if row["path"] not in seen_files:
|
|
610
|
+
_mark_asset_deleted(conn, row["asset_id"], seen_at)
|
|
611
|
+
deleted_files += 1
|
|
612
|
+
dir_rows = conn.execute(
|
|
613
|
+
"SELECT path FROM local_index_dirs WHERE parent_path=? AND status='active'",
|
|
614
|
+
(parent,),
|
|
615
|
+
).fetchall()
|
|
616
|
+
for row in dir_rows:
|
|
617
|
+
if row["path"] not in seen_dirs:
|
|
618
|
+
deleted_files += _mark_dir_subtree_deleted(conn, row["path"], seen_at)
|
|
619
|
+
deleted_dirs += 1
|
|
620
|
+
return deleted_files, deleted_dirs
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _scan_known_directory(
|
|
624
|
+
conn,
|
|
625
|
+
root_id: int,
|
|
626
|
+
directory: Path,
|
|
627
|
+
root_depth: int,
|
|
628
|
+
exclusions: list[str],
|
|
629
|
+
stats: dict,
|
|
630
|
+
*,
|
|
631
|
+
file_limit: int,
|
|
632
|
+
dir_limit: int,
|
|
633
|
+
) -> None:
|
|
634
|
+
stack = [directory]
|
|
635
|
+
seen_at = now()
|
|
636
|
+
scanned_dirs = 0
|
|
637
|
+
while stack and stats["files_scanned"] < file_limit and scanned_dirs < dir_limit:
|
|
638
|
+
current = stack.pop()
|
|
639
|
+
if _is_excluded(str(current), exclusions):
|
|
640
|
+
_mark_dir_subtree_deleted(conn, str(current), seen_at)
|
|
641
|
+
stats["excluded_dirs"] += 1
|
|
642
|
+
continue
|
|
643
|
+
if current != directory and should_skip_tree(str(current)):
|
|
644
|
+
continue
|
|
645
|
+
try:
|
|
646
|
+
st = current.stat()
|
|
647
|
+
if not current.is_dir():
|
|
648
|
+
continue
|
|
649
|
+
entries = sorted(current.iterdir(), key=lambda item: str(item).lower())
|
|
650
|
+
except Exception:
|
|
651
|
+
stats["errors"] += 1
|
|
652
|
+
continue
|
|
653
|
+
scanned_dirs += 1
|
|
654
|
+
stats["dirs_scanned"] += 1
|
|
655
|
+
_upsert_dir(conn, root_id, current, seen_at, st)
|
|
656
|
+
seen_files: set[str] = set()
|
|
657
|
+
seen_dirs: set[str] = set()
|
|
658
|
+
for entry in entries:
|
|
659
|
+
if _is_excluded(str(entry), exclusions):
|
|
660
|
+
continue
|
|
661
|
+
try:
|
|
662
|
+
if entry.is_symlink():
|
|
663
|
+
continue
|
|
664
|
+
if entry.is_dir():
|
|
665
|
+
if should_skip_tree(str(entry)):
|
|
666
|
+
continue
|
|
667
|
+
changed, _ = _upsert_dir(conn, root_id, entry, seen_at)
|
|
668
|
+
seen_dirs.add(norm_path(entry))
|
|
669
|
+
if changed and scanned_dirs + len(stack) < dir_limit:
|
|
670
|
+
stack.append(entry)
|
|
671
|
+
continue
|
|
672
|
+
if entry.is_file():
|
|
673
|
+
seen_files.add(norm_path(entry))
|
|
674
|
+
if stats["files_scanned"] >= file_limit:
|
|
675
|
+
continue
|
|
676
|
+
_, changed, state = _upsert_asset(conn, root_id, entry, seen_at, root_depth)
|
|
677
|
+
stats["files_scanned"] += 1
|
|
678
|
+
if changed:
|
|
679
|
+
stats["files_changed"] += 1
|
|
680
|
+
if state != "ok":
|
|
681
|
+
stats["errors"] += 1
|
|
682
|
+
except Exception:
|
|
683
|
+
stats["errors"] += 1
|
|
684
|
+
deleted_files, deleted_dirs = _prune_missing_children(conn, current, seen_files, seen_dirs, seen_at)
|
|
685
|
+
stats["files_deleted"] += deleted_files
|
|
686
|
+
stats["dirs_deleted"] += deleted_dirs
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _reconcile_known_dirs(conn, exclusions: list[str], *, dir_limit: int, file_limit: int) -> dict:
|
|
690
|
+
stats = {
|
|
691
|
+
"checked": 0,
|
|
692
|
+
"changed": 0,
|
|
693
|
+
"dirs_scanned": 0,
|
|
694
|
+
"files_scanned": 0,
|
|
695
|
+
"files_changed": 0,
|
|
696
|
+
"files_deleted": 0,
|
|
697
|
+
"dirs_deleted": 0,
|
|
698
|
+
"excluded_dirs": 0,
|
|
699
|
+
"offline": 0,
|
|
700
|
+
"errors": 0,
|
|
701
|
+
}
|
|
702
|
+
if dir_limit <= 0 or file_limit <= 0:
|
|
703
|
+
return stats
|
|
704
|
+
rows = conn.execute(
|
|
705
|
+
"""
|
|
706
|
+
SELECT d.dir_id, d.path, d.quick_fingerprint, d.root_id, r.root_path, r.depth
|
|
707
|
+
FROM local_index_dirs d
|
|
708
|
+
LEFT JOIN local_index_roots r ON r.id = d.root_id
|
|
709
|
+
WHERE d.status='active'
|
|
710
|
+
ORDER BY d.updated_at ASC
|
|
711
|
+
LIMIT ?
|
|
712
|
+
""",
|
|
713
|
+
(int(dir_limit),),
|
|
714
|
+
).fetchall()
|
|
715
|
+
seen_at = now()
|
|
716
|
+
for row in rows:
|
|
717
|
+
stats["checked"] += 1
|
|
718
|
+
dir_path = Path(row["path"])
|
|
719
|
+
root_path = Path(row["root_path"]).expanduser() if row["root_path"] else None
|
|
720
|
+
if _is_excluded(str(dir_path), exclusions):
|
|
721
|
+
stats["files_deleted"] += _mark_dir_subtree_deleted(conn, str(dir_path), seen_at)
|
|
722
|
+
stats["excluded_dirs"] += 1
|
|
723
|
+
continue
|
|
724
|
+
if root_path is not None and not root_path.exists():
|
|
725
|
+
stats["offline"] += 1
|
|
726
|
+
continue
|
|
727
|
+
try:
|
|
728
|
+
if not dir_path.exists() or not dir_path.is_dir():
|
|
729
|
+
stats["files_deleted"] += _mark_dir_subtree_deleted(conn, str(dir_path), seen_at)
|
|
730
|
+
stats["dirs_deleted"] += 1
|
|
731
|
+
continue
|
|
732
|
+
st = dir_path.stat()
|
|
733
|
+
fingerprint = _dir_fingerprint(dir_path, st)
|
|
734
|
+
except Exception:
|
|
735
|
+
stats["errors"] += 1
|
|
736
|
+
continue
|
|
737
|
+
if fingerprint != row["quick_fingerprint"]:
|
|
738
|
+
stats["changed"] += 1
|
|
739
|
+
_scan_known_directory(
|
|
740
|
+
conn,
|
|
741
|
+
int(row["root_id"] or 0),
|
|
742
|
+
dir_path,
|
|
743
|
+
int(row["depth"] or 2),
|
|
744
|
+
exclusions,
|
|
745
|
+
stats,
|
|
746
|
+
file_limit=file_limit,
|
|
747
|
+
dir_limit=dir_limit,
|
|
748
|
+
)
|
|
749
|
+
else:
|
|
750
|
+
conn.execute("UPDATE local_index_dirs SET updated_at=? WHERE dir_id=?", (seen_at, row["dir_id"]))
|
|
751
|
+
return stats
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def reconcile_live_changes(
|
|
755
|
+
*,
|
|
756
|
+
asset_limit: int = DEFAULT_LIVE_ASSET_LIMIT,
|
|
757
|
+
dir_limit: int = DEFAULT_LIVE_DIR_LIMIT,
|
|
758
|
+
file_limit: int = DEFAULT_LIVE_FILE_LIMIT,
|
|
759
|
+
) -> dict:
|
|
760
|
+
conn = _conn()
|
|
761
|
+
if _is_paused():
|
|
762
|
+
return {"ok": True, "paused": True, "assets": {}, "dirs": {}}
|
|
763
|
+
exclusions = [row["path"] for row in list_exclusions()]
|
|
764
|
+
asset_stats = _reconcile_known_assets(conn, exclusions, limit=int(asset_limit or 0))
|
|
765
|
+
dir_stats = _reconcile_known_dirs(conn, exclusions, dir_limit=int(dir_limit or 0), file_limit=int(file_limit or 0))
|
|
766
|
+
conn.commit()
|
|
767
|
+
changed_total = (
|
|
768
|
+
int(asset_stats.get("modified", 0))
|
|
769
|
+
+ int(asset_stats.get("deleted", 0))
|
|
770
|
+
+ int(asset_stats.get("excluded", 0))
|
|
771
|
+
+ int(dir_stats.get("files_changed", 0))
|
|
772
|
+
+ int(dir_stats.get("files_deleted", 0))
|
|
773
|
+
+ int(dir_stats.get("dirs_deleted", 0))
|
|
774
|
+
+ int(dir_stats.get("excluded_dirs", 0))
|
|
775
|
+
)
|
|
776
|
+
if changed_total:
|
|
777
|
+
log_event("info", "live_reconcile_finished", "Local memory live changes reconciled", assets=asset_stats, dirs=dir_stats)
|
|
778
|
+
return {"ok": True, "assets": asset_stats, "dirs": dir_stats}
|
|
779
|
+
|
|
780
|
+
|
|
430
781
|
def scan_once(*, limit: int | None = None) -> dict:
|
|
431
782
|
conn = _conn()
|
|
432
783
|
if _is_paused():
|
|
@@ -455,7 +806,15 @@ def scan_once(*, limit: int | None = None) -> dict:
|
|
|
455
806
|
cycle_started_at = float(checkpoint["cycle_started_at"])
|
|
456
807
|
seen_for_root = 0
|
|
457
808
|
last_seen_path = ""
|
|
458
|
-
for file_path in _iter_files(
|
|
809
|
+
for file_path in _iter_files(
|
|
810
|
+
conn,
|
|
811
|
+
root_id,
|
|
812
|
+
root_path,
|
|
813
|
+
exclusions,
|
|
814
|
+
limit=limit,
|
|
815
|
+
start_after=str(checkpoint["current_path"] or ""),
|
|
816
|
+
seen_at=cycle_started_at,
|
|
817
|
+
):
|
|
459
818
|
asset_id, changed, state = _upsert_asset(conn, root_id, file_path, cycle_started_at, int(root["depth"] or 2))
|
|
460
819
|
last_seen_path = norm_path(file_path)
|
|
461
820
|
totals["seen"] += 1
|
|
@@ -630,12 +989,25 @@ def process_jobs(*, limit: int = 100) -> dict:
|
|
|
630
989
|
return {"ok": True, "processed": processed, "failed": failed}
|
|
631
990
|
|
|
632
991
|
|
|
633
|
-
def run_once(
|
|
992
|
+
def run_once(
|
|
993
|
+
*,
|
|
994
|
+
root: str | None = None,
|
|
995
|
+
limit: int | None = None,
|
|
996
|
+
process_limit: int = 100,
|
|
997
|
+
live_asset_limit: int = DEFAULT_LIVE_ASSET_LIMIT,
|
|
998
|
+
live_dir_limit: int = DEFAULT_LIVE_DIR_LIMIT,
|
|
999
|
+
live_file_limit: int = DEFAULT_LIVE_FILE_LIMIT,
|
|
1000
|
+
) -> dict:
|
|
634
1001
|
if root:
|
|
635
1002
|
add_root(root)
|
|
1003
|
+
live_result = reconcile_live_changes(
|
|
1004
|
+
asset_limit=live_asset_limit,
|
|
1005
|
+
dir_limit=live_dir_limit,
|
|
1006
|
+
file_limit=live_file_limit,
|
|
1007
|
+
)
|
|
636
1008
|
scan_result = scan_once(limit=limit)
|
|
637
1009
|
job_result = process_jobs(limit=process_limit)
|
|
638
|
-
return {"ok": True, "scan": scan_result, "jobs": job_result}
|
|
1010
|
+
return {"ok": True, "live": live_result, "scan": scan_result, "jobs": job_result}
|
|
639
1011
|
|
|
640
1012
|
|
|
641
1013
|
def _problem_rows(conn) -> list[dict]:
|
|
@@ -955,6 +1327,21 @@ def context_query(query: str, *, intent: str = "answer", limit: int = 12, eviden
|
|
|
955
1327
|
(f"%{query.lower()}%", int(limit)),
|
|
956
1328
|
).fetchall()
|
|
957
1329
|
entities_payload = [dict(row) for row in entity_rows]
|
|
1330
|
+
relations_payload: list[dict] = []
|
|
1331
|
+
if seen_assets:
|
|
1332
|
+
asset_ids = list(seen_assets)[: int(limit)]
|
|
1333
|
+
placeholders = ",".join("?" for _ in asset_ids)
|
|
1334
|
+
relation_rows = conn.execute(
|
|
1335
|
+
f"""
|
|
1336
|
+
SELECT relation_id, source_asset_id, target_ref, relation_type, confidence, evidence
|
|
1337
|
+
FROM local_relations
|
|
1338
|
+
WHERE active=1 AND source_asset_id IN ({placeholders})
|
|
1339
|
+
ORDER BY confidence DESC
|
|
1340
|
+
LIMIT ?
|
|
1341
|
+
""",
|
|
1342
|
+
[*asset_ids, int(limit) * 3],
|
|
1343
|
+
).fetchall()
|
|
1344
|
+
relations_payload = [dict(row) for row in relation_rows]
|
|
958
1345
|
warnings = []
|
|
959
1346
|
if evidence_required and not evidence_refs:
|
|
960
1347
|
warnings.append("No local evidence found for this query.")
|
|
@@ -984,7 +1371,7 @@ def context_query(query: str, *, intent: str = "answer", limit: int = 12, eviden
|
|
|
984
1371
|
"summary": summary,
|
|
985
1372
|
"assets": assets,
|
|
986
1373
|
"entities": entities_payload,
|
|
987
|
-
"relations":
|
|
1374
|
+
"relations": relations_payload,
|
|
988
1375
|
"chunks": chunks,
|
|
989
1376
|
"warnings": warnings,
|
|
990
1377
|
"evidence_refs": evidence_refs,
|
|
@@ -1034,6 +1421,7 @@ def clear_index() -> dict:
|
|
|
1034
1421
|
"local_chunks",
|
|
1035
1422
|
"local_entities",
|
|
1036
1423
|
"local_relations",
|
|
1424
|
+
"local_index_dirs",
|
|
1037
1425
|
"local_index_errors",
|
|
1038
1426
|
"local_index_jobs",
|
|
1039
1427
|
"local_asset_versions",
|
package/src/runtime_power.py
CHANGED
|
@@ -920,7 +920,20 @@ def ensure_full_disk_access_choice(
|
|
|
920
920
|
probe = probe_fn()
|
|
921
921
|
if probe.get("granted") is True:
|
|
922
922
|
verified = True
|
|
923
|
-
|
|
923
|
+
schedule[FULL_DISK_ACCESS_STATUS_KEY] = FULL_DISK_ACCESS_GRANTED
|
|
924
|
+
schedule[FULL_DISK_ACCESS_REASONS_KEY] = []
|
|
925
|
+
clear_full_disk_access_required_state()
|
|
926
|
+
save_schedule_config(schedule)
|
|
927
|
+
return {
|
|
928
|
+
"status": FULL_DISK_ACCESS_GRANTED,
|
|
929
|
+
"prompted": False,
|
|
930
|
+
"verified": True,
|
|
931
|
+
"settings_opened": False,
|
|
932
|
+
"reasons": [],
|
|
933
|
+
"schedule_file": str(SCHEDULE_FILE),
|
|
934
|
+
"message": "",
|
|
935
|
+
"relevant": False,
|
|
936
|
+
}
|
|
924
937
|
else:
|
|
925
938
|
status = FULL_DISK_ACCESS_LATER
|
|
926
939
|
message = (
|
|
@@ -39,6 +39,9 @@ LOCK_FILE = LOG_DIR / "local-index.lock"
|
|
|
39
39
|
LOCK_STALE_SECONDS = int(os.environ.get("NEXO_LOCAL_INDEX_LOCK_STALE_SECONDS", "1800") or "1800")
|
|
40
40
|
SCAN_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_SCAN_LIMIT", "1000") or "1000")
|
|
41
41
|
PROCESS_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_PROCESS_LIMIT", "200") or "200")
|
|
42
|
+
LIVE_ASSET_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_ASSET_LIMIT", "2000") or "2000")
|
|
43
|
+
LIVE_DIR_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_DIR_LIMIT", "300") or "300")
|
|
44
|
+
LIVE_FILE_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_FILE_LIMIT", "1000") or "1000")
|
|
42
45
|
|
|
43
46
|
|
|
44
47
|
def log(message: str) -> None:
|
|
@@ -79,7 +82,13 @@ def main() -> int:
|
|
|
79
82
|
try:
|
|
80
83
|
if os.environ.get("NEXO_LOCAL_INDEX_DISABLE_DEFAULT_ROOTS", "").strip() != "1":
|
|
81
84
|
api.ensure_default_roots()
|
|
82
|
-
result = api.run_once(
|
|
85
|
+
result = api.run_once(
|
|
86
|
+
limit=SCAN_LIMIT,
|
|
87
|
+
process_limit=PROCESS_LIMIT,
|
|
88
|
+
live_asset_limit=LIVE_ASSET_LIMIT,
|
|
89
|
+
live_dir_limit=LIVE_DIR_LIMIT,
|
|
90
|
+
live_file_limit=LIVE_FILE_LIMIT,
|
|
91
|
+
)
|
|
83
92
|
log_event("info", "service_cycle_finished", "Local memory service cycle finished", result=result)
|
|
84
93
|
log(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
85
94
|
return 0 if result.get("ok") else 2
|