nexo-brain 7.18.1 → 7.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +5 -1
- package/package.json +1 -1
- package/src/auto_update.py +5 -0
- package/src/cli.py +21 -1
- package/src/db/_schema.py +31 -0
- package/src/local_context/__init__.py +2 -0
- package/src/local_context/api.py +537 -14
- package/src/local_context/privacy.py +23 -0
- package/src/runtime_power.py +14 -1
- package/src/runtime_versioning.py +31 -1
- package/src/scripts/nexo-local-index.py +10 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.20.0",
|
|
4
4
|
"description": "Local cognitive runtime for Claude Code \u2014 persistent memory, overnight learning, doctor diagnostics, personal scripts, recovery-aware jobs, startup preflight, and optional dashboard/power helper.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "NEXO Brain",
|
package/README.md
CHANGED
|
@@ -18,7 +18,11 @@
|
|
|
18
18
|
|
|
19
19
|
[Watch the overview video](https://nexo-brain.com/watch/) · [Watch on YouTube](https://www.youtube.com/watch?v=i2lkGhKyVqI) · [Open the infographic](https://nexo-brain.com/assets/nexo-brain-infographic-v5.png)
|
|
20
20
|
|
|
21
|
-
Version `7.
|
|
21
|
+
Version `7.20.0` is the current packaged-runtime line. Minor release over v7.19.0 — the Local Context index now reconciles known files and folders on every service cycle, so created, modified, deleted and newly excluded local files are reflected automatically between full scans.
|
|
22
|
+
|
|
23
|
+
Previously in `7.19.0`: minor release over v7.18.1 - bundle-managed installations (NEXO Desktop `brain-bundle/`) can now pin Brain to the host application release cycle via `NEXO_BRAIN_AUTO_UPDATE=false`, and the server auto-exits with code 75 on fingerprint mismatch so MCP clients respawn the server with the new code instead of leaving stale `server.py` processes alive.
|
|
24
|
+
|
|
25
|
+
Previously in `7.18.1`: patch release over v7.18.0 - packaged Brain runtimes now include the `local_context` package, so Desktop Local Memory and `nexo local-context` do not get stuck behind `ModuleNotFoundError` or zero-file status; the local-index service also keeps detecting newly mounted volumes automatically.
|
|
22
26
|
|
|
23
27
|
Previously in `7.18.0`: minor release over v7.17.8 - Brain adds the Local Context Layer: a local-only background memory index with checkpoints, extraction, graph links, embeddings, MCP/CLI controls, and pre-action evidence for NEXO agents.
|
|
24
28
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nexo-brain",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.20.0",
|
|
4
4
|
"mcpName": "io.github.wazionapps/nexo",
|
|
5
5
|
"description": "NEXO Brain — Shared brain for AI agents. Persistent memory, semantic RAG, natural forgetting, metacognitive guard, trust scoring, 150+ MCP tools. Works with Claude Code, Codex, Claude Desktop & any MCP client. 100% local, free.",
|
|
6
6
|
"homepage": "https://nexo-brain.com",
|
package/src/auto_update.py
CHANGED
|
@@ -5372,6 +5372,11 @@ def startup_preflight(*, entrypoint: str, interactive: bool = False) -> dict:
|
|
|
5372
5372
|
return result
|
|
5373
5373
|
|
|
5374
5374
|
try:
|
|
5375
|
+
env_auto_update = os.environ.get("NEXO_BRAIN_AUTO_UPDATE", "").strip().lower()
|
|
5376
|
+
if env_auto_update in ("0", "false", "off", "no"):
|
|
5377
|
+
result["skipped_reason"] = "auto_update disabled via NEXO_BRAIN_AUTO_UPDATE env var"
|
|
5378
|
+
_write_update_summary(result)
|
|
5379
|
+
return result
|
|
5375
5380
|
last_check = _read_last_check()
|
|
5376
5381
|
now = time.time()
|
|
5377
5382
|
schedule_file = _runtime_config_dir(NEXO_HOME) / "schedule.json"
|
package/src/cli.py
CHANGED
|
@@ -20,7 +20,7 @@ Entry points:
|
|
|
20
20
|
nexo scripts run NAME_OR_PATH [-- args...]
|
|
21
21
|
nexo scripts doctor [NAME_OR_PATH] [--json]
|
|
22
22
|
nexo scripts call TOOL --input JSON [--json-output]
|
|
23
|
-
nexo local-context status|run-once|pause|resume|roots|exclusions|query|diagnostics|models [--json]
|
|
23
|
+
nexo local-context status|run-once|reconcile|pause|resume|roots|exclusions|query|diagnostics|models [--json]
|
|
24
24
|
nexo automations reactivate NAME [--test-run] [--json]
|
|
25
25
|
nexo skills list [--level ...] [--source-kind ...] [--json]
|
|
26
26
|
nexo skills get ID [--json]
|
|
@@ -1291,6 +1291,18 @@ def _local_context_run_once(args) -> int:
|
|
|
1291
1291
|
)
|
|
1292
1292
|
|
|
1293
1293
|
|
|
1294
|
+
def _local_context_reconcile(args) -> int:
|
|
1295
|
+
import local_context
|
|
1296
|
+
return _local_context_emit(
|
|
1297
|
+
local_context.reconcile_live_changes(
|
|
1298
|
+
asset_limit=int(getattr(args, "asset_limit", 2000) or 0),
|
|
1299
|
+
dir_limit=int(getattr(args, "dir_limit", 300) or 0),
|
|
1300
|
+
file_limit=int(getattr(args, "file_limit", 1000) or 0),
|
|
1301
|
+
),
|
|
1302
|
+
args,
|
|
1303
|
+
)
|
|
1304
|
+
|
|
1305
|
+
|
|
1294
1306
|
def _local_context_control(args) -> int:
|
|
1295
1307
|
import local_context
|
|
1296
1308
|
command = str(getattr(args, "local_context_command", "") or "")
|
|
@@ -3154,6 +3166,12 @@ def main():
|
|
|
3154
3166
|
local_context_run_p.add_argument("--process-limit", type=int, default=100, help="Maximum extraction/graph jobs to process")
|
|
3155
3167
|
local_context_run_p.add_argument("--json", action="store_true", help="JSON output")
|
|
3156
3168
|
|
|
3169
|
+
local_context_reconcile_p = local_context_sub.add_parser("reconcile", help="Reconcile changed, deleted and new local files")
|
|
3170
|
+
local_context_reconcile_p.add_argument("--asset-limit", type=int, default=2000, help="Maximum known files to verify")
|
|
3171
|
+
local_context_reconcile_p.add_argument("--dir-limit", type=int, default=300, help="Maximum known folders to verify")
|
|
3172
|
+
local_context_reconcile_p.add_argument("--file-limit", type=int, default=1000, help="Maximum files to scan inside changed folders")
|
|
3173
|
+
local_context_reconcile_p.add_argument("--json", action="store_true", help="JSON output")
|
|
3174
|
+
|
|
3157
3175
|
local_context_pause_p = local_context_sub.add_parser("pause", help="Pause local memory indexing")
|
|
3158
3176
|
local_context_pause_p.add_argument("--json", action="store_true", help="JSON output")
|
|
3159
3177
|
|
|
@@ -3734,6 +3752,8 @@ def main():
|
|
|
3734
3752
|
return _local_context_status(args)
|
|
3735
3753
|
if args.local_context_command == "run-once":
|
|
3736
3754
|
return _local_context_run_once(args)
|
|
3755
|
+
if args.local_context_command == "reconcile":
|
|
3756
|
+
return _local_context_reconcile(args)
|
|
3737
3757
|
if args.local_context_command in {"pause", "resume", "clear-index"}:
|
|
3738
3758
|
return _local_context_control(args)
|
|
3739
3759
|
if args.local_context_command == "roots":
|
package/src/db/_schema.py
CHANGED
|
@@ -1972,6 +1972,36 @@ def _m63_local_context_layer(conn):
|
|
|
1972
1972
|
)
|
|
1973
1973
|
|
|
1974
1974
|
|
|
1975
|
+
def _m64_local_context_live_dirs(conn):
|
|
1976
|
+
"""Track known folders so local context can detect new/deleted/changed files quickly."""
|
|
1977
|
+
conn.executescript(
|
|
1978
|
+
"""
|
|
1979
|
+
CREATE TABLE IF NOT EXISTS local_index_dirs (
|
|
1980
|
+
dir_id TEXT PRIMARY KEY,
|
|
1981
|
+
root_id INTEGER,
|
|
1982
|
+
path TEXT NOT NULL UNIQUE,
|
|
1983
|
+
display_path TEXT NOT NULL,
|
|
1984
|
+
parent_path TEXT NOT NULL DEFAULT '',
|
|
1985
|
+
quick_fingerprint TEXT NOT NULL DEFAULT '',
|
|
1986
|
+
status TEXT NOT NULL DEFAULT 'active',
|
|
1987
|
+
first_seen_at REAL NOT NULL,
|
|
1988
|
+
last_seen_at REAL NOT NULL,
|
|
1989
|
+
updated_at REAL NOT NULL,
|
|
1990
|
+
deleted_at REAL
|
|
1991
|
+
);
|
|
1992
|
+
|
|
1993
|
+
CREATE INDEX IF NOT EXISTS idx_local_index_dirs_root_status
|
|
1994
|
+
ON local_index_dirs(root_id, status);
|
|
1995
|
+
CREATE INDEX IF NOT EXISTS idx_local_index_dirs_path
|
|
1996
|
+
ON local_index_dirs(path);
|
|
1997
|
+
CREATE INDEX IF NOT EXISTS idx_local_index_dirs_parent
|
|
1998
|
+
ON local_index_dirs(parent_path);
|
|
1999
|
+
CREATE INDEX IF NOT EXISTS idx_local_assets_updated
|
|
2000
|
+
ON local_assets(updated_at);
|
|
2001
|
+
"""
|
|
2002
|
+
)
|
|
2003
|
+
|
|
2004
|
+
|
|
1975
2005
|
MIGRATIONS = [
|
|
1976
2006
|
(1, "learnings_columns", _m1_learnings_columns),
|
|
1977
2007
|
(2, "followups_reasoning", _m2_followups_reasoning),
|
|
@@ -2036,6 +2066,7 @@ MIGRATIONS = [
|
|
|
2036
2066
|
(61, "memory_observations_fts", _m61_memory_observations_fts),
|
|
2037
2067
|
(62, "memory_observations_fts_trigger_fix", _m62_memory_observations_fts_trigger_fix),
|
|
2038
2068
|
(63, "local_context_layer", _m63_local_context_layer),
|
|
2069
|
+
(64, "local_context_live_dirs", _m64_local_context_live_dirs),
|
|
2039
2070
|
]
|
|
2040
2071
|
|
|
2041
2072
|
|
|
@@ -19,6 +19,7 @@ from .api import (
|
|
|
19
19
|
model_status,
|
|
20
20
|
pause,
|
|
21
21
|
purge_asset,
|
|
22
|
+
reconcile_live_changes,
|
|
22
23
|
remove_exclusion,
|
|
23
24
|
remove_root,
|
|
24
25
|
resume,
|
|
@@ -41,6 +42,7 @@ __all__ = [
|
|
|
41
42
|
"model_status",
|
|
42
43
|
"pause",
|
|
43
44
|
"purge_asset",
|
|
45
|
+
"reconcile_live_changes",
|
|
44
46
|
"remove_exclusion",
|
|
45
47
|
"remove_root",
|
|
46
48
|
"resume",
|
package/src/local_context/api.py
CHANGED
|
@@ -5,6 +5,7 @@ import os
|
|
|
5
5
|
import shutil
|
|
6
6
|
import stat
|
|
7
7
|
import hashlib
|
|
8
|
+
import subprocess
|
|
8
9
|
import sys
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
from typing import Any
|
|
@@ -15,9 +16,17 @@ from db._schema import run_migrations
|
|
|
15
16
|
from . import embeddings
|
|
16
17
|
from .extractors import chunk_text, entities, extract_text, summarize
|
|
17
18
|
from .logging import log_event, tail
|
|
18
|
-
from .privacy import classify_path, should_extract
|
|
19
|
+
from .privacy import classify_path, should_extract, should_skip_tree
|
|
19
20
|
from .util import content_hash, json_dumps, json_loads, norm_path, now, quick_fingerprint, redact_path, stable_id, system_label, tokenize
|
|
20
21
|
|
|
22
|
+
LOCAL_INDEX_SERVICE_LABEL = "com.nexo.local-index"
|
|
23
|
+
LOCAL_INDEX_SCRIPT_NAME = "nexo-local-index.py"
|
|
24
|
+
LOCAL_INDEX_WINDOWS_TASK = "NEXO Local Memory"
|
|
25
|
+
LOCAL_INDEX_LINUX_UNIT = "nexo-local-index.service"
|
|
26
|
+
DEFAULT_LIVE_ASSET_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_ASSET_LIMIT", "2000") or "2000")
|
|
27
|
+
DEFAULT_LIVE_DIR_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_DIR_LIMIT", "300") or "300")
|
|
28
|
+
DEFAULT_LIVE_FILE_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_FILE_LIMIT", "1000") or "1000")
|
|
29
|
+
|
|
21
30
|
|
|
22
31
|
def ensure_ready() -> None:
|
|
23
32
|
init_db()
|
|
@@ -199,6 +208,11 @@ def _is_excluded(path: str, exclusions: list[str]) -> bool:
|
|
|
199
208
|
return any(value == item or value.startswith(item + os.sep) for item in exclusions)
|
|
200
209
|
|
|
201
210
|
|
|
211
|
+
def _path_prefix(path: str) -> str:
|
|
212
|
+
normalized = norm_path(path)
|
|
213
|
+
return normalized + os.sep if normalized else os.sep
|
|
214
|
+
|
|
215
|
+
|
|
202
216
|
def _file_type(path: Path) -> str:
|
|
203
217
|
if path.is_dir():
|
|
204
218
|
return "folder"
|
|
@@ -226,6 +240,65 @@ def _permission_state(path: Path) -> str:
|
|
|
226
240
|
return "granted"
|
|
227
241
|
|
|
228
242
|
|
|
243
|
+
def _dir_fingerprint(path: Path, stat_result: os.stat_result | None = None) -> str:
|
|
244
|
+
st = stat_result or path.stat()
|
|
245
|
+
ctime_ns = getattr(st, "st_ctime_ns", int(float(st.st_ctime) * 1_000_000_000))
|
|
246
|
+
return f"{int(st.st_mtime_ns)}:{int(ctime_ns)}"
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _upsert_dir(
|
|
250
|
+
conn,
|
|
251
|
+
root_id: int,
|
|
252
|
+
path: Path,
|
|
253
|
+
seen_at: float,
|
|
254
|
+
stat_result: os.stat_result | None = None,
|
|
255
|
+
) -> tuple[bool, str]:
|
|
256
|
+
raw_path = str(path)
|
|
257
|
+
normalized = norm_path(raw_path)
|
|
258
|
+
dir_id = stable_id("dir", normalized)
|
|
259
|
+
parent = norm_path(path.parent)
|
|
260
|
+
try:
|
|
261
|
+
fingerprint = _dir_fingerprint(path, stat_result)
|
|
262
|
+
except Exception:
|
|
263
|
+
return False, "error"
|
|
264
|
+
row = conn.execute(
|
|
265
|
+
"SELECT quick_fingerprint, status FROM local_index_dirs WHERE dir_id=?",
|
|
266
|
+
(dir_id,),
|
|
267
|
+
).fetchone()
|
|
268
|
+
changed = not row or row["quick_fingerprint"] != fingerprint or row["status"] == "deleted"
|
|
269
|
+
conn.execute(
|
|
270
|
+
"""
|
|
271
|
+
INSERT INTO local_index_dirs(
|
|
272
|
+
dir_id, root_id, path, display_path, parent_path, quick_fingerprint,
|
|
273
|
+
status, first_seen_at, last_seen_at, updated_at, deleted_at
|
|
274
|
+
)
|
|
275
|
+
VALUES (?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, NULL)
|
|
276
|
+
ON CONFLICT(dir_id) DO UPDATE SET
|
|
277
|
+
root_id=excluded.root_id,
|
|
278
|
+
path=excluded.path,
|
|
279
|
+
display_path=excluded.display_path,
|
|
280
|
+
parent_path=excluded.parent_path,
|
|
281
|
+
quick_fingerprint=excluded.quick_fingerprint,
|
|
282
|
+
status='active',
|
|
283
|
+
last_seen_at=excluded.last_seen_at,
|
|
284
|
+
updated_at=excluded.updated_at,
|
|
285
|
+
deleted_at=NULL
|
|
286
|
+
""",
|
|
287
|
+
(
|
|
288
|
+
dir_id,
|
|
289
|
+
root_id,
|
|
290
|
+
normalized,
|
|
291
|
+
raw_path,
|
|
292
|
+
parent,
|
|
293
|
+
fingerprint,
|
|
294
|
+
seen_at,
|
|
295
|
+
seen_at,
|
|
296
|
+
seen_at,
|
|
297
|
+
),
|
|
298
|
+
)
|
|
299
|
+
return changed, fingerprint
|
|
300
|
+
|
|
301
|
+
|
|
229
302
|
def _upsert_asset(conn, root_id: int, path: Path, seen_at: float, root_depth: int) -> tuple[str, bool, str]:
|
|
230
303
|
raw_path = str(path)
|
|
231
304
|
normalized = norm_path(raw_path)
|
|
@@ -315,6 +388,43 @@ def _upsert_asset(conn, root_id: int, path: Path, seen_at: float, root_depth: in
|
|
|
315
388
|
return asset_id, changed, "ok"
|
|
316
389
|
|
|
317
390
|
|
|
391
|
+
def _mark_asset_deleted(conn, asset_id: str, deleted_at: float | None = None) -> None:
|
|
392
|
+
deleted_at = deleted_at or now()
|
|
393
|
+
conn.execute(
|
|
394
|
+
"UPDATE local_assets SET status='deleted', deleted_at=?, updated_at=? WHERE asset_id=? AND status!='deleted'",
|
|
395
|
+
(deleted_at, deleted_at, asset_id),
|
|
396
|
+
)
|
|
397
|
+
conn.execute(
|
|
398
|
+
"""
|
|
399
|
+
UPDATE local_index_jobs
|
|
400
|
+
SET status='done', last_error_code='asset_deleted', updated_at=?
|
|
401
|
+
WHERE asset_id=? AND status IN ('pending', 'running')
|
|
402
|
+
""",
|
|
403
|
+
(deleted_at, asset_id),
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _mark_dir_subtree_deleted(conn, dir_path: str, deleted_at: float | None = None) -> int:
|
|
408
|
+
deleted_at = deleted_at or now()
|
|
409
|
+
normalized = norm_path(dir_path)
|
|
410
|
+
prefix = _path_prefix(normalized)
|
|
411
|
+
conn.execute(
|
|
412
|
+
"""
|
|
413
|
+
UPDATE local_index_dirs
|
|
414
|
+
SET status='deleted', deleted_at=?, updated_at=?
|
|
415
|
+
WHERE status='active' AND (path=? OR path LIKE ?)
|
|
416
|
+
""",
|
|
417
|
+
(deleted_at, deleted_at, normalized, prefix + "%"),
|
|
418
|
+
)
|
|
419
|
+
rows = conn.execute(
|
|
420
|
+
"SELECT asset_id FROM local_assets WHERE status='active' AND (path=? OR path LIKE ?)",
|
|
421
|
+
(normalized, prefix + "%"),
|
|
422
|
+
).fetchall()
|
|
423
|
+
for row in rows:
|
|
424
|
+
_mark_asset_deleted(conn, row["asset_id"], deleted_at)
|
|
425
|
+
return len(rows)
|
|
426
|
+
|
|
427
|
+
|
|
318
428
|
def enqueue_job(conn, asset_id: str, job_type: str, *, priority: int = 50) -> str:
|
|
319
429
|
job_id = stable_id("job", f"{asset_id}:{job_type}")
|
|
320
430
|
conn.execute(
|
|
@@ -328,7 +438,17 @@ def enqueue_job(conn, asset_id: str, job_type: str, *, priority: int = 50) -> st
|
|
|
328
438
|
return job_id
|
|
329
439
|
|
|
330
440
|
|
|
331
|
-
def _iter_files(
|
|
441
|
+
def _iter_files(
|
|
442
|
+
conn,
|
|
443
|
+
root_id: int,
|
|
444
|
+
root: Path,
|
|
445
|
+
exclusions: list[str],
|
|
446
|
+
*,
|
|
447
|
+
limit: int | None = None,
|
|
448
|
+
start_after: str = "",
|
|
449
|
+
seen_at: float | None = None,
|
|
450
|
+
):
|
|
451
|
+
seen_at = seen_at or now()
|
|
332
452
|
seen_dirs: set[tuple[int, int]] = set()
|
|
333
453
|
count = 0
|
|
334
454
|
stack = [root]
|
|
@@ -337,6 +457,8 @@ def _iter_files(root: Path, exclusions: list[str], *, limit: int | None = None,
|
|
|
337
457
|
current = stack.pop()
|
|
338
458
|
if _is_excluded(str(current), exclusions):
|
|
339
459
|
continue
|
|
460
|
+
if current != root and should_skip_tree(str(current)):
|
|
461
|
+
continue
|
|
340
462
|
try:
|
|
341
463
|
st = current.stat()
|
|
342
464
|
except Exception:
|
|
@@ -345,6 +467,7 @@ def _iter_files(root: Path, exclusions: list[str], *, limit: int | None = None,
|
|
|
345
467
|
if key in seen_dirs:
|
|
346
468
|
continue
|
|
347
469
|
seen_dirs.add(key)
|
|
470
|
+
_upsert_dir(conn, root_id, current, seen_at, st)
|
|
348
471
|
try:
|
|
349
472
|
entries = sorted(current.iterdir(), key=lambda item: str(item).lower())
|
|
350
473
|
except Exception:
|
|
@@ -356,6 +479,8 @@ def _iter_files(root: Path, exclusions: list[str], *, limit: int | None = None,
|
|
|
356
479
|
if entry.is_symlink():
|
|
357
480
|
continue
|
|
358
481
|
if entry.is_dir():
|
|
482
|
+
if should_skip_tree(str(entry)):
|
|
483
|
+
continue
|
|
359
484
|
dirs.append(entry)
|
|
360
485
|
continue
|
|
361
486
|
if entry.is_file():
|
|
@@ -417,6 +542,242 @@ def _clear_checkpoint(conn, root_id: int) -> None:
|
|
|
417
542
|
conn.execute("DELETE FROM local_index_checkpoints WHERE root_id=? AND phase='quick_index'", (root_id,))
|
|
418
543
|
|
|
419
544
|
|
|
545
|
+
def _reconcile_known_assets(conn, exclusions: list[str], *, limit: int) -> dict:
|
|
546
|
+
stats = {"checked": 0, "modified": 0, "deleted": 0, "excluded": 0, "offline": 0, "errors": 0}
|
|
547
|
+
if limit <= 0:
|
|
548
|
+
return stats
|
|
549
|
+
rows = conn.execute(
|
|
550
|
+
"""
|
|
551
|
+
SELECT a.asset_id, a.path, a.root_id, a.quick_fingerprint, a.depth, r.root_path
|
|
552
|
+
FROM local_assets a
|
|
553
|
+
LEFT JOIN local_index_roots r ON r.id = a.root_id
|
|
554
|
+
WHERE a.status='active'
|
|
555
|
+
ORDER BY a.updated_at ASC
|
|
556
|
+
LIMIT ?
|
|
557
|
+
""",
|
|
558
|
+
(int(limit),),
|
|
559
|
+
).fetchall()
|
|
560
|
+
seen_at = now()
|
|
561
|
+
for row in rows:
|
|
562
|
+
stats["checked"] += 1
|
|
563
|
+
path = str(row["path"])
|
|
564
|
+
root_path = Path(row["root_path"]).expanduser() if row["root_path"] else None
|
|
565
|
+
if _is_excluded(path, exclusions):
|
|
566
|
+
_mark_asset_deleted(conn, row["asset_id"], seen_at)
|
|
567
|
+
stats["excluded"] += 1
|
|
568
|
+
continue
|
|
569
|
+
if root_path is not None and not root_path.exists():
|
|
570
|
+
stats["offline"] += 1
|
|
571
|
+
continue
|
|
572
|
+
file_path = Path(path)
|
|
573
|
+
try:
|
|
574
|
+
if not file_path.exists() or not file_path.is_file():
|
|
575
|
+
_mark_asset_deleted(conn, row["asset_id"], seen_at)
|
|
576
|
+
stats["deleted"] += 1
|
|
577
|
+
continue
|
|
578
|
+
st = file_path.stat()
|
|
579
|
+
fingerprint = quick_fingerprint(file_path, st)
|
|
580
|
+
except Exception:
|
|
581
|
+
stats["errors"] += 1
|
|
582
|
+
continue
|
|
583
|
+
if fingerprint != row["quick_fingerprint"]:
|
|
584
|
+
_, changed, state = _upsert_asset(conn, int(row["root_id"] or 0), file_path, seen_at, int(row["depth"] or 2))
|
|
585
|
+
if changed:
|
|
586
|
+
stats["modified"] += 1
|
|
587
|
+
if state != "ok":
|
|
588
|
+
stats["errors"] += 1
|
|
589
|
+
else:
|
|
590
|
+
conn.execute("UPDATE local_assets SET updated_at=? WHERE asset_id=?", (seen_at, row["asset_id"]))
|
|
591
|
+
return stats
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def _prune_missing_children(
|
|
595
|
+
conn,
|
|
596
|
+
directory: Path,
|
|
597
|
+
seen_files: set[str],
|
|
598
|
+
seen_dirs: set[str],
|
|
599
|
+
seen_at: float,
|
|
600
|
+
) -> tuple[int, int]:
|
|
601
|
+
parent = norm_path(directory)
|
|
602
|
+
deleted_files = 0
|
|
603
|
+
deleted_dirs = 0
|
|
604
|
+
file_rows = conn.execute(
|
|
605
|
+
"SELECT asset_id, path FROM local_assets WHERE parent_path=? AND status='active'",
|
|
606
|
+
(parent,),
|
|
607
|
+
).fetchall()
|
|
608
|
+
for row in file_rows:
|
|
609
|
+
if row["path"] not in seen_files:
|
|
610
|
+
_mark_asset_deleted(conn, row["asset_id"], seen_at)
|
|
611
|
+
deleted_files += 1
|
|
612
|
+
dir_rows = conn.execute(
|
|
613
|
+
"SELECT path FROM local_index_dirs WHERE parent_path=? AND status='active'",
|
|
614
|
+
(parent,),
|
|
615
|
+
).fetchall()
|
|
616
|
+
for row in dir_rows:
|
|
617
|
+
if row["path"] not in seen_dirs:
|
|
618
|
+
deleted_files += _mark_dir_subtree_deleted(conn, row["path"], seen_at)
|
|
619
|
+
deleted_dirs += 1
|
|
620
|
+
return deleted_files, deleted_dirs
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def _scan_known_directory(
|
|
624
|
+
conn,
|
|
625
|
+
root_id: int,
|
|
626
|
+
directory: Path,
|
|
627
|
+
root_depth: int,
|
|
628
|
+
exclusions: list[str],
|
|
629
|
+
stats: dict,
|
|
630
|
+
*,
|
|
631
|
+
file_limit: int,
|
|
632
|
+
dir_limit: int,
|
|
633
|
+
) -> None:
|
|
634
|
+
stack = [directory]
|
|
635
|
+
seen_at = now()
|
|
636
|
+
scanned_dirs = 0
|
|
637
|
+
while stack and stats["files_scanned"] < file_limit and scanned_dirs < dir_limit:
|
|
638
|
+
current = stack.pop()
|
|
639
|
+
if _is_excluded(str(current), exclusions):
|
|
640
|
+
_mark_dir_subtree_deleted(conn, str(current), seen_at)
|
|
641
|
+
stats["excluded_dirs"] += 1
|
|
642
|
+
continue
|
|
643
|
+
if current != directory and should_skip_tree(str(current)):
|
|
644
|
+
continue
|
|
645
|
+
try:
|
|
646
|
+
st = current.stat()
|
|
647
|
+
if not current.is_dir():
|
|
648
|
+
continue
|
|
649
|
+
entries = sorted(current.iterdir(), key=lambda item: str(item).lower())
|
|
650
|
+
except Exception:
|
|
651
|
+
stats["errors"] += 1
|
|
652
|
+
continue
|
|
653
|
+
scanned_dirs += 1
|
|
654
|
+
stats["dirs_scanned"] += 1
|
|
655
|
+
_upsert_dir(conn, root_id, current, seen_at, st)
|
|
656
|
+
seen_files: set[str] = set()
|
|
657
|
+
seen_dirs: set[str] = set()
|
|
658
|
+
for entry in entries:
|
|
659
|
+
if _is_excluded(str(entry), exclusions):
|
|
660
|
+
continue
|
|
661
|
+
try:
|
|
662
|
+
if entry.is_symlink():
|
|
663
|
+
continue
|
|
664
|
+
if entry.is_dir():
|
|
665
|
+
if should_skip_tree(str(entry)):
|
|
666
|
+
continue
|
|
667
|
+
changed, _ = _upsert_dir(conn, root_id, entry, seen_at)
|
|
668
|
+
seen_dirs.add(norm_path(entry))
|
|
669
|
+
if changed and scanned_dirs + len(stack) < dir_limit:
|
|
670
|
+
stack.append(entry)
|
|
671
|
+
continue
|
|
672
|
+
if entry.is_file():
|
|
673
|
+
seen_files.add(norm_path(entry))
|
|
674
|
+
if stats["files_scanned"] >= file_limit:
|
|
675
|
+
continue
|
|
676
|
+
_, changed, state = _upsert_asset(conn, root_id, entry, seen_at, root_depth)
|
|
677
|
+
stats["files_scanned"] += 1
|
|
678
|
+
if changed:
|
|
679
|
+
stats["files_changed"] += 1
|
|
680
|
+
if state != "ok":
|
|
681
|
+
stats["errors"] += 1
|
|
682
|
+
except Exception:
|
|
683
|
+
stats["errors"] += 1
|
|
684
|
+
deleted_files, deleted_dirs = _prune_missing_children(conn, current, seen_files, seen_dirs, seen_at)
|
|
685
|
+
stats["files_deleted"] += deleted_files
|
|
686
|
+
stats["dirs_deleted"] += deleted_dirs
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _reconcile_known_dirs(conn, exclusions: list[str], *, dir_limit: int, file_limit: int) -> dict:
|
|
690
|
+
stats = {
|
|
691
|
+
"checked": 0,
|
|
692
|
+
"changed": 0,
|
|
693
|
+
"dirs_scanned": 0,
|
|
694
|
+
"files_scanned": 0,
|
|
695
|
+
"files_changed": 0,
|
|
696
|
+
"files_deleted": 0,
|
|
697
|
+
"dirs_deleted": 0,
|
|
698
|
+
"excluded_dirs": 0,
|
|
699
|
+
"offline": 0,
|
|
700
|
+
"errors": 0,
|
|
701
|
+
}
|
|
702
|
+
if dir_limit <= 0 or file_limit <= 0:
|
|
703
|
+
return stats
|
|
704
|
+
rows = conn.execute(
|
|
705
|
+
"""
|
|
706
|
+
SELECT d.dir_id, d.path, d.quick_fingerprint, d.root_id, r.root_path, r.depth
|
|
707
|
+
FROM local_index_dirs d
|
|
708
|
+
LEFT JOIN local_index_roots r ON r.id = d.root_id
|
|
709
|
+
WHERE d.status='active'
|
|
710
|
+
ORDER BY d.updated_at ASC
|
|
711
|
+
LIMIT ?
|
|
712
|
+
""",
|
|
713
|
+
(int(dir_limit),),
|
|
714
|
+
).fetchall()
|
|
715
|
+
seen_at = now()
|
|
716
|
+
for row in rows:
|
|
717
|
+
stats["checked"] += 1
|
|
718
|
+
dir_path = Path(row["path"])
|
|
719
|
+
root_path = Path(row["root_path"]).expanduser() if row["root_path"] else None
|
|
720
|
+
if _is_excluded(str(dir_path), exclusions):
|
|
721
|
+
stats["files_deleted"] += _mark_dir_subtree_deleted(conn, str(dir_path), seen_at)
|
|
722
|
+
stats["excluded_dirs"] += 1
|
|
723
|
+
continue
|
|
724
|
+
if root_path is not None and not root_path.exists():
|
|
725
|
+
stats["offline"] += 1
|
|
726
|
+
continue
|
|
727
|
+
try:
|
|
728
|
+
if not dir_path.exists() or not dir_path.is_dir():
|
|
729
|
+
stats["files_deleted"] += _mark_dir_subtree_deleted(conn, str(dir_path), seen_at)
|
|
730
|
+
stats["dirs_deleted"] += 1
|
|
731
|
+
continue
|
|
732
|
+
st = dir_path.stat()
|
|
733
|
+
fingerprint = _dir_fingerprint(dir_path, st)
|
|
734
|
+
except Exception:
|
|
735
|
+
stats["errors"] += 1
|
|
736
|
+
continue
|
|
737
|
+
if fingerprint != row["quick_fingerprint"]:
|
|
738
|
+
stats["changed"] += 1
|
|
739
|
+
_scan_known_directory(
|
|
740
|
+
conn,
|
|
741
|
+
int(row["root_id"] or 0),
|
|
742
|
+
dir_path,
|
|
743
|
+
int(row["depth"] or 2),
|
|
744
|
+
exclusions,
|
|
745
|
+
stats,
|
|
746
|
+
file_limit=file_limit,
|
|
747
|
+
dir_limit=dir_limit,
|
|
748
|
+
)
|
|
749
|
+
else:
|
|
750
|
+
conn.execute("UPDATE local_index_dirs SET updated_at=? WHERE dir_id=?", (seen_at, row["dir_id"]))
|
|
751
|
+
return stats
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def reconcile_live_changes(
|
|
755
|
+
*,
|
|
756
|
+
asset_limit: int = DEFAULT_LIVE_ASSET_LIMIT,
|
|
757
|
+
dir_limit: int = DEFAULT_LIVE_DIR_LIMIT,
|
|
758
|
+
file_limit: int = DEFAULT_LIVE_FILE_LIMIT,
|
|
759
|
+
) -> dict:
|
|
760
|
+
conn = _conn()
|
|
761
|
+
if _is_paused():
|
|
762
|
+
return {"ok": True, "paused": True, "assets": {}, "dirs": {}}
|
|
763
|
+
exclusions = [row["path"] for row in list_exclusions()]
|
|
764
|
+
asset_stats = _reconcile_known_assets(conn, exclusions, limit=int(asset_limit or 0))
|
|
765
|
+
dir_stats = _reconcile_known_dirs(conn, exclusions, dir_limit=int(dir_limit or 0), file_limit=int(file_limit or 0))
|
|
766
|
+
conn.commit()
|
|
767
|
+
changed_total = (
|
|
768
|
+
int(asset_stats.get("modified", 0))
|
|
769
|
+
+ int(asset_stats.get("deleted", 0))
|
|
770
|
+
+ int(asset_stats.get("excluded", 0))
|
|
771
|
+
+ int(dir_stats.get("files_changed", 0))
|
|
772
|
+
+ int(dir_stats.get("files_deleted", 0))
|
|
773
|
+
+ int(dir_stats.get("dirs_deleted", 0))
|
|
774
|
+
+ int(dir_stats.get("excluded_dirs", 0))
|
|
775
|
+
)
|
|
776
|
+
if changed_total:
|
|
777
|
+
log_event("info", "live_reconcile_finished", "Local memory live changes reconciled", assets=asset_stats, dirs=dir_stats)
|
|
778
|
+
return {"ok": True, "assets": asset_stats, "dirs": dir_stats}
|
|
779
|
+
|
|
780
|
+
|
|
420
781
|
def scan_once(*, limit: int | None = None) -> dict:
|
|
421
782
|
conn = _conn()
|
|
422
783
|
if _is_paused():
|
|
@@ -445,7 +806,15 @@ def scan_once(*, limit: int | None = None) -> dict:
|
|
|
445
806
|
cycle_started_at = float(checkpoint["cycle_started_at"])
|
|
446
807
|
seen_for_root = 0
|
|
447
808
|
last_seen_path = ""
|
|
448
|
-
for file_path in _iter_files(
|
|
809
|
+
for file_path in _iter_files(
|
|
810
|
+
conn,
|
|
811
|
+
root_id,
|
|
812
|
+
root_path,
|
|
813
|
+
exclusions,
|
|
814
|
+
limit=limit,
|
|
815
|
+
start_after=str(checkpoint["current_path"] or ""),
|
|
816
|
+
seen_at=cycle_started_at,
|
|
817
|
+
):
|
|
449
818
|
asset_id, changed, state = _upsert_asset(conn, root_id, file_path, cycle_started_at, int(root["depth"] or 2))
|
|
450
819
|
last_seen_path = norm_path(file_path)
|
|
451
820
|
totals["seen"] += 1
|
|
@@ -620,12 +989,25 @@ def process_jobs(*, limit: int = 100) -> dict:
|
|
|
620
989
|
return {"ok": True, "processed": processed, "failed": failed}
|
|
621
990
|
|
|
622
991
|
|
|
623
|
-
def run_once(
|
|
992
|
+
def run_once(
|
|
993
|
+
*,
|
|
994
|
+
root: str | None = None,
|
|
995
|
+
limit: int | None = None,
|
|
996
|
+
process_limit: int = 100,
|
|
997
|
+
live_asset_limit: int = DEFAULT_LIVE_ASSET_LIMIT,
|
|
998
|
+
live_dir_limit: int = DEFAULT_LIVE_DIR_LIMIT,
|
|
999
|
+
live_file_limit: int = DEFAULT_LIVE_FILE_LIMIT,
|
|
1000
|
+
) -> dict:
|
|
624
1001
|
if root:
|
|
625
1002
|
add_root(root)
|
|
1003
|
+
live_result = reconcile_live_changes(
|
|
1004
|
+
asset_limit=live_asset_limit,
|
|
1005
|
+
dir_limit=live_dir_limit,
|
|
1006
|
+
file_limit=live_file_limit,
|
|
1007
|
+
)
|
|
626
1008
|
scan_result = scan_once(limit=limit)
|
|
627
1009
|
job_result = process_jobs(limit=process_limit)
|
|
628
|
-
return {"ok": True, "scan": scan_result, "jobs": job_result}
|
|
1010
|
+
return {"ok": True, "live": live_result, "scan": scan_result, "jobs": job_result}
|
|
629
1011
|
|
|
630
1012
|
|
|
631
1013
|
def _problem_rows(conn) -> list[dict]:
|
|
@@ -653,6 +1035,136 @@ def _problem_rows(conn) -> list[dict]:
|
|
|
653
1035
|
]
|
|
654
1036
|
|
|
655
1037
|
|
|
1038
|
+
def _command_output(args: list[str], *, timeout: int = 2) -> tuple[int, str, str]:
|
|
1039
|
+
try:
|
|
1040
|
+
result = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
|
1041
|
+
except FileNotFoundError as exc:
|
|
1042
|
+
return 127, "", str(exc)
|
|
1043
|
+
except subprocess.TimeoutExpired as exc:
|
|
1044
|
+
return 124, exc.stdout or "", exc.stderr or "timeout"
|
|
1045
|
+
except Exception as exc:
|
|
1046
|
+
return 1, "", str(exc)
|
|
1047
|
+
return result.returncode, result.stdout or "", result.stderr or ""
|
|
1048
|
+
|
|
1049
|
+
|
|
1050
|
+
def _process_running(pattern: str) -> bool:
|
|
1051
|
+
if system_label() == "windows":
|
|
1052
|
+
command = (
|
|
1053
|
+
"$pattern = '" + pattern.replace("'", "''") + "'; "
|
|
1054
|
+
"$match = Get-CimInstance Win32_Process | "
|
|
1055
|
+
"Where-Object { $_.CommandLine -like \"*$pattern*\" } | "
|
|
1056
|
+
"Select-Object -First 1 -ExpandProperty ProcessId; "
|
|
1057
|
+
"if ($match) { Write-Output $match }"
|
|
1058
|
+
)
|
|
1059
|
+
code, stdout, _ = _command_output(["powershell", "-NoProfile", "-Command", command], timeout=4)
|
|
1060
|
+
return code == 0 and bool(stdout.strip())
|
|
1061
|
+
|
|
1062
|
+
code, stdout, _ = _command_output(["pgrep", "-f", pattern], timeout=2)
|
|
1063
|
+
if code == 0 and stdout.strip():
|
|
1064
|
+
return True
|
|
1065
|
+
code, stdout, _ = _command_output(["ps", "aux"], timeout=2)
|
|
1066
|
+
return code == 0 and pattern in stdout
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _macos_local_index_service_status() -> dict:
|
|
1070
|
+
plist_path = Path.home() / "Library" / "LaunchAgents" / f"{LOCAL_INDEX_SERVICE_LABEL}.plist"
|
|
1071
|
+
installed = plist_path.is_file()
|
|
1072
|
+
running = False
|
|
1073
|
+
active_process = False
|
|
1074
|
+
pid = ""
|
|
1075
|
+
|
|
1076
|
+
code, stdout, _ = _command_output(["launchctl", "list"], timeout=2)
|
|
1077
|
+
if code == 0:
|
|
1078
|
+
for line in stdout.splitlines():
|
|
1079
|
+
parts = line.split()
|
|
1080
|
+
if len(parts) >= 3 and parts[-1] == LOCAL_INDEX_SERVICE_LABEL:
|
|
1081
|
+
installed = True
|
|
1082
|
+
pid = parts[0]
|
|
1083
|
+
running = True
|
|
1084
|
+
active_process = pid.isdigit() and int(pid) > 0
|
|
1085
|
+
break
|
|
1086
|
+
|
|
1087
|
+
if not installed:
|
|
1088
|
+
code, _, _ = _command_output(["launchctl", "print", f"gui/{os.getuid()}/{LOCAL_INDEX_SERVICE_LABEL}"], timeout=2)
|
|
1089
|
+
installed = code == 0
|
|
1090
|
+
|
|
1091
|
+
if not active_process:
|
|
1092
|
+
active_process = _process_running(LOCAL_INDEX_SCRIPT_NAME)
|
|
1093
|
+
running = running or active_process
|
|
1094
|
+
|
|
1095
|
+
return {
|
|
1096
|
+
"installed": installed,
|
|
1097
|
+
"running": running,
|
|
1098
|
+
"active_process": active_process,
|
|
1099
|
+
"manager": "launchagent",
|
|
1100
|
+
"label": LOCAL_INDEX_SERVICE_LABEL,
|
|
1101
|
+
"pid": pid,
|
|
1102
|
+
"config_path": str(plist_path),
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def _windows_local_index_service_status() -> dict:
|
|
1107
|
+
command = (
|
|
1108
|
+
"$task = Get-ScheduledTask -TaskName 'NEXO Local Memory' -ErrorAction SilentlyContinue; "
|
|
1109
|
+
"if ($task) { Write-Output $task.State }"
|
|
1110
|
+
)
|
|
1111
|
+
code, stdout, _ = _command_output(["powershell", "-NoProfile", "-Command", command], timeout=4)
|
|
1112
|
+
task_state = stdout.strip()
|
|
1113
|
+
task_state_key = task_state.lower()
|
|
1114
|
+
installed = code == 0 and bool(task_state)
|
|
1115
|
+
active_process = task_state_key == "running"
|
|
1116
|
+
if not active_process:
|
|
1117
|
+
active_process = _process_running(LOCAL_INDEX_SCRIPT_NAME)
|
|
1118
|
+
running = task_state_key in {"ready", "running"} or active_process
|
|
1119
|
+
return {
|
|
1120
|
+
"installed": installed,
|
|
1121
|
+
"running": running,
|
|
1122
|
+
"active_process": active_process,
|
|
1123
|
+
"manager": "scheduled_task",
|
|
1124
|
+
"task_name": LOCAL_INDEX_WINDOWS_TASK,
|
|
1125
|
+
"task_state": task_state,
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
|
|
1129
|
+
def _linux_local_index_service_status() -> dict:
|
|
1130
|
+
unit_dir = Path.home() / ".config" / "systemd" / "user"
|
|
1131
|
+
unit_path = unit_dir / LOCAL_INDEX_LINUX_UNIT
|
|
1132
|
+
timer_path = unit_dir / "nexo-local-index.timer"
|
|
1133
|
+
installed = unit_path.is_file() or timer_path.is_file()
|
|
1134
|
+
|
|
1135
|
+
code, stdout, _ = _command_output(["systemctl", "--user", "is-active", LOCAL_INDEX_LINUX_UNIT], timeout=2)
|
|
1136
|
+
unit_state = stdout.strip()
|
|
1137
|
+
running = code == 0 and unit_state == "active"
|
|
1138
|
+
active_process = _process_running(LOCAL_INDEX_SCRIPT_NAME)
|
|
1139
|
+
running = running or active_process
|
|
1140
|
+
|
|
1141
|
+
return {
|
|
1142
|
+
"installed": installed,
|
|
1143
|
+
"running": running,
|
|
1144
|
+
"active_process": active_process,
|
|
1145
|
+
"manager": "systemd_user",
|
|
1146
|
+
"unit": LOCAL_INDEX_LINUX_UNIT,
|
|
1147
|
+
"unit_state": unit_state,
|
|
1148
|
+
"config_path": str(unit_path),
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def _local_index_service_status() -> dict:
|
|
1153
|
+
platform_value = system_label()
|
|
1154
|
+
if platform_value == "macos":
|
|
1155
|
+
service = _macos_local_index_service_status()
|
|
1156
|
+
elif platform_value == "windows":
|
|
1157
|
+
service = _windows_local_index_service_status()
|
|
1158
|
+
else:
|
|
1159
|
+
service = _linux_local_index_service_status()
|
|
1160
|
+
service.setdefault("installed", False)
|
|
1161
|
+
service.setdefault("running", False)
|
|
1162
|
+
service["platform"] = platform_value
|
|
1163
|
+
service["started_at"] = ""
|
|
1164
|
+
service["last_heartbeat_at"] = ""
|
|
1165
|
+
return service
|
|
1166
|
+
|
|
1167
|
+
|
|
656
1168
|
def status() -> dict:
|
|
657
1169
|
conn = _conn()
|
|
658
1170
|
paused = _is_paused()
|
|
@@ -670,16 +1182,11 @@ def status() -> dict:
|
|
|
670
1182
|
).fetchall()
|
|
671
1183
|
for row in by_volume:
|
|
672
1184
|
volumes.append({"id": row["volume_id"], "label": row["volume_id"] or "Disk", "files": row["files"], "status": "active"})
|
|
1185
|
+
service = _local_index_service_status()
|
|
1186
|
+
service["state"] = "paused" if paused else ("idle" if pending == 0 else "indexing")
|
|
673
1187
|
return {
|
|
674
1188
|
"ok": True,
|
|
675
|
-
"service":
|
|
676
|
-
"installed": False,
|
|
677
|
-
"running": False,
|
|
678
|
-
"state": "paused" if paused else ("idle" if pending == 0 else "indexing"),
|
|
679
|
-
"platform": system_label(),
|
|
680
|
-
"started_at": "",
|
|
681
|
-
"last_heartbeat_at": "",
|
|
682
|
-
},
|
|
1189
|
+
"service": service,
|
|
683
1190
|
"global": {
|
|
684
1191
|
"phase": "paused" if paused else ("idle" if pending == 0 else "light_extraction"),
|
|
685
1192
|
"percent": percent,
|
|
@@ -820,6 +1327,21 @@ def context_query(query: str, *, intent: str = "answer", limit: int = 12, eviden
|
|
|
820
1327
|
(f"%{query.lower()}%", int(limit)),
|
|
821
1328
|
).fetchall()
|
|
822
1329
|
entities_payload = [dict(row) for row in entity_rows]
|
|
1330
|
+
relations_payload: list[dict] = []
|
|
1331
|
+
if seen_assets:
|
|
1332
|
+
asset_ids = list(seen_assets)[: int(limit)]
|
|
1333
|
+
placeholders = ",".join("?" for _ in asset_ids)
|
|
1334
|
+
relation_rows = conn.execute(
|
|
1335
|
+
f"""
|
|
1336
|
+
SELECT relation_id, source_asset_id, target_ref, relation_type, confidence, evidence
|
|
1337
|
+
FROM local_relations
|
|
1338
|
+
WHERE active=1 AND source_asset_id IN ({placeholders})
|
|
1339
|
+
ORDER BY confidence DESC
|
|
1340
|
+
LIMIT ?
|
|
1341
|
+
""",
|
|
1342
|
+
[*asset_ids, int(limit) * 3],
|
|
1343
|
+
).fetchall()
|
|
1344
|
+
relations_payload = [dict(row) for row in relation_rows]
|
|
823
1345
|
warnings = []
|
|
824
1346
|
if evidence_required and not evidence_refs:
|
|
825
1347
|
warnings.append("No local evidence found for this query.")
|
|
@@ -849,7 +1371,7 @@ def context_query(query: str, *, intent: str = "answer", limit: int = 12, eviden
|
|
|
849
1371
|
"summary": summary,
|
|
850
1372
|
"assets": assets,
|
|
851
1373
|
"entities": entities_payload,
|
|
852
|
-
"relations":
|
|
1374
|
+
"relations": relations_payload,
|
|
853
1375
|
"chunks": chunks,
|
|
854
1376
|
"warnings": warnings,
|
|
855
1377
|
"evidence_refs": evidence_refs,
|
|
@@ -899,6 +1421,7 @@ def clear_index() -> dict:
|
|
|
899
1421
|
"local_chunks",
|
|
900
1422
|
"local_entities",
|
|
901
1423
|
"local_relations",
|
|
1424
|
+
"local_index_dirs",
|
|
902
1425
|
"local_index_errors",
|
|
903
1426
|
"local_index_jobs",
|
|
904
1427
|
"local_asset_versions",
|
|
@@ -36,10 +36,24 @@ NOISY_PARTS = {
|
|
|
36
36
|
"dist",
|
|
37
37
|
"build",
|
|
38
38
|
".git",
|
|
39
|
+
".venv",
|
|
40
|
+
"venv",
|
|
41
|
+
"env",
|
|
39
42
|
".cache",
|
|
40
43
|
"cache",
|
|
41
44
|
"coverage",
|
|
42
45
|
"__pycache__",
|
|
46
|
+
".tox",
|
|
47
|
+
".mypy_cache",
|
|
48
|
+
".pytest_cache",
|
|
49
|
+
".ruff_cache",
|
|
50
|
+
".next",
|
|
51
|
+
".nuxt",
|
|
52
|
+
".turbo",
|
|
53
|
+
".parcel-cache",
|
|
54
|
+
".bun",
|
|
55
|
+
".gradle",
|
|
56
|
+
"target",
|
|
43
57
|
}
|
|
44
58
|
|
|
45
59
|
SYSTEM_PARTS = {
|
|
@@ -71,6 +85,15 @@ def classify_path(path: str) -> tuple[int, str, str]:
|
|
|
71
85
|
return 2, "normal", "default"
|
|
72
86
|
|
|
73
87
|
|
|
88
|
+
def should_skip_tree(path: str) -> bool:
|
|
89
|
+
p = Path(path)
|
|
90
|
+
lowered = str(p).replace("\\", "/").lower()
|
|
91
|
+
parts = {part.lower() for part in p.parts}
|
|
92
|
+
if any(item in lowered for item in SYSTEM_PARTS):
|
|
93
|
+
return True
|
|
94
|
+
return bool(parts & NOISY_PARTS)
|
|
95
|
+
|
|
96
|
+
|
|
74
97
|
def should_extract(path: str, depth: int) -> bool:
|
|
75
98
|
if depth < 2:
|
|
76
99
|
return False
|
package/src/runtime_power.py
CHANGED
|
@@ -920,7 +920,20 @@ def ensure_full_disk_access_choice(
|
|
|
920
920
|
probe = probe_fn()
|
|
921
921
|
if probe.get("granted") is True:
|
|
922
922
|
verified = True
|
|
923
|
-
|
|
923
|
+
schedule[FULL_DISK_ACCESS_STATUS_KEY] = FULL_DISK_ACCESS_GRANTED
|
|
924
|
+
schedule[FULL_DISK_ACCESS_REASONS_KEY] = []
|
|
925
|
+
clear_full_disk_access_required_state()
|
|
926
|
+
save_schedule_config(schedule)
|
|
927
|
+
return {
|
|
928
|
+
"status": FULL_DISK_ACCESS_GRANTED,
|
|
929
|
+
"prompted": False,
|
|
930
|
+
"verified": True,
|
|
931
|
+
"settings_opened": False,
|
|
932
|
+
"reasons": [],
|
|
933
|
+
"schedule_file": str(SCHEDULE_FILE),
|
|
934
|
+
"message": "",
|
|
935
|
+
"relevant": False,
|
|
936
|
+
}
|
|
924
937
|
else:
|
|
925
938
|
status = FULL_DISK_ACCESS_LATER
|
|
926
939
|
message = (
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
import contextlib
|
|
4
5
|
import hashlib
|
|
5
6
|
import json
|
|
6
7
|
import os
|
|
7
8
|
import re
|
|
8
9
|
import shutil
|
|
10
|
+
import sys
|
|
9
11
|
import time
|
|
10
12
|
from dataclasses import dataclass
|
|
11
13
|
from pathlib import Path
|
|
@@ -814,6 +816,31 @@ def prime_process_fingerprint() -> str:
|
|
|
814
816
|
return PROCESS_FINGERPRINT
|
|
815
817
|
|
|
816
818
|
|
|
819
|
+
_DRIFT_AUTOEXIT_SCHEDULED = False
|
|
820
|
+
_DRIFT_EXIT_CODE = 75
|
|
821
|
+
_DRIFT_EXIT_DELAY_SECONDS = 0.5
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _request_drift_exit() -> None:
|
|
825
|
+
try:
|
|
826
|
+
os._exit(_DRIFT_EXIT_CODE)
|
|
827
|
+
except Exception:
|
|
828
|
+
os._exit(1)
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _schedule_drift_autoexit() -> None:
|
|
832
|
+
global _DRIFT_AUTOEXIT_SCHEDULED
|
|
833
|
+
if _DRIFT_AUTOEXIT_SCHEDULED:
|
|
834
|
+
return
|
|
835
|
+
_DRIFT_AUTOEXIT_SCHEDULED = True
|
|
836
|
+
try:
|
|
837
|
+
loop = asyncio.get_running_loop()
|
|
838
|
+
except RuntimeError:
|
|
839
|
+
_request_drift_exit()
|
|
840
|
+
return
|
|
841
|
+
loop.call_later(_DRIFT_EXIT_DELAY_SECONDS, _request_drift_exit)
|
|
842
|
+
|
|
843
|
+
|
|
817
844
|
@dataclass
|
|
818
845
|
class RestartRequiredMiddleware(Middleware):
|
|
819
846
|
client: str = ""
|
|
@@ -893,4 +920,7 @@ class RestartRequiredMiddleware(Middleware):
|
|
|
893
920
|
"reason": state["reason"],
|
|
894
921
|
"client_action": state["client_action"],
|
|
895
922
|
}
|
|
896
|
-
|
|
923
|
+
result = await self._tool_result_for_restart_required(context, payload)
|
|
924
|
+
if state.get("reason") == "fingerprint_mismatch":
|
|
925
|
+
_schedule_drift_autoexit()
|
|
926
|
+
return result
|
|
@@ -39,6 +39,9 @@ LOCK_FILE = LOG_DIR / "local-index.lock"
|
|
|
39
39
|
LOCK_STALE_SECONDS = int(os.environ.get("NEXO_LOCAL_INDEX_LOCK_STALE_SECONDS", "1800") or "1800")
|
|
40
40
|
SCAN_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_SCAN_LIMIT", "1000") or "1000")
|
|
41
41
|
PROCESS_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_PROCESS_LIMIT", "200") or "200")
|
|
42
|
+
LIVE_ASSET_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_ASSET_LIMIT", "2000") or "2000")
|
|
43
|
+
LIVE_DIR_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_DIR_LIMIT", "300") or "300")
|
|
44
|
+
LIVE_FILE_LIMIT = int(os.environ.get("NEXO_LOCAL_INDEX_LIVE_FILE_LIMIT", "1000") or "1000")
|
|
42
45
|
|
|
43
46
|
|
|
44
47
|
def log(message: str) -> None:
|
|
@@ -79,7 +82,13 @@ def main() -> int:
|
|
|
79
82
|
try:
|
|
80
83
|
if os.environ.get("NEXO_LOCAL_INDEX_DISABLE_DEFAULT_ROOTS", "").strip() != "1":
|
|
81
84
|
api.ensure_default_roots()
|
|
82
|
-
result = api.run_once(
|
|
85
|
+
result = api.run_once(
|
|
86
|
+
limit=SCAN_LIMIT,
|
|
87
|
+
process_limit=PROCESS_LIMIT,
|
|
88
|
+
live_asset_limit=LIVE_ASSET_LIMIT,
|
|
89
|
+
live_dir_limit=LIVE_DIR_LIMIT,
|
|
90
|
+
live_file_limit=LIVE_FILE_LIMIT,
|
|
91
|
+
)
|
|
83
92
|
log_event("info", "service_cycle_finished", "Local memory service cycle finished", result=result)
|
|
84
93
|
log(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
85
94
|
return 0 if result.get("ok") else 2
|