nexo-brain 7.23.2 → 7.23.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +7 -1
- package/package.json +1 -1
- package/scripts/sync_release_artifacts.py +28 -0
- package/src/auto_update.py +25 -47
- package/src/automation_reconciler.py +383 -0
- package/src/automation_supervisor.py +86 -9
- package/src/backup_retention.py +70 -0
- package/src/cli.py +55 -2
- package/src/cognitive/_core.py +4 -3
- package/src/cognitive_paths.py +194 -0
- package/src/dashboard/app.py +2 -1
- package/src/db/_episodic.py +85 -7
- package/src/db/_schema.py +81 -0
- package/src/db/_skills.py +3 -3
- package/src/disk_recovery/__init__.py +11 -0
- package/src/disk_recovery/handlers/__init__.py +1 -0
- package/src/disk_recovery/handlers/common.py +37 -0
- package/src/disk_recovery/handlers/macos.py +39 -0
- package/src/disk_recovery/handlers/windows.py +49 -0
- package/src/disk_recovery/registry.py +135 -0
- package/src/doctor/providers/boot.py +115 -15
- package/src/kg_populate.py +2 -5
- package/src/paths.py +321 -5
- package/src/plugins/update.py +14 -36
- package/src/pre_answer_router.py +21 -0
- package/src/runtime_service.py +30 -3
- package/src/runtime_versioning.py +272 -10
- package/src/script_registry.py +3 -2
- package/src/scripts/backfill_task_owner.py +10 -4
- package/src/scripts/deep-sleep/apply_findings.py +2 -5
- package/src/scripts/deep-sleep/collect.py +2 -5
- package/src/scripts/nexo-cognitive-decay.py +2 -1
- package/src/scripts/nexo-daily-self-audit.py +36 -10
- package/src/scripts/nexo-followup-runner.py +1 -1
- package/src/scripts/nexo-immune.py +2 -1
- package/src/scripts/nexo-migrate.py +2 -3
- package/src/scripts/post_disk_recovery_sweep.py +75 -0
- package/src/scripts/prune_runtime_backups.py +78 -11
- package/src/server.py +13 -1
- package/src/storage_router.py +2 -3
- package/src/support_snapshot.py +25 -0
- package/src/transcript_index.py +234 -0
- package/src/transcript_utils.py +31 -8
- package/src/user_data_portability.py +2 -3
- package/tool-enforcement-map.json +15 -0
|
@@ -2,8 +2,11 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import os
|
|
5
|
+
import json
|
|
5
6
|
import shutil
|
|
7
|
+
import subprocess
|
|
6
8
|
import sys
|
|
9
|
+
import time
|
|
7
10
|
from pathlib import Path
|
|
8
11
|
|
|
9
12
|
from doctor.models import DoctorCheck, safe_check
|
|
@@ -356,33 +359,130 @@ def check_required_dirs() -> DoctorCheck:
|
|
|
356
359
|
)
|
|
357
360
|
|
|
358
361
|
|
|
362
|
+
def _disk_recovery_state_file(paths_module) -> Path:
|
|
363
|
+
return paths_module.runtime_state_dir() / "disk-recovery-state.json"
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _read_disk_recovery_state(paths_module) -> dict:
|
|
367
|
+
try:
|
|
368
|
+
path = _disk_recovery_state_file(paths_module)
|
|
369
|
+
if path.is_file():
|
|
370
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
371
|
+
except Exception:
|
|
372
|
+
pass
|
|
373
|
+
return {}
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _write_disk_recovery_state(paths_module, payload: dict) -> None:
|
|
377
|
+
try:
|
|
378
|
+
path = _disk_recovery_state_file(paths_module)
|
|
379
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
380
|
+
path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
|
|
381
|
+
except Exception:
|
|
382
|
+
pass
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _post_disk_recovery_sweep(paths_module, *, reason: str, free_bytes: int) -> dict:
|
|
386
|
+
candidates = [
|
|
387
|
+
Path(__file__).resolve().parents[2] / "scripts" / "post_disk_recovery_sweep.py",
|
|
388
|
+
paths_module.core_scripts_dir() / "post_disk_recovery_sweep.py",
|
|
389
|
+
]
|
|
390
|
+
script = next((candidate for candidate in candidates if candidate.is_file()), None)
|
|
391
|
+
if script is None:
|
|
392
|
+
return {"ok": False, "skipped": True, "reason": "script_missing"}
|
|
393
|
+
try:
|
|
394
|
+
proc = subprocess.run(
|
|
395
|
+
[
|
|
396
|
+
sys.executable,
|
|
397
|
+
str(script),
|
|
398
|
+
"--reason",
|
|
399
|
+
reason,
|
|
400
|
+
"--json",
|
|
401
|
+
"--network-window-seconds",
|
|
402
|
+
"0",
|
|
403
|
+
],
|
|
404
|
+
capture_output=True,
|
|
405
|
+
text=True,
|
|
406
|
+
timeout=60,
|
|
407
|
+
)
|
|
408
|
+
return {
|
|
409
|
+
"ok": proc.returncode == 0,
|
|
410
|
+
"returncode": proc.returncode,
|
|
411
|
+
"free_bytes": free_bytes,
|
|
412
|
+
"stdout": proc.stdout[-1000:],
|
|
413
|
+
"stderr": proc.stderr[-1000:],
|
|
414
|
+
}
|
|
415
|
+
except Exception as exc:
|
|
416
|
+
return {"ok": False, "error": str(exc), "free_bytes": free_bytes}
|
|
417
|
+
|
|
418
|
+
|
|
359
419
|
def check_disk_space() -> DoctorCheck:
|
|
360
|
-
"""Check disk free space
|
|
420
|
+
"""Check disk free space after silently purging NEXO-owned backups."""
|
|
421
|
+
import paths
|
|
422
|
+
|
|
361
423
|
try:
|
|
362
|
-
|
|
424
|
+
state = _read_disk_recovery_state(paths)
|
|
425
|
+
floor = int(paths.backup_min_free_bytes())
|
|
426
|
+
critical_floor = 1 * 1024 ** 3
|
|
427
|
+
usage_before = shutil.disk_usage(str(paths.home()))
|
|
428
|
+
cleanup_report = None
|
|
429
|
+
|
|
430
|
+
if usage_before.free < floor:
|
|
431
|
+
cleanup_report = paths.aggressive_runtime_backup_prune(
|
|
432
|
+
min_free_bytes=floor,
|
|
433
|
+
reason="doctor_boot_disk_space",
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
usage = shutil.disk_usage(str(paths.home()))
|
|
363
437
|
free_gb = usage.free / (1024 ** 3)
|
|
364
438
|
pct_free = (usage.free / usage.total) * 100
|
|
365
|
-
|
|
366
|
-
if
|
|
439
|
+
evidence = [f"Total: {usage.total / (1024**3):.0f} GB, Free: {free_gb:.1f} GB"]
|
|
440
|
+
if cleanup_report:
|
|
441
|
+
evidence.append(f"NEXO backup self-cleanup: {cleanup_report.get('steps')}")
|
|
442
|
+
|
|
443
|
+
if usage.free < floor:
|
|
444
|
+
_write_disk_recovery_state(paths, {
|
|
445
|
+
"low": True,
|
|
446
|
+
"free_bytes": int(usage.free),
|
|
447
|
+
"threshold_bytes": floor,
|
|
448
|
+
"updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
449
|
+
})
|
|
450
|
+
status = "critical" if usage.free < critical_floor else "degraded"
|
|
451
|
+
severity = "error" if status == "critical" else "warn"
|
|
367
452
|
return DoctorCheck(
|
|
368
453
|
id="boot.disk_space",
|
|
369
454
|
tier="boot",
|
|
370
|
-
status=
|
|
371
|
-
severity=
|
|
372
|
-
summary=f"
|
|
373
|
-
evidence=
|
|
374
|
-
repair_plan=["
|
|
375
|
-
escalation_prompt="Disk
|
|
455
|
+
status=status,
|
|
456
|
+
severity=severity,
|
|
457
|
+
summary=f"Disk almost full ({free_gb:.1f} GB free). NEXO has already cleaned up its own backups. Review personal files to free space.",
|
|
458
|
+
evidence=evidence,
|
|
459
|
+
repair_plan=["Open the user folder and review personal files to free space"],
|
|
460
|
+
escalation_prompt=f"Disk almost full ({free_gb:.1f} GB free). NEXO has already cleaned up its own backups. Review personal files to free space.",
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
if usage_before.free < floor or state.get("low"):
|
|
464
|
+
sweep = _post_disk_recovery_sweep(
|
|
465
|
+
paths,
|
|
466
|
+
reason="doctor_disk_low_to_ok",
|
|
467
|
+
free_bytes=int(usage.free),
|
|
376
468
|
)
|
|
377
|
-
|
|
469
|
+
_write_disk_recovery_state(paths, {
|
|
470
|
+
"low": False,
|
|
471
|
+
"free_bytes": int(usage.free),
|
|
472
|
+
"threshold_bytes": floor,
|
|
473
|
+
"last_recovery_sweep": sweep,
|
|
474
|
+
"updated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
475
|
+
})
|
|
378
476
|
return DoctorCheck(
|
|
379
477
|
id="boot.disk_space",
|
|
380
478
|
tier="boot",
|
|
381
|
-
status="
|
|
382
|
-
severity="
|
|
383
|
-
summary=f"
|
|
384
|
-
evidence=[f"
|
|
479
|
+
status="healthy",
|
|
480
|
+
severity="info",
|
|
481
|
+
summary=f"Disk space recovered after NEXO backup self-cleanup: {free_gb:.1f} GB free ({pct_free:.0f}%)",
|
|
482
|
+
evidence=evidence + [f"Post-disk recovery sweep: {sweep}"],
|
|
483
|
+
fixed=True,
|
|
385
484
|
)
|
|
485
|
+
|
|
386
486
|
return DoctorCheck(
|
|
387
487
|
id="boot.disk_space",
|
|
388
488
|
tier="boot",
|
package/src/kg_populate.py
CHANGED
|
@@ -5,6 +5,7 @@ import os
|
|
|
5
5
|
import sqlite3
|
|
6
6
|
from typing import Optional
|
|
7
7
|
|
|
8
|
+
from cognitive_paths import resolve_cognitive_db
|
|
8
9
|
import knowledge_graph as kg
|
|
9
10
|
from db import get_db
|
|
10
11
|
|
|
@@ -13,11 +14,7 @@ from db import get_db
|
|
|
13
14
|
|
|
14
15
|
def _cognitive_db():
|
|
15
16
|
"""Direct cognitive.db connection (for somatic_markers)."""
|
|
16
|
-
|
|
17
|
-
data_dir = os.path.join(nexo_home, "data")
|
|
18
|
-
os.makedirs(data_dir, exist_ok=True)
|
|
19
|
-
path = os.path.join(data_dir, "cognitive.db")
|
|
20
|
-
conn = sqlite3.connect(path)
|
|
17
|
+
conn = sqlite3.connect(str(resolve_cognitive_db(for_write=True)))
|
|
21
18
|
conn.row_factory = sqlite3.Row
|
|
22
19
|
return conn
|
|
23
20
|
|
package/src/paths.py
CHANGED
|
@@ -55,6 +55,11 @@ that hardcoded the old paths keeps resolving via symlink during the
|
|
|
55
55
|
from __future__ import annotations
|
|
56
56
|
|
|
57
57
|
import os
|
|
58
|
+
import json
|
|
59
|
+
import shutil
|
|
60
|
+
import subprocess
|
|
61
|
+
import sys
|
|
62
|
+
import time
|
|
58
63
|
from pathlib import Path
|
|
59
64
|
|
|
60
65
|
NEXO_HOME = Path(os.environ.get("NEXO_HOME", str(Path.home() / ".nexo")))
|
|
@@ -320,6 +325,321 @@ def backups_dir() -> Path:
|
|
|
320
325
|
return new
|
|
321
326
|
|
|
322
327
|
|
|
328
|
+
_GiB = 1024 ** 3
|
|
329
|
+
BACKUP_DEFAULT_MAX_BYTES = 50 * _GiB
|
|
330
|
+
BACKUP_MIN_CAP_BYTES = 10 * _GiB
|
|
331
|
+
BACKUP_MAX_CAP_BYTES = 50 * _GiB
|
|
332
|
+
BACKUP_DEFAULT_MIN_FREE_BYTES = 5 * _GiB
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class BackupSnapshotPath:
|
|
336
|
+
"""Path returned by backup helpers; usable as a post-pruning context."""
|
|
337
|
+
|
|
338
|
+
def __init__(self, value: str | Path, *, backups_root: Path | None = None):
|
|
339
|
+
self._path = Path(value)
|
|
340
|
+
self._nexo_backups_root = Path(backups_root) if backups_root is not None else None
|
|
341
|
+
self._nexo_finalized = False
|
|
342
|
+
|
|
343
|
+
def __fspath__(self) -> str:
|
|
344
|
+
return os.fspath(self._path)
|
|
345
|
+
|
|
346
|
+
def __str__(self) -> str:
|
|
347
|
+
return str(self._path)
|
|
348
|
+
|
|
349
|
+
def __repr__(self) -> str:
|
|
350
|
+
return f"BackupSnapshotPath({self._path!r})"
|
|
351
|
+
|
|
352
|
+
def __truediv__(self, key):
|
|
353
|
+
return self._path / key
|
|
354
|
+
|
|
355
|
+
def __eq__(self, other) -> bool:
|
|
356
|
+
return self._path == Path(other)
|
|
357
|
+
|
|
358
|
+
def __getattr__(self, name: str):
|
|
359
|
+
return getattr(self._path, name)
|
|
360
|
+
|
|
361
|
+
def __enter__(self):
|
|
362
|
+
return self
|
|
363
|
+
|
|
364
|
+
def __exit__(self, _exc_type, _exc, _tb):
|
|
365
|
+
self.finalize()
|
|
366
|
+
return False
|
|
367
|
+
|
|
368
|
+
def finalize(self) -> dict:
|
|
369
|
+
if self._nexo_finalized:
|
|
370
|
+
return {"ok": True, "skipped": True, "reason": "already_finalized"}
|
|
371
|
+
self._nexo_finalized = True
|
|
372
|
+
return finalize_backup_snapshot(self, backups_root=self._nexo_backups_root)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def parse_size_bytes(value: str | int | None, *, default: int = BACKUP_DEFAULT_MAX_BYTES) -> int:
|
|
376
|
+
"""Parse size strings like ``10G`` / ``512M`` into bytes."""
|
|
377
|
+
if value is None or value == "":
|
|
378
|
+
return default
|
|
379
|
+
if isinstance(value, int):
|
|
380
|
+
return max(0, value)
|
|
381
|
+
raw = str(value).strip().lower()
|
|
382
|
+
if not raw:
|
|
383
|
+
return default
|
|
384
|
+
multiplier = 1
|
|
385
|
+
if raw[-1:] in {"k", "m", "g", "t"}:
|
|
386
|
+
unit = raw[-1]
|
|
387
|
+
raw = raw[:-1].strip()
|
|
388
|
+
multiplier = {"k": 1024, "m": 1024 ** 2, "g": _GiB, "t": 1024 ** 4}[unit]
|
|
389
|
+
try:
|
|
390
|
+
return max(0, int(float(raw) * multiplier))
|
|
391
|
+
except ValueError:
|
|
392
|
+
return default
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def backup_min_free_bytes() -> int:
|
|
396
|
+
return parse_size_bytes(os.environ.get("NEXO_BACKUP_MIN_FREE_BYTES"), default=BACKUP_DEFAULT_MIN_FREE_BYTES)
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def backup_retention_cap_bytes(*, backups_root: Path | None = None, configured: str | int | None = None) -> int:
|
|
400
|
+
"""Return the effective technical-backup cap for this install.
|
|
401
|
+
|
|
402
|
+
The default adapts to the user's disk: 5% of total capacity, floored at
|
|
403
|
+
10 GiB and capped at 50 GiB. ``NEXO_BACKUP_MAX_BYTES`` remains an upper
|
|
404
|
+
bound, and explicit lower caps are allowed for emergency prune steps.
|
|
405
|
+
"""
|
|
406
|
+
raw = parse_size_bytes(
|
|
407
|
+
configured if configured is not None else os.environ.get("NEXO_BACKUP_MAX_BYTES"),
|
|
408
|
+
default=BACKUP_DEFAULT_MAX_BYTES,
|
|
409
|
+
)
|
|
410
|
+
if raw < BACKUP_MIN_CAP_BYTES:
|
|
411
|
+
return raw
|
|
412
|
+
root = Path(backups_root or backups_dir())
|
|
413
|
+
probe = root if root.exists() else root.parent
|
|
414
|
+
try:
|
|
415
|
+
usage = shutil.disk_usage(str(probe))
|
|
416
|
+
adaptive = int(usage.total * 0.05)
|
|
417
|
+
except Exception:
|
|
418
|
+
adaptive = BACKUP_DEFAULT_MAX_BYTES
|
|
419
|
+
adaptive = max(BACKUP_MIN_CAP_BYTES, min(BACKUP_MAX_CAP_BYTES, adaptive))
|
|
420
|
+
return min(raw, adaptive)
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def backup_free_bytes(*, backups_root: Path | None = None) -> int | None:
|
|
424
|
+
root = Path(backups_root or backups_dir())
|
|
425
|
+
probe = root if root.exists() else root.parent
|
|
426
|
+
try:
|
|
427
|
+
return int(shutil.disk_usage(str(probe)).free)
|
|
428
|
+
except Exception:
|
|
429
|
+
return None
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def _backup_pruner_script() -> Path | None:
|
|
433
|
+
candidates = [
|
|
434
|
+
Path(__file__).resolve().parent / "scripts" / "prune_runtime_backups.py",
|
|
435
|
+
core_scripts_dir() / "prune_runtime_backups.py",
|
|
436
|
+
]
|
|
437
|
+
seen: set[str] = set()
|
|
438
|
+
for candidate in candidates:
|
|
439
|
+
key = str(candidate)
|
|
440
|
+
if key in seen:
|
|
441
|
+
continue
|
|
442
|
+
seen.add(key)
|
|
443
|
+
if candidate.is_file():
|
|
444
|
+
return candidate
|
|
445
|
+
return None
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _append_backup_retention_event(event: dict) -> None:
|
|
449
|
+
try:
|
|
450
|
+
log_path = operations_dir() / "backup-retention-events.jsonl"
|
|
451
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
452
|
+
payload = {
|
|
453
|
+
"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
454
|
+
**event,
|
|
455
|
+
"os": sys.platform,
|
|
456
|
+
}
|
|
457
|
+
with log_path.open("a", encoding="utf-8") as fh:
|
|
458
|
+
fh.write(json.dumps(payload, sort_keys=True) + "\n")
|
|
459
|
+
except Exception:
|
|
460
|
+
pass
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def run_runtime_backup_prune(
|
|
464
|
+
*,
|
|
465
|
+
max_bytes: str | int | None = None,
|
|
466
|
+
backups_root: Path | None = None,
|
|
467
|
+
delete_all_technical: bool = False,
|
|
468
|
+
timeout: int = 120,
|
|
469
|
+
) -> dict:
|
|
470
|
+
"""Run the technical-backup pruner. Safe no-op when the script is absent."""
|
|
471
|
+
script = _backup_pruner_script()
|
|
472
|
+
root = Path(backups_root or backups_dir())
|
|
473
|
+
cap = parse_size_bytes(max_bytes, default=backup_retention_cap_bytes(backups_root=root))
|
|
474
|
+
if max_bytes is None:
|
|
475
|
+
cap = backup_retention_cap_bytes(backups_root=root)
|
|
476
|
+
if script is None:
|
|
477
|
+
return {"ok": False, "skipped": True, "reason": "pruner_missing", "root": str(root)}
|
|
478
|
+
args = [
|
|
479
|
+
sys.executable,
|
|
480
|
+
str(script),
|
|
481
|
+
"--root",
|
|
482
|
+
str(root),
|
|
483
|
+
"--apply",
|
|
484
|
+
"--json",
|
|
485
|
+
"--max-bytes",
|
|
486
|
+
str(cap),
|
|
487
|
+
]
|
|
488
|
+
if delete_all_technical:
|
|
489
|
+
args.append("--delete-all-technical")
|
|
490
|
+
try:
|
|
491
|
+
proc = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
|
|
492
|
+
report = json.loads(proc.stdout or "{}") if proc.stdout.strip().startswith("{") else {}
|
|
493
|
+
return {
|
|
494
|
+
"ok": proc.returncode == 0,
|
|
495
|
+
"returncode": proc.returncode,
|
|
496
|
+
"max_bytes": cap,
|
|
497
|
+
"root": str(root),
|
|
498
|
+
"stdout": proc.stdout[-4000:],
|
|
499
|
+
"stderr": proc.stderr[-4000:],
|
|
500
|
+
"report": report,
|
|
501
|
+
}
|
|
502
|
+
except Exception as exc:
|
|
503
|
+
return {"ok": False, "error": str(exc), "max_bytes": cap, "root": str(root)}
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def aggressive_runtime_backup_prune(
|
|
507
|
+
*,
|
|
508
|
+
min_free_bytes: int | None = None,
|
|
509
|
+
backups_root: Path | None = None,
|
|
510
|
+
reason: str = "",
|
|
511
|
+
) -> dict:
|
|
512
|
+
"""Escalate NEXO-owned backup pruning before any user-facing disk alert.
|
|
513
|
+
|
|
514
|
+
Escalation never targets protected business/hourly backup classes; the
|
|
515
|
+
pruner enforces that policy.
|
|
516
|
+
"""
|
|
517
|
+
root = Path(backups_root or backups_dir())
|
|
518
|
+
floor = int(min_free_bytes if min_free_bytes is not None else backup_min_free_bytes())
|
|
519
|
+
steps: list[dict] = []
|
|
520
|
+
escalated = False
|
|
521
|
+
plan = [
|
|
522
|
+
("standard", None, False),
|
|
523
|
+
("cap-10gb", 10 * _GiB, False),
|
|
524
|
+
("cap-5gb", 5 * _GiB, False),
|
|
525
|
+
("delete-all-technical", 0, True),
|
|
526
|
+
]
|
|
527
|
+
for label, cap, delete_all in plan:
|
|
528
|
+
before = backup_free_bytes(backups_root=root)
|
|
529
|
+
if before is not None and before >= floor and steps:
|
|
530
|
+
break
|
|
531
|
+
if label != "standard":
|
|
532
|
+
escalated = True
|
|
533
|
+
result = run_runtime_backup_prune(max_bytes=cap, backups_root=root, delete_all_technical=delete_all)
|
|
534
|
+
after = backup_free_bytes(backups_root=root)
|
|
535
|
+
steps.append({
|
|
536
|
+
"step": label,
|
|
537
|
+
"before_free_bytes": before,
|
|
538
|
+
"after_free_bytes": after,
|
|
539
|
+
"ok": result.get("ok") is True,
|
|
540
|
+
"delete_count": (((result.get("report") or {}).get("totals") or {}).get("delete_count")),
|
|
541
|
+
"delete_bytes": (((result.get("report") or {}).get("totals") or {}).get("delete_bytes")),
|
|
542
|
+
})
|
|
543
|
+
if after is not None and after >= floor:
|
|
544
|
+
break
|
|
545
|
+
final_free = backup_free_bytes(backups_root=root)
|
|
546
|
+
if escalated:
|
|
547
|
+
dominant = ""
|
|
548
|
+
try:
|
|
549
|
+
deletes = []
|
|
550
|
+
for step in steps:
|
|
551
|
+
# Detailed family data is emitted by the script report; keep
|
|
552
|
+
# this anonymous and compact for product telemetry.
|
|
553
|
+
if step.get("delete_bytes"):
|
|
554
|
+
deletes.append((int(step.get("delete_bytes") or 0), step.get("step") or ""))
|
|
555
|
+
dominant = max(deletes)[1] if deletes else ""
|
|
556
|
+
except Exception:
|
|
557
|
+
dominant = ""
|
|
558
|
+
_append_backup_retention_event({
|
|
559
|
+
"event": "backup_prune_escalated",
|
|
560
|
+
"reason": reason,
|
|
561
|
+
"steps": len(steps),
|
|
562
|
+
"dominant_prefix": dominant,
|
|
563
|
+
"final_free_bytes": final_free,
|
|
564
|
+
})
|
|
565
|
+
return {
|
|
566
|
+
"ok": final_free is None or final_free >= floor,
|
|
567
|
+
"root": str(root),
|
|
568
|
+
"min_free_bytes": floor,
|
|
569
|
+
"final_free_bytes": final_free,
|
|
570
|
+
"steps": steps,
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
def backup_space_error(
|
|
575
|
+
*,
|
|
576
|
+
reason: str = "",
|
|
577
|
+
min_free_bytes: int | None = None,
|
|
578
|
+
backups_root: Path | None = None,
|
|
579
|
+
) -> str | None:
|
|
580
|
+
report = aggressive_runtime_backup_prune(
|
|
581
|
+
min_free_bytes=min_free_bytes,
|
|
582
|
+
backups_root=backups_root,
|
|
583
|
+
reason=reason,
|
|
584
|
+
)
|
|
585
|
+
free = report.get("final_free_bytes")
|
|
586
|
+
floor = int(report.get("min_free_bytes") or backup_min_free_bytes())
|
|
587
|
+
if free is not None and free < floor:
|
|
588
|
+
return (
|
|
589
|
+
"free disk below NEXO backup safety floor after NEXO self-cleanup "
|
|
590
|
+
f"({free}B < {floor}B)"
|
|
591
|
+
)
|
|
592
|
+
return None
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def create_backup_dir(prefix: str, *, backups_root: Path | None = None) -> Path:
|
|
596
|
+
"""Create a technical backup directory through the universal guard."""
|
|
597
|
+
clean = str(prefix or "").strip().strip("-")
|
|
598
|
+
if not clean or any(sep in clean for sep in ("/", "\\")):
|
|
599
|
+
raise ValueError("backup prefix must be a single path segment")
|
|
600
|
+
err = backup_space_error(reason=f"create_backup_dir:{clean}", backups_root=backups_root)
|
|
601
|
+
if err:
|
|
602
|
+
raise RuntimeError(err)
|
|
603
|
+
root = Path(backups_root or backups_dir())
|
|
604
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
605
|
+
stamp = time.strftime("%Y-%m-%d-%H%M%S", time.gmtime())
|
|
606
|
+
candidate = root / f"{clean}-{stamp}"
|
|
607
|
+
suffix = 2
|
|
608
|
+
while candidate.exists():
|
|
609
|
+
candidate = root / f"{clean}-{stamp}-{suffix}"
|
|
610
|
+
suffix += 1
|
|
611
|
+
candidate.mkdir(parents=True, exist_ok=False)
|
|
612
|
+
return BackupSnapshotPath(candidate, backups_root=root)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def create_backup_path(prefix: str, suffix: str = "", *, backups_root: Path | None = None) -> Path:
|
|
616
|
+
"""Reserve a backup file path under runtime/backups via the same guard."""
|
|
617
|
+
clean = str(prefix or "").strip().strip("-")
|
|
618
|
+
if not clean or any(sep in clean for sep in ("/", "\\")):
|
|
619
|
+
raise ValueError("backup prefix must be a single path segment")
|
|
620
|
+
err = backup_space_error(reason=f"create_backup_path:{clean}", backups_root=backups_root)
|
|
621
|
+
if err:
|
|
622
|
+
raise RuntimeError(err)
|
|
623
|
+
root = Path(backups_root or backups_dir())
|
|
624
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
625
|
+
stamp = time.strftime("%Y-%m-%d-%H%M%S", time.gmtime())
|
|
626
|
+
candidate = root / f"{clean}-{stamp}{suffix}"
|
|
627
|
+
index = 2
|
|
628
|
+
while candidate.exists():
|
|
629
|
+
candidate = root / f"{clean}-{stamp}-{index}{suffix}"
|
|
630
|
+
index += 1
|
|
631
|
+
return BackupSnapshotPath(candidate, backups_root=root)
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def finalize_backup_snapshot(_path: Path | str | None = None, *, backups_root: Path | None = None) -> dict:
|
|
635
|
+
"""Post-snapshot cleanup; callers invoke after writing large artifacts."""
|
|
636
|
+
root = Path(backups_root) if backups_root is not None else None
|
|
637
|
+
if root is None and _path is not None:
|
|
638
|
+
snapshot = Path(_path)
|
|
639
|
+
root = snapshot.parent
|
|
640
|
+
return run_runtime_backup_prune(backups_root=root)
|
|
641
|
+
|
|
642
|
+
|
|
323
643
|
def memory_dir() -> Path:
|
|
324
644
|
new = runtime_dir() / "memory"
|
|
325
645
|
legacy = home() / "memory"
|
|
@@ -329,11 +649,7 @@ def memory_dir() -> Path:
|
|
|
329
649
|
|
|
330
650
|
|
|
331
651
|
def cognitive_dir() -> Path:
|
|
332
|
-
|
|
333
|
-
legacy = home() / "cognitive"
|
|
334
|
-
if not new.exists() and legacy.exists():
|
|
335
|
-
return legacy
|
|
336
|
-
return new
|
|
652
|
+
return runtime_dir() / "cognitive"
|
|
337
653
|
|
|
338
654
|
|
|
339
655
|
def models_dir() -> Path:
|
package/src/plugins/update.py
CHANGED
|
@@ -155,8 +155,8 @@ def _env_int(name: str, default: int) -> int:
|
|
|
155
155
|
return default
|
|
156
156
|
|
|
157
157
|
|
|
158
|
-
BACKUP_MAX_BYTES =
|
|
159
|
-
BACKUP_MIN_FREE_BYTES =
|
|
158
|
+
BACKUP_MAX_BYTES = paths.backup_retention_cap_bytes(backups_root=BACKUP_BASE)
|
|
159
|
+
BACKUP_MIN_FREE_BYTES = paths.backup_min_free_bytes()
|
|
160
160
|
LOCAL_CONTEXT_MAX_BACKUP_BYTES = _env_int("NEXO_LOCAL_CONTEXT_MAX_BACKUP_BYTES", 2 * 1024 * 1024 * 1024)
|
|
161
161
|
|
|
162
162
|
# In packaged installs, update.py lives at <NEXO_HOME>/plugins/update.py.
|
|
@@ -199,26 +199,7 @@ def _rotate_backup_family(prefix: str, keep: int = TECHNICAL_BACKUP_KEEP) -> int
|
|
|
199
199
|
|
|
200
200
|
|
|
201
201
|
def _run_runtime_backup_prune() -> None:
|
|
202
|
-
|
|
203
|
-
if not script.is_file():
|
|
204
|
-
return
|
|
205
|
-
try:
|
|
206
|
-
subprocess.run(
|
|
207
|
-
[
|
|
208
|
-
sys.executable,
|
|
209
|
-
str(script),
|
|
210
|
-
"--root",
|
|
211
|
-
str(BACKUP_BASE),
|
|
212
|
-
"--apply",
|
|
213
|
-
"--max-bytes",
|
|
214
|
-
str(BACKUP_MAX_BYTES),
|
|
215
|
-
],
|
|
216
|
-
capture_output=True,
|
|
217
|
-
text=True,
|
|
218
|
-
timeout=120,
|
|
219
|
-
)
|
|
220
|
-
except Exception:
|
|
221
|
-
pass
|
|
202
|
+
paths.run_runtime_backup_prune(max_bytes=BACKUP_MAX_BYTES, backups_root=BACKUP_BASE)
|
|
222
203
|
|
|
223
204
|
|
|
224
205
|
def _backup_free_bytes() -> int | None:
|
|
@@ -230,14 +211,11 @@ def _backup_free_bytes() -> int | None:
|
|
|
230
211
|
|
|
231
212
|
|
|
232
213
|
def _backup_space_error() -> str | None:
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
f"({free}B < {BACKUP_MIN_FREE_BYTES}B)"
|
|
239
|
-
)
|
|
240
|
-
return None
|
|
214
|
+
return paths.backup_space_error(
|
|
215
|
+
reason="manual_update",
|
|
216
|
+
min_free_bytes=BACKUP_MIN_FREE_BYTES,
|
|
217
|
+
backups_root=BACKUP_BASE,
|
|
218
|
+
)
|
|
241
219
|
|
|
242
220
|
|
|
243
221
|
def _should_include_local_context_backup(path: Path) -> bool:
|
|
@@ -578,8 +556,7 @@ def _backup_databases() -> tuple[str, str | None]:
|
|
|
578
556
|
a backup that silently loses data returns an error instead of a green
|
|
579
557
|
"backup_dir" string that the rollback logic would then restore from.
|
|
580
558
|
"""
|
|
581
|
-
|
|
582
|
-
backup_dir = BACKUP_BASE / f"pre-update-{timestamp}"
|
|
559
|
+
backup_dir: Path | None = None
|
|
583
560
|
|
|
584
561
|
db_files = list(DATA_DIR.glob("*.db")) if DATA_DIR.is_dir() else []
|
|
585
562
|
local_context_db = paths.memory_dir() / "local-context.db"
|
|
@@ -593,13 +570,14 @@ def _backup_databases() -> tuple[str, str | None]:
|
|
|
593
570
|
db_files.append(src_db)
|
|
594
571
|
|
|
595
572
|
if not db_files:
|
|
573
|
+
backup_dir = paths.create_backup_dir("pre-update", backups_root=BACKUP_BASE)
|
|
596
574
|
return str(backup_dir), None # No DBs to backup, not an error
|
|
597
575
|
|
|
598
576
|
space_err = _backup_space_error()
|
|
599
577
|
if space_err:
|
|
600
578
|
return str(backup_dir), space_err
|
|
601
579
|
|
|
602
|
-
backup_dir.
|
|
580
|
+
backup_dir = paths.create_backup_dir("pre-update", backups_root=BACKUP_BASE)
|
|
603
581
|
|
|
604
582
|
for db_file in db_files:
|
|
605
583
|
dest = backup_dir / db_file.name
|
|
@@ -618,6 +596,7 @@ def _backup_databases() -> tuple[str, str | None]:
|
|
|
618
596
|
_rotate_backup_family("pre-update-")
|
|
619
597
|
except Exception:
|
|
620
598
|
pass
|
|
599
|
+
paths.finalize_backup_snapshot(backup_dir)
|
|
621
600
|
return str(backup_dir), None
|
|
622
601
|
|
|
623
602
|
|
|
@@ -988,8 +967,7 @@ def _sync_hooks_to_home():
|
|
|
988
967
|
|
|
989
968
|
def _backup_code_tree() -> tuple[str | None, str | None]:
|
|
990
969
|
"""Snapshot NEXO_HOME code dirs before npm update. Returns (backup_dir, error)."""
|
|
991
|
-
|
|
992
|
-
backup_dir = BACKUP_BASE / f"code-tree-{timestamp}"
|
|
970
|
+
backup_dir = paths.create_backup_dir("code-tree", backups_root=BACKUP_BASE)
|
|
993
971
|
# Directories and flat files that postinstall copies into NEXO_HOME
|
|
994
972
|
code_dirs = [
|
|
995
973
|
"bin",
|
|
@@ -1010,7 +988,6 @@ def _backup_code_tree() -> tuple[str | None, str | None]:
|
|
|
1010
988
|
]
|
|
1011
989
|
code_files_glob = ["*.py", "requirements.txt", "package.json"]
|
|
1012
990
|
try:
|
|
1013
|
-
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
1014
991
|
# Backup directories
|
|
1015
992
|
for d in code_dirs:
|
|
1016
993
|
src = NEXO_HOME / d
|
|
@@ -1031,6 +1008,7 @@ def _backup_code_tree() -> tuple[str | None, str | None]:
|
|
|
1031
1008
|
_rotate_backup_family("code-tree-")
|
|
1032
1009
|
except Exception:
|
|
1033
1010
|
pass
|
|
1011
|
+
paths.finalize_backup_snapshot(backup_dir)
|
|
1034
1012
|
return str(backup_dir), None
|
|
1035
1013
|
|
|
1036
1014
|
|
package/src/pre_answer_router.py
CHANGED
|
@@ -1098,6 +1098,27 @@ def _source_diary(request: SourceRequest) -> SourceResult:
|
|
|
1098
1098
|
|
|
1099
1099
|
|
|
1100
1100
|
def _source_transcripts(request: SourceRequest) -> SourceResult:
|
|
1101
|
+
try:
|
|
1102
|
+
from transcript_index import index_recent_transcripts, search_transcript_index
|
|
1103
|
+
|
|
1104
|
+
index_recent_transcripts(hours=72, limit=120, min_user_messages=1)
|
|
1105
|
+
indexed_rows = search_transcript_index(request.query, hours=72, limit=4)
|
|
1106
|
+
if indexed_rows:
|
|
1107
|
+
indexed_result = _rows_result(
|
|
1108
|
+
"transcript_index",
|
|
1109
|
+
indexed_rows,
|
|
1110
|
+
("source_client", "display_name", "session_id", "sanitized_summary", "modified_at"),
|
|
1111
|
+
request.max_chars,
|
|
1112
|
+
)
|
|
1113
|
+
return SourceResult(
|
|
1114
|
+
source="transcripts",
|
|
1115
|
+
rendered=indexed_result.rendered,
|
|
1116
|
+
evidence_refs=indexed_result.evidence_refs,
|
|
1117
|
+
result_count=indexed_result.result_count,
|
|
1118
|
+
)
|
|
1119
|
+
except Exception:
|
|
1120
|
+
pass
|
|
1121
|
+
|
|
1101
1122
|
from tools_transcripts import handle_transcript_search
|
|
1102
1123
|
|
|
1103
1124
|
rendered = handle_transcript_search(request.query, hours=72, limit=4)
|