nexo-brain 7.23.2 → 7.23.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +7 -1
- package/package.json +1 -1
- package/scripts/sync_release_artifacts.py +28 -0
- package/src/auto_update.py +25 -47
- package/src/automation_reconciler.py +383 -0
- package/src/automation_supervisor.py +86 -9
- package/src/backup_retention.py +70 -0
- package/src/cli.py +55 -2
- package/src/cognitive/_core.py +4 -3
- package/src/cognitive_paths.py +194 -0
- package/src/dashboard/app.py +2 -1
- package/src/db/_episodic.py +85 -7
- package/src/db/_schema.py +81 -0
- package/src/db/_skills.py +3 -3
- package/src/disk_recovery/__init__.py +11 -0
- package/src/disk_recovery/handlers/__init__.py +1 -0
- package/src/disk_recovery/handlers/common.py +37 -0
- package/src/disk_recovery/handlers/macos.py +39 -0
- package/src/disk_recovery/handlers/windows.py +49 -0
- package/src/disk_recovery/registry.py +135 -0
- package/src/doctor/providers/boot.py +115 -15
- package/src/kg_populate.py +2 -5
- package/src/paths.py +321 -5
- package/src/plugins/update.py +14 -36
- package/src/pre_answer_router.py +21 -0
- package/src/runtime_service.py +30 -3
- package/src/runtime_versioning.py +272 -10
- package/src/script_registry.py +3 -2
- package/src/scripts/backfill_task_owner.py +10 -4
- package/src/scripts/deep-sleep/apply_findings.py +2 -5
- package/src/scripts/deep-sleep/collect.py +2 -5
- package/src/scripts/nexo-cognitive-decay.py +2 -1
- package/src/scripts/nexo-daily-self-audit.py +36 -10
- package/src/scripts/nexo-followup-runner.py +1 -1
- package/src/scripts/nexo-immune.py +2 -1
- package/src/scripts/nexo-migrate.py +2 -3
- package/src/scripts/post_disk_recovery_sweep.py +75 -0
- package/src/scripts/prune_runtime_backups.py +78 -11
- package/src/server.py +13 -1
- package/src/storage_router.py +2 -3
- package/src/support_snapshot.py +25 -0
- package/src/transcript_index.py +234 -0
- package/src/transcript_utils.py +31 -8
- package/src/user_data_portability.py +2 -3
- package/tool-enforcement-map.json +15 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Nudge background sync apps after disk pressure has been relieved."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
SOURCE_ROOT = Path(__file__).resolve().parents[1]
|
|
12
|
+
if str(SOURCE_ROOT) not in sys.path:
|
|
13
|
+
sys.path.insert(0, str(SOURCE_ROOT))
|
|
14
|
+
|
|
15
|
+
import paths
|
|
16
|
+
from disk_recovery.registry import run_sweep
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _network_snapshot() -> dict:
|
|
20
|
+
# Best-effort and intentionally coarse: enough to confirm activity changed
|
|
21
|
+
# without collecting destinations or user content.
|
|
22
|
+
try:
|
|
23
|
+
import psutil # type: ignore
|
|
24
|
+
|
|
25
|
+
counters = psutil.net_io_counters()
|
|
26
|
+
return {"bytes_sent": int(counters.bytes_sent), "bytes_recv": int(counters.bytes_recv)}
|
|
27
|
+
except Exception:
|
|
28
|
+
return {"bytes_sent": None, "bytes_recv": None}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _network_delta(before: dict, after: dict) -> dict:
|
|
32
|
+
out: dict[str, int | None] = {}
|
|
33
|
+
for key in ("bytes_sent", "bytes_recv"):
|
|
34
|
+
a = after.get(key)
|
|
35
|
+
b = before.get(key)
|
|
36
|
+
out[key] = int(a) - int(b) if isinstance(a, int) and isinstance(b, int) else None
|
|
37
|
+
return out
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def append_log(payload: dict) -> None:
|
|
41
|
+
log_path = paths.operations_dir() / "post-disk-recovery-sweep.jsonl"
|
|
42
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
43
|
+
with log_path.open("a", encoding="utf-8") as fh:
|
|
44
|
+
fh.write(json.dumps(payload, sort_keys=True) + "\n")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def main(argv: list[str] | None = None) -> int:
|
|
48
|
+
parser = argparse.ArgumentParser(description="Run post-disk-recovery sync sweep")
|
|
49
|
+
parser.add_argument("--platform", help="Override platform for tests: darwin/windows")
|
|
50
|
+
parser.add_argument("--dry-run", action="store_true", help="Plan actions without running commands")
|
|
51
|
+
parser.add_argument("--json", action="store_true", help="Print JSON report")
|
|
52
|
+
parser.add_argument("--network-window-seconds", type=float, default=0, help="Seconds between network snapshots")
|
|
53
|
+
parser.add_argument("--reason", default="manual", help="Reason written to the operations log")
|
|
54
|
+
args = parser.parse_args(argv)
|
|
55
|
+
|
|
56
|
+
before = _network_snapshot()
|
|
57
|
+
report = run_sweep(platform=args.platform, dry_run=args.dry_run)
|
|
58
|
+
if args.network_window_seconds > 0:
|
|
59
|
+
time.sleep(args.network_window_seconds)
|
|
60
|
+
after = _network_snapshot()
|
|
61
|
+
payload = {
|
|
62
|
+
"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
63
|
+
"reason": args.reason,
|
|
64
|
+
"network_window_seconds": args.network_window_seconds,
|
|
65
|
+
"network_delta": _network_delta(before, after),
|
|
66
|
+
**report,
|
|
67
|
+
}
|
|
68
|
+
append_log(payload)
|
|
69
|
+
if args.json:
|
|
70
|
+
print(json.dumps(payload, indent=2, sort_keys=True))
|
|
71
|
+
return 0 if payload.get("ok") else 1
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
if __name__ == "__main__":
|
|
75
|
+
raise SystemExit(main())
|
|
@@ -115,6 +115,8 @@ TEMPORARY_RE = re.compile(r"(^|.*[.])tmp([.-].*)?$|.*\.tmp\..*|.*-journal$|.*\.d
|
|
|
115
115
|
# Big ad-hoc DB files at the root — rare, include for reporting but never auto-prune.
|
|
116
116
|
ROOT_DB_RE = re.compile(r"^(pre-obs-clean|pre-sleep-wrapper-apply|pre-.*)-\d{4}-\d{2}-\d{2}-\d{4}\.db$")
|
|
117
117
|
DEFAULT_MAX_BYTES = 50 * 1024 * 1024 * 1024
|
|
118
|
+
MIN_ADAPTIVE_MAX_BYTES = 10 * 1024 * 1024 * 1024
|
|
119
|
+
MAX_ADAPTIVE_MAX_BYTES = 50 * 1024 * 1024 * 1024
|
|
118
120
|
|
|
119
121
|
# Timestamp patterns embedded in directory names.
|
|
120
122
|
TS_PATTERNS = (
|
|
@@ -214,6 +216,20 @@ def parse_size_bytes(value: str | int | None, *, default: int = DEFAULT_MAX_BYTE
|
|
|
214
216
|
return default
|
|
215
217
|
|
|
216
218
|
|
|
219
|
+
def effective_max_bytes(backups_root: Path, raw_max_bytes: int) -> int:
|
|
220
|
+
"""Apply the adaptive default cap without blocking emergency lower caps."""
|
|
221
|
+
if raw_max_bytes < MIN_ADAPTIVE_MAX_BYTES:
|
|
222
|
+
return raw_max_bytes
|
|
223
|
+
probe = backups_root if backups_root.exists() else backups_root.parent
|
|
224
|
+
try:
|
|
225
|
+
total = int(shutil.disk_usage(str(probe)).total)
|
|
226
|
+
adaptive = int(total * 0.05)
|
|
227
|
+
except Exception:
|
|
228
|
+
adaptive = DEFAULT_MAX_BYTES
|
|
229
|
+
adaptive = max(MIN_ADAPTIVE_MAX_BYTES, min(MAX_ADAPTIVE_MAX_BYTES, adaptive))
|
|
230
|
+
return min(raw_max_bytes, adaptive)
|
|
231
|
+
|
|
232
|
+
|
|
217
233
|
def gather_entries(backups_root: Path) -> list[dict]:
|
|
218
234
|
items: list[dict] = []
|
|
219
235
|
for entry in backups_root.iterdir():
|
|
@@ -303,7 +319,7 @@ def plan_prunes(
|
|
|
303
319
|
else:
|
|
304
320
|
to_keep.append(it)
|
|
305
321
|
|
|
306
|
-
for klass, keep_count in (("LOCAL_CONTEXT_DB", local_context_keep),
|
|
322
|
+
for klass, keep_count in (("LOCAL_CONTEXT_DB", local_context_keep),):
|
|
307
323
|
group = [it for it in items if it["class"] == klass]
|
|
308
324
|
group.sort(key=lambda x: (x["ts"] or datetime.min.replace(tzinfo=timezone.utc)), reverse=True)
|
|
309
325
|
for it in group[:max(0, keep_count)]:
|
|
@@ -321,12 +337,10 @@ def plan_prunes(
|
|
|
321
337
|
for it in items:
|
|
322
338
|
by_budget_family.setdefault((it["class"], it["family"]), []).append(it)
|
|
323
339
|
for (klass, _family), group in by_budget_family.items():
|
|
324
|
-
if klass not in {"TECHNICAL", "LOCAL_CONTEXT_DB", "
|
|
340
|
+
if klass not in {"TECHNICAL", "LOCAL_CONTEXT_DB", "TEMPORARY"}:
|
|
325
341
|
continue
|
|
326
342
|
min_keep = 0
|
|
327
|
-
if klass == "
|
|
328
|
-
min_keep = max(1, hourly_keep)
|
|
329
|
-
elif klass == "LOCAL_CONTEXT_DB":
|
|
343
|
+
if klass == "LOCAL_CONTEXT_DB":
|
|
330
344
|
min_keep = max(0, local_context_keep)
|
|
331
345
|
group.sort(key=lambda x: (x["ts"] or datetime.min.replace(tzinfo=timezone.utc)), reverse=True)
|
|
332
346
|
for it in group[:min_keep]:
|
|
@@ -336,7 +350,7 @@ def plan_prunes(
|
|
|
336
350
|
it for it in items
|
|
337
351
|
if id(it) not in delete_ids
|
|
338
352
|
and id(it) not in protected_keep_ids
|
|
339
|
-
and it["class"] in {"TECHNICAL", "LOCAL_CONTEXT_DB", "
|
|
353
|
+
and it["class"] in {"TECHNICAL", "LOCAL_CONTEXT_DB", "TEMPORARY"}
|
|
340
354
|
]
|
|
341
355
|
budget_candidates.sort(
|
|
342
356
|
key=lambda x: (
|
|
@@ -358,6 +372,28 @@ def plan_prunes(
|
|
|
358
372
|
return to_delete, to_keep
|
|
359
373
|
|
|
360
374
|
|
|
375
|
+
def restore_point_guard(items: list[dict], to_delete: list[dict]) -> tuple[bool, list[str], dict]:
|
|
376
|
+
"""Validate that apply mode never removes protected restore classes."""
|
|
377
|
+
delete_ids = {id(item) for item in to_delete}
|
|
378
|
+
protected_classes = {"BUSINESS", "HOURLY_DB", "ROOT_DB", "UNKNOWN"}
|
|
379
|
+
protected_names = set(PROTECTED_NAMES)
|
|
380
|
+
violations: list[str] = []
|
|
381
|
+
for item in items:
|
|
382
|
+
if id(item) not in delete_ids:
|
|
383
|
+
continue
|
|
384
|
+
if item["class"] in protected_classes or item["name"] in protected_names:
|
|
385
|
+
violations.append(f"{item['name']} ({item['class']})")
|
|
386
|
+
hourly_count = sum(1 for item in items if item["class"] == "HOURLY_DB")
|
|
387
|
+
weekly_present = any(item["name"] == "weekly" for item in items)
|
|
388
|
+
business_count = sum(1 for item in items if item["class"] == "BUSINESS")
|
|
389
|
+
return not violations, violations, {
|
|
390
|
+
"hourly_db_present": hourly_count,
|
|
391
|
+
"weekly_present": weekly_present,
|
|
392
|
+
"business_protected": business_count,
|
|
393
|
+
"protected_delete_violations": violations,
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
|
|
361
397
|
def run(args: argparse.Namespace) -> int:
|
|
362
398
|
backups_root = Path(args.root or (default_nexo_home() / "runtime" / "backups"))
|
|
363
399
|
if not backups_root.is_dir():
|
|
@@ -371,7 +407,7 @@ def run(args: argparse.Namespace) -> int:
|
|
|
371
407
|
local_context_items = [i for i in items if i["class"] == "LOCAL_CONTEXT_DB"]
|
|
372
408
|
temporary_items = [i for i in items if i["class"] == "TEMPORARY"]
|
|
373
409
|
unknown_items = [i for i in items if i["class"] == "UNKNOWN"]
|
|
374
|
-
max_bytes = parse_size_bytes(args.max_bytes)
|
|
410
|
+
max_bytes = effective_max_bytes(backups_root, parse_size_bytes(args.max_bytes))
|
|
375
411
|
|
|
376
412
|
to_delete, to_keep = plan_prunes(
|
|
377
413
|
items,
|
|
@@ -384,6 +420,16 @@ def run(args: argparse.Namespace) -> int:
|
|
|
384
420
|
hourly_keep=max(0, args.hourly_keep),
|
|
385
421
|
)
|
|
386
422
|
|
|
423
|
+
if args.delete_all_technical:
|
|
424
|
+
delete_ids = {id(item) for item in to_delete}
|
|
425
|
+
for item in items:
|
|
426
|
+
if item["class"] in {"TECHNICAL", "TEMPORARY"} and id(item) not in delete_ids:
|
|
427
|
+
to_delete.append(item)
|
|
428
|
+
delete_ids.add(id(item))
|
|
429
|
+
to_keep = [item for item in items if id(item) not in delete_ids]
|
|
430
|
+
|
|
431
|
+
restore_guard_ok, restore_guard_violations, restore_guard = restore_point_guard(items, to_delete)
|
|
432
|
+
|
|
387
433
|
total_all = sum(i["size"] for i in items)
|
|
388
434
|
total_del = sum(i["size"] for i in to_delete)
|
|
389
435
|
|
|
@@ -396,9 +442,11 @@ def run(args: argparse.Namespace) -> int:
|
|
|
396
442
|
"only": args.only,
|
|
397
443
|
"max_bytes": max_bytes,
|
|
398
444
|
"max_human": human_size(max_bytes),
|
|
445
|
+
"delete_all_technical": args.delete_all_technical,
|
|
399
446
|
"tmp_ttl_minutes": args.tmp_ttl_minutes,
|
|
400
447
|
"local_context_keep": args.local_context_keep,
|
|
401
448
|
"hourly_keep": args.hourly_keep,
|
|
449
|
+
"restore_point_guard": restore_guard,
|
|
402
450
|
},
|
|
403
451
|
"totals": {
|
|
404
452
|
"all_bytes": total_all,
|
|
@@ -429,9 +477,7 @@ def run(args: argparse.Namespace) -> int:
|
|
|
429
477
|
"unknown": [i["name"] for i in unknown_items],
|
|
430
478
|
}
|
|
431
479
|
|
|
432
|
-
if args.json:
|
|
433
|
-
print(json.dumps(report, indent=2))
|
|
434
|
-
else:
|
|
480
|
+
if not args.json:
|
|
435
481
|
print(f"NEXO backup prune — root: {backups_root}")
|
|
436
482
|
print(f" total on disk: {human_size(total_all)} ({len(items)} entries)")
|
|
437
483
|
print(f" technical: {len(tech_items)}")
|
|
@@ -457,10 +503,21 @@ def run(args: argparse.Namespace) -> int:
|
|
|
457
503
|
print(f" ? {it['name']}")
|
|
458
504
|
|
|
459
505
|
if not args.apply:
|
|
506
|
+
if args.json:
|
|
507
|
+
print(json.dumps(report, indent=2))
|
|
508
|
+
return 0
|
|
460
509
|
if not args.json:
|
|
461
510
|
print("\n(dry-run: pass --apply to delete)")
|
|
462
511
|
return 0
|
|
463
512
|
|
|
513
|
+
if not restore_guard_ok:
|
|
514
|
+
print(
|
|
515
|
+
"ERROR: refusing to prune protected restore artifacts: "
|
|
516
|
+
+ ", ".join(restore_guard_violations),
|
|
517
|
+
file=sys.stderr,
|
|
518
|
+
)
|
|
519
|
+
return 1
|
|
520
|
+
|
|
464
521
|
deleted = 0
|
|
465
522
|
failed = 0
|
|
466
523
|
freed = 0
|
|
@@ -476,7 +533,16 @@ def run(args: argparse.Namespace) -> int:
|
|
|
476
533
|
except OSError as e:
|
|
477
534
|
failed += 1
|
|
478
535
|
print(f"WARN: failed to delete {p}: {e}", file=sys.stderr)
|
|
479
|
-
|
|
536
|
+
report["apply"] = {
|
|
537
|
+
"deleted": deleted,
|
|
538
|
+
"freed_bytes": freed,
|
|
539
|
+
"freed_human": human_size(freed),
|
|
540
|
+
"failures": failed,
|
|
541
|
+
}
|
|
542
|
+
if args.json:
|
|
543
|
+
print(json.dumps(report, indent=2))
|
|
544
|
+
else:
|
|
545
|
+
print(f"\nDELETED {deleted} entries, freed {human_size(freed)}, failures: {failed}")
|
|
480
546
|
return 0 if failed == 0 else 1
|
|
481
547
|
|
|
482
548
|
|
|
@@ -489,6 +555,7 @@ def main() -> int:
|
|
|
489
555
|
ap.add_argument("--window-days", type=int, default=90, help="month-spaced retention window (default: 90)")
|
|
490
556
|
ap.add_argument("--only", help="restrict to one technical family (e.g. 'pre-backfill-owner')")
|
|
491
557
|
ap.add_argument("--max-bytes", default=os.environ.get("NEXO_BACKUP_MAX_BYTES", str(DEFAULT_MAX_BYTES)), help="global product-generated backup hard cap, bytes or K/M/G/T (default: 50G)")
|
|
558
|
+
ap.add_argument("--delete-all-technical", action="store_true", help="emergency mode: delete all technical rollback snapshots; protected business/weekly/hourly DB backups remain untouched")
|
|
492
559
|
ap.add_argument("--tmp-ttl-minutes", type=int, default=int(os.environ.get("NEXO_BACKUP_TMP_TTL_MINUTES", "30")), help="delete orphan temporary backup files older than this (default: 30)")
|
|
493
560
|
ap.add_argument("--local-context-keep", type=int, default=int(os.environ.get("NEXO_LOCAL_CONTEXT_BACKUP_KEEP_LAST", "1")), help="local-context backup files to keep under the global cap (default: 1)")
|
|
494
561
|
ap.add_argument("--hourly-keep", type=int, default=int(os.environ.get("NEXO_BACKUP_KEEP_LAST", "3")), help="hourly nexo DB backups to keep under the global cap (default: 3)")
|
package/src/server.py
CHANGED
|
@@ -1429,7 +1429,7 @@ def nexo_saved_not_used_audit(markdown: bool = False) -> str:
|
|
|
1429
1429
|
|
|
1430
1430
|
@mcp.tool
|
|
1431
1431
|
def nexo_automation_supervisor(markdown: bool = False) -> str:
|
|
1432
|
-
"""Read-only supervisor report for automations, cron runs
|
|
1432
|
+
"""Read-only supervisor report for automations, cron runs, cron spool and Evolution policy."""
|
|
1433
1433
|
from automation_supervisor import audit_automation, format_markdown
|
|
1434
1434
|
|
|
1435
1435
|
report = audit_automation()
|
|
@@ -1438,6 +1438,18 @@ def nexo_automation_supervisor(markdown: bool = False) -> str:
|
|
|
1438
1438
|
return json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)
|
|
1439
1439
|
|
|
1440
1440
|
|
|
1441
|
+
@mcp.tool
|
|
1442
|
+
def nexo_automation_reconcile(apply: bool = False) -> str:
|
|
1443
|
+
"""Build or apply the safe automation reconciliation plan."""
|
|
1444
|
+
from automation_reconciler import apply_reconciliation_plan, build_reconciliation_plan
|
|
1445
|
+
|
|
1446
|
+
plan = build_reconciliation_plan()
|
|
1447
|
+
if apply:
|
|
1448
|
+
result = apply_reconciliation_plan(plan)
|
|
1449
|
+
return json.dumps({"ok": result.get("ok", False), "plan": plan, "apply": result}, ensure_ascii=False, indent=2)
|
|
1450
|
+
return json.dumps(plan, ensure_ascii=False, indent=2)
|
|
1451
|
+
|
|
1452
|
+
|
|
1441
1453
|
@mcp.tool
|
|
1442
1454
|
def nexo_system_catalog(section: str = "", query: str = "", limit: int = 20) -> str:
|
|
1443
1455
|
"""Read NEXO's live system catalog built from core tools, plugins, skills, scripts, crons, projects, and artifacts."""
|
package/src/storage_router.py
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import os
|
|
4
4
|
|
|
5
5
|
import paths
|
|
6
|
+
from cognitive_paths import resolve_cognitive_db
|
|
6
7
|
|
|
7
8
|
NEXO_HOME = os.environ.get("NEXO_HOME", os.path.expanduser("~/.nexo"))
|
|
8
9
|
|
|
@@ -20,9 +21,7 @@ class StorageRouter:
|
|
|
20
21
|
|
|
21
22
|
def cognitive_db_path(self) -> str:
|
|
22
23
|
if self.tenant_id == "default":
|
|
23
|
-
|
|
24
|
-
cognitive_dir.mkdir(parents=True, exist_ok=True)
|
|
25
|
-
return str(cognitive_dir / "cognitive.db")
|
|
24
|
+
return str(resolve_cognitive_db(for_write=True))
|
|
26
25
|
return os.path.join(NEXO_HOME, "tenants", self.tenant_id, "cognitive.db")
|
|
27
26
|
|
|
28
27
|
|
package/src/support_snapshot.py
CHANGED
|
@@ -111,6 +111,30 @@ def _recent_logs(lines: int = 80) -> dict[str, Any]:
|
|
|
111
111
|
}
|
|
112
112
|
|
|
113
113
|
|
|
114
|
+
def _backup_retention_status(lines: int = 20) -> dict[str, Any]:
|
|
115
|
+
root = paths.backups_dir()
|
|
116
|
+
status: dict[str, Any] = {
|
|
117
|
+
"root": str(root),
|
|
118
|
+
"exists": root.exists(),
|
|
119
|
+
"cap_bytes": paths.backup_retention_cap_bytes(backups_root=root),
|
|
120
|
+
"min_free_bytes": paths.backup_min_free_bytes(),
|
|
121
|
+
"free_bytes": paths.backup_free_bytes(backups_root=root),
|
|
122
|
+
"recent_events": [],
|
|
123
|
+
}
|
|
124
|
+
event_log = paths.operations_dir() / "backup-retention-events.jsonl"
|
|
125
|
+
if event_log.is_file():
|
|
126
|
+
try:
|
|
127
|
+
raw = event_log.read_text(encoding="utf-8", errors="ignore").splitlines()[-lines:]
|
|
128
|
+
for line in raw:
|
|
129
|
+
try:
|
|
130
|
+
status["recent_events"].append(json.loads(line))
|
|
131
|
+
except Exception:
|
|
132
|
+
continue
|
|
133
|
+
except Exception as exc:
|
|
134
|
+
status["recent_events"] = [{"error": str(exc)}]
|
|
135
|
+
return status
|
|
136
|
+
|
|
137
|
+
|
|
114
138
|
def collect_snapshot(*, log_lines: int = 80, include_doctor: bool = False) -> dict[str, Any]:
|
|
115
139
|
system = platform.system()
|
|
116
140
|
release = platform.release()
|
|
@@ -130,6 +154,7 @@ def collect_snapshot(*, log_lines: int = 80, include_doctor: bool = False) -> di
|
|
|
130
154
|
},
|
|
131
155
|
"paths": _path_status(),
|
|
132
156
|
"health": collect_health(),
|
|
157
|
+
"backup_retention": _backup_retention_status(),
|
|
133
158
|
"logs": _recent_logs(log_lines),
|
|
134
159
|
}
|
|
135
160
|
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
"""Structured transcript metadata index for pre-answer continuity.
|
|
4
|
+
|
|
5
|
+
This index stores compact, redacted metadata and short snippets only. Raw JSONL
|
|
6
|
+
transcripts remain a last-resort fallback and are not copied into the database.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
from datetime import datetime, timedelta
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from db import get_db
|
|
16
|
+
from transcript_utils import (
|
|
17
|
+
DEFAULT_TRANSCRIPT_HOURS,
|
|
18
|
+
_score_text_match,
|
|
19
|
+
_tokenize,
|
|
20
|
+
_truncate,
|
|
21
|
+
list_recent_transcripts,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _ensure_transcript_index_table() -> None:
|
|
26
|
+
conn = get_db()
|
|
27
|
+
conn.execute("""
|
|
28
|
+
CREATE TABLE IF NOT EXISTS transcript_index (
|
|
29
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
30
|
+
source_client TEXT NOT NULL,
|
|
31
|
+
conversation_id TEXT DEFAULT '',
|
|
32
|
+
session_id TEXT DEFAULT '',
|
|
33
|
+
message_count INTEGER DEFAULT 0,
|
|
34
|
+
user_message_count INTEGER DEFAULT 0,
|
|
35
|
+
first_user_at TEXT DEFAULT '',
|
|
36
|
+
last_user_at TEXT DEFAULT '',
|
|
37
|
+
path_ref TEXT NOT NULL,
|
|
38
|
+
display_name TEXT DEFAULT '',
|
|
39
|
+
indexed_at TEXT DEFAULT (datetime('now')),
|
|
40
|
+
modified_at TEXT DEFAULT '',
|
|
41
|
+
content_hash TEXT NOT NULL,
|
|
42
|
+
sanitized_summary TEXT DEFAULT '',
|
|
43
|
+
metadata_json TEXT DEFAULT '{}',
|
|
44
|
+
UNIQUE(source_client, path_ref)
|
|
45
|
+
)
|
|
46
|
+
""")
|
|
47
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_transcript_index_client_modified ON transcript_index(source_client, modified_at)")
|
|
48
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_transcript_index_session ON transcript_index(session_id)")
|
|
49
|
+
conn.execute("CREATE INDEX IF NOT EXISTS idx_transcript_index_conversation ON transcript_index(conversation_id)")
|
|
50
|
+
conn.commit()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _session_identity(session: dict[str, Any]) -> tuple[str, str, str, str]:
|
|
54
|
+
source_client = str(session.get("client") or "")
|
|
55
|
+
session_id = str(session.get("session_uid") or session.get("session_file") or session.get("display_name") or "")
|
|
56
|
+
conversation_id = str(session.get("conversation_id") or session.get("session_uid") or session_id)
|
|
57
|
+
path_ref = str(session.get("session_path") or session.get("path") or "")
|
|
58
|
+
return source_client, session_id, conversation_id, path_ref
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _session_modified_at(session: dict[str, Any]) -> str:
|
|
62
|
+
modified = str(session.get("modified") or "").strip()
|
|
63
|
+
if modified:
|
|
64
|
+
return modified
|
|
65
|
+
path_ref = str(session.get("session_path") or "").strip()
|
|
66
|
+
if not path_ref:
|
|
67
|
+
return ""
|
|
68
|
+
try:
|
|
69
|
+
return datetime.fromtimestamp(Path(path_ref).stat().st_mtime).isoformat()
|
|
70
|
+
except OSError:
|
|
71
|
+
return ""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _content_hash(session: dict[str, Any]) -> str:
|
|
75
|
+
digest = hashlib.sha256()
|
|
76
|
+
digest.update(str(session.get("client") or "").encode())
|
|
77
|
+
digest.update(str(session.get("session_file") or "").encode())
|
|
78
|
+
for message in session.get("messages") or []:
|
|
79
|
+
digest.update(str(message.get("role") or "").encode())
|
|
80
|
+
digest.update(str(message.get("index") or "").encode())
|
|
81
|
+
digest.update(str(message.get("text") or "").encode())
|
|
82
|
+
return digest.hexdigest()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _sanitized_summary(session: dict[str, Any], *, limit: int = 900) -> str:
|
|
86
|
+
user_snippets: list[str] = []
|
|
87
|
+
assistant_snippets: list[str] = []
|
|
88
|
+
for message in session.get("messages") or []:
|
|
89
|
+
role = str(message.get("role") or "")
|
|
90
|
+
text = _truncate(str(message.get("text") or ""), 180)
|
|
91
|
+
if not text:
|
|
92
|
+
continue
|
|
93
|
+
if role == "user" and len(user_snippets) < 3:
|
|
94
|
+
user_snippets.append(text)
|
|
95
|
+
elif role == "assistant" and len(assistant_snippets) < 2:
|
|
96
|
+
assistant_snippets.append(text)
|
|
97
|
+
parts = []
|
|
98
|
+
if user_snippets:
|
|
99
|
+
parts.append("user: " + " | ".join(user_snippets))
|
|
100
|
+
if assistant_snippets:
|
|
101
|
+
parts.append("assistant: " + " | ".join(assistant_snippets))
|
|
102
|
+
summary = " ".join(parts)
|
|
103
|
+
return _truncate(summary, limit)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def index_transcript_session(session: dict[str, Any]) -> dict[str, Any]:
|
|
107
|
+
"""Upsert a single transcript metadata row and return it."""
|
|
108
|
+
_ensure_transcript_index_table()
|
|
109
|
+
source_client, session_id, conversation_id, path_ref = _session_identity(session)
|
|
110
|
+
if not source_client or not path_ref:
|
|
111
|
+
raise ValueError("transcript session requires client and session_path")
|
|
112
|
+
|
|
113
|
+
metadata = {
|
|
114
|
+
"source": session.get("source", ""),
|
|
115
|
+
"cwd": session.get("cwd", ""),
|
|
116
|
+
"originator": session.get("originator", ""),
|
|
117
|
+
"tool_use_count": session.get("tool_use_count", 0),
|
|
118
|
+
}
|
|
119
|
+
conn = get_db()
|
|
120
|
+
conn.execute(
|
|
121
|
+
"""
|
|
122
|
+
INSERT INTO transcript_index (
|
|
123
|
+
source_client, conversation_id, session_id, message_count,
|
|
124
|
+
user_message_count, first_user_at, last_user_at, path_ref,
|
|
125
|
+
display_name, indexed_at, modified_at, content_hash,
|
|
126
|
+
sanitized_summary, metadata_json
|
|
127
|
+
)
|
|
128
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'), ?, ?, ?, ?)
|
|
129
|
+
ON CONFLICT(source_client, path_ref) DO UPDATE SET
|
|
130
|
+
conversation_id = excluded.conversation_id,
|
|
131
|
+
session_id = excluded.session_id,
|
|
132
|
+
message_count = excluded.message_count,
|
|
133
|
+
user_message_count = excluded.user_message_count,
|
|
134
|
+
first_user_at = excluded.first_user_at,
|
|
135
|
+
last_user_at = excluded.last_user_at,
|
|
136
|
+
display_name = excluded.display_name,
|
|
137
|
+
indexed_at = datetime('now'),
|
|
138
|
+
modified_at = excluded.modified_at,
|
|
139
|
+
content_hash = excluded.content_hash,
|
|
140
|
+
sanitized_summary = excluded.sanitized_summary,
|
|
141
|
+
metadata_json = excluded.metadata_json
|
|
142
|
+
""",
|
|
143
|
+
(
|
|
144
|
+
source_client,
|
|
145
|
+
conversation_id,
|
|
146
|
+
session_id,
|
|
147
|
+
int(session.get("message_count") or len(session.get("messages") or [])),
|
|
148
|
+
int(session.get("user_message_count") or 0),
|
|
149
|
+
str(session.get("first_user_at") or ""),
|
|
150
|
+
str(session.get("last_user_at") or ""),
|
|
151
|
+
path_ref,
|
|
152
|
+
str(session.get("display_name") or ""),
|
|
153
|
+
_session_modified_at(session),
|
|
154
|
+
_content_hash(session),
|
|
155
|
+
_sanitized_summary(session),
|
|
156
|
+
json.dumps(metadata, ensure_ascii=False, sort_keys=True),
|
|
157
|
+
),
|
|
158
|
+
)
|
|
159
|
+
conn.commit()
|
|
160
|
+
row = conn.execute(
|
|
161
|
+
"SELECT * FROM transcript_index WHERE source_client = ? AND path_ref = ?",
|
|
162
|
+
(source_client, path_ref),
|
|
163
|
+
).fetchone()
|
|
164
|
+
return dict(row) if row else {}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def index_recent_transcripts(
|
|
168
|
+
*,
|
|
169
|
+
hours: int = DEFAULT_TRANSCRIPT_HOURS,
|
|
170
|
+
client: str = "",
|
|
171
|
+
limit: int = 200,
|
|
172
|
+
min_user_messages: int = 1,
|
|
173
|
+
) -> list[dict[str, Any]]:
|
|
174
|
+
rows = list_recent_transcripts(
|
|
175
|
+
hours=hours,
|
|
176
|
+
client=client,
|
|
177
|
+
limit=limit,
|
|
178
|
+
min_user_messages=min_user_messages,
|
|
179
|
+
)
|
|
180
|
+
indexed = []
|
|
181
|
+
for session in rows:
|
|
182
|
+
try:
|
|
183
|
+
indexed.append(index_transcript_session(session))
|
|
184
|
+
except Exception:
|
|
185
|
+
continue
|
|
186
|
+
return indexed
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def search_transcript_index(
|
|
190
|
+
query: str = "",
|
|
191
|
+
*,
|
|
192
|
+
hours: int = 72,
|
|
193
|
+
client: str = "",
|
|
194
|
+
limit: int = 10,
|
|
195
|
+
) -> list[dict[str, Any]]:
|
|
196
|
+
_ensure_transcript_index_table()
|
|
197
|
+
conn = get_db()
|
|
198
|
+
params: list[Any] = []
|
|
199
|
+
where = "1=1"
|
|
200
|
+
if client:
|
|
201
|
+
where += " AND source_client = ?"
|
|
202
|
+
params.append(client)
|
|
203
|
+
rows = [dict(row) for row in conn.execute(
|
|
204
|
+
f"SELECT * FROM transcript_index WHERE {where} ORDER BY modified_at DESC LIMIT 500",
|
|
205
|
+
tuple(params),
|
|
206
|
+
).fetchall()]
|
|
207
|
+
|
|
208
|
+
cutoff = datetime.now() - timedelta(hours=max(1, int(hours or 72)))
|
|
209
|
+
query_tokens = _tokenize(query)
|
|
210
|
+
matches = []
|
|
211
|
+
for row in rows:
|
|
212
|
+
modified = str(row.get("modified_at") or "")
|
|
213
|
+
if modified:
|
|
214
|
+
try:
|
|
215
|
+
if datetime.fromisoformat(modified) < cutoff:
|
|
216
|
+
continue
|
|
217
|
+
except Exception:
|
|
218
|
+
pass
|
|
219
|
+
if not query_tokens:
|
|
220
|
+
row["_score"] = 0.0
|
|
221
|
+
matches.append(row)
|
|
222
|
+
continue
|
|
223
|
+
haystack = " ".join(
|
|
224
|
+
str(row.get(field) or "")
|
|
225
|
+
for field in ("sanitized_summary", "display_name", "session_id", "conversation_id", "metadata_json")
|
|
226
|
+
)
|
|
227
|
+
score = _score_text_match(query_tokens, haystack)
|
|
228
|
+
if score <= 0:
|
|
229
|
+
continue
|
|
230
|
+
row["_score"] = round(score, 4)
|
|
231
|
+
matches.append(row)
|
|
232
|
+
|
|
233
|
+
matches.sort(key=lambda row: (float(row.get("_score") or 0), str(row.get("modified_at") or "")), reverse=True)
|
|
234
|
+
return matches[: max(1, int(limit or 10))]
|
package/src/transcript_utils.py
CHANGED
|
@@ -118,7 +118,15 @@ def find_codex_session_files() -> list[Path]:
|
|
|
118
118
|
return files
|
|
119
119
|
|
|
120
120
|
|
|
121
|
-
def
|
|
121
|
+
def _min_user_messages(value: int | str | None = MIN_USER_MESSAGES) -> int:
|
|
122
|
+
try:
|
|
123
|
+
parsed = int(value if value is not None else MIN_USER_MESSAGES)
|
|
124
|
+
except Exception:
|
|
125
|
+
parsed = MIN_USER_MESSAGES
|
|
126
|
+
return max(1, parsed)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def extract_claude_session(jsonl_path: Path, *, min_user_messages: int = MIN_USER_MESSAGES) -> dict | None:
|
|
122
130
|
messages = []
|
|
123
131
|
tool_uses = []
|
|
124
132
|
user_msg_count = 0
|
|
@@ -183,7 +191,7 @@ def extract_claude_session(jsonl_path: Path) -> dict | None:
|
|
|
183
191
|
except Exception:
|
|
184
192
|
return None
|
|
185
193
|
|
|
186
|
-
if user_msg_count <
|
|
194
|
+
if user_msg_count < _min_user_messages(min_user_messages):
|
|
187
195
|
return None
|
|
188
196
|
|
|
189
197
|
return {
|
|
@@ -200,7 +208,7 @@ def extract_claude_session(jsonl_path: Path) -> dict | None:
|
|
|
200
208
|
}
|
|
201
209
|
|
|
202
210
|
|
|
203
|
-
def extract_codex_session(jsonl_path: Path) -> dict | None:
|
|
211
|
+
def extract_codex_session(jsonl_path: Path, *, min_user_messages: int = MIN_USER_MESSAGES) -> dict | None:
|
|
204
212
|
messages = []
|
|
205
213
|
tool_uses = []
|
|
206
214
|
user_msg_count = 0
|
|
@@ -266,7 +274,7 @@ def extract_codex_session(jsonl_path: Path) -> dict | None:
|
|
|
266
274
|
except Exception:
|
|
267
275
|
return None
|
|
268
276
|
|
|
269
|
-
if user_msg_count <
|
|
277
|
+
if user_msg_count < _min_user_messages(min_user_messages):
|
|
270
278
|
return None
|
|
271
279
|
|
|
272
280
|
return {
|
|
@@ -286,7 +294,12 @@ def extract_codex_session(jsonl_path: Path) -> dict | None:
|
|
|
286
294
|
}
|
|
287
295
|
|
|
288
296
|
|
|
289
|
-
def collect_transcripts_since(
|
|
297
|
+
def collect_transcripts_since(
|
|
298
|
+
since_iso: str,
|
|
299
|
+
until_iso: str = "",
|
|
300
|
+
*,
|
|
301
|
+
min_user_messages: int = MIN_USER_MESSAGES,
|
|
302
|
+
) -> list[dict]:
|
|
290
303
|
since_dt = datetime.fromisoformat(since_iso)
|
|
291
304
|
until_dt = datetime.fromisoformat(until_iso) if until_iso else datetime.now()
|
|
292
305
|
sessions = []
|
|
@@ -302,7 +315,11 @@ def collect_transcripts_since(since_iso: str, until_iso: str = "") -> list[dict]
|
|
|
302
315
|
continue
|
|
303
316
|
if not (since_dt < mtime <= until_dt):
|
|
304
317
|
continue
|
|
305
|
-
session =
|
|
318
|
+
session = (
|
|
319
|
+
extract_codex_session(session_file, min_user_messages=min_user_messages)
|
|
320
|
+
if client == "codex"
|
|
321
|
+
else extract_claude_session(session_file, min_user_messages=min_user_messages)
|
|
322
|
+
)
|
|
306
323
|
if session:
|
|
307
324
|
session["modified"] = mtime.isoformat()
|
|
308
325
|
sessions.append(session)
|
|
@@ -310,10 +327,16 @@ def collect_transcripts_since(since_iso: str, until_iso: str = "") -> list[dict]
|
|
|
310
327
|
return sessions
|
|
311
328
|
|
|
312
329
|
|
|
313
|
-
def list_recent_transcripts(
|
|
330
|
+
def list_recent_transcripts(
|
|
331
|
+
hours: int = DEFAULT_TRANSCRIPT_HOURS,
|
|
332
|
+
client: str = "",
|
|
333
|
+
limit: int = 10,
|
|
334
|
+
*,
|
|
335
|
+
min_user_messages: int = MIN_USER_MESSAGES,
|
|
336
|
+
) -> list[dict]:
|
|
314
337
|
window = clamp_transcript_hours(hours)
|
|
315
338
|
since = datetime.now() - timedelta(hours=window)
|
|
316
|
-
sessions = collect_transcripts_since(since.isoformat())
|
|
339
|
+
sessions = collect_transcripts_since(since.isoformat(), min_user_messages=min_user_messages)
|
|
317
340
|
filtered = []
|
|
318
341
|
for item in sessions:
|
|
319
342
|
if client and item.get("client") != client:
|