@agentikos/omega-os 0.1.0 → 0.19.5

package/omega/Agentik_Engine/omega_engine/cleanup.py

@@ -0,0 +1,673 @@
+"""`omega cleanup` — disk cleanup adapted to the OmegaOS layout.
+
+Why this exists (and why we don't just call the AISB cleanup script)
+--------------------------------------------------------------------
+
+The AISB cleanup script (``~/.claude/lib/cleanup.sh``) targets the live VPS
+layout — ``~/.aisb/state``, ``~/.claude-mem``, ``~/VibeCoding/**``, etc. None
+of those paths exist in a clean OmegaOS install. OmegaOS owns its own tree
+under ``$OMEGA_HOME``:
+
+  Agentik_Engine/     code (DO NOT touch)
+  Agentik_SSOT/       source of truth for agents/charters (DO NOT touch)
+  Agentik_Tools/      installed binaries (DO NOT touch)
+  Agentik_Vault/      encrypted secrets (DO NOT touch)
+  Agentik_Coding/     project working trees + node_modules
+  Agentik_Runtime/    SQLite DBs + per-task sessions + memory index
+  Agentik_Extra/      var/{tmp,cache,logs}, staging, transcripts
+  Agentik_Providers/  registry yaml (DO NOT touch)
+  Agentik_Orchestration/  topology yaml (DO NOT touch)
+
+The tier list below mirrors the AISB script's spirit but every path is
+OmegaOS-native. Tier 3 delegates to :mod:`omega_engine.prune` so the
+SQLite-bounded growth and per-task session cleanup are one source of truth.
+Tier 4 delegates to :mod:`omega_engine.tmux` so stale workers go through the
+existing kill path.
+
+Modes
+-----
+
+  cleanup(dry_run=True)              — print what would be freed, touch nothing
+  cleanup(dry_run=False)             — apply tiers 1-6
+  cleanup(deep=True)                 — also runs the deep tiers (staging,
+                                        transcripts, VACUUM, opensrc cache)
+  cleanup(target_pct=25)             — stop early once disk usage < target_pct
+  cleanup(idle_days=14)              — override node_modules idle threshold
+
+Returns a list of :class:`TierResult`. Each report fits on one line in the
+CLI summary.
+
+Never touched (hardcoded never-list)
+------------------------------------
+
+  * ``Agentik_Engine/``, ``Agentik_SSOT/``, ``Agentik_Tools/``,
+    ``Agentik_Vault/``, ``Agentik_Providers/``, ``Agentik_Orchestration/``
+  * ``Agentik_Coding/projects/<slug>/`` source files (only ``node_modules``
+    are candidates, and only when the project itself hasn't been touched
+    for ``idle_days`` days)
+  * Per-task session dirs that have a ``brief.json`` but no ``.done.json``
+    (the worker is still running)
+  * Any file currently held open by a live tmux session
+  * Anything under ``$OMEGA_HOME/.vault-key`` or ``.vault-pub``
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import sqlite3
+import subprocess
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class TierResult:
+    tier: str
+    items_removed: int = 0
+    bytes_freed: int = 0
+    dry_run: bool = False
+    detail: str = ""
+    skipped: list[str] = field(default_factory=list)
+
+    @property
+    def line(self) -> str:
+        bytes_str = _human(self.bytes_freed)
+        flag = " [dry-run]" if self.dry_run else ""
+        return (
+            f"{self.tier:30s}  items={self.items_removed:4d}  "
+            f"freed={bytes_str:>8s}{flag}  {self.detail}"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _human(n: int) -> str:
+    if n < 1024:
+        return f"{n}B"
+    for unit in ("K", "M", "G", "T"):
+        n_f = n / 1024.0
+        if n_f < 1024 or unit == "T":
+            return f"{n_f:.1f}{unit}"
+        n = int(n_f)
+    return f"{n}B"
+
+
+def _dir_size(path: Path) -> int:
+    if not path.exists():
+        return 0
+    total = 0
+    try:
+        for p in path.rglob("*"):
+            try:
+                if p.is_file() and not p.is_symlink():
+                    total += p.stat().st_size
+            except OSError:
+                pass
+    except OSError:
+        pass
+    return total
+
+
+def _rm_tree(path: Path, *, dry_run: bool) -> int:
+    """Remove a dir tree. Returns bytes freed (size before)."""
+    if not path.exists():
+        return 0
+    sz = _dir_size(path)
+    if not dry_run:
+        shutil.rmtree(path, ignore_errors=True)
+    return sz
+
+
+def _disk_usage_pct(home: Path) -> int:
+    """Return the % full of the filesystem holding `home`."""
+    try:
+        stat = os.statvfs(str(home))
+        used = (stat.f_blocks - stat.f_bavail) * stat.f_frsize
+        total = stat.f_blocks * stat.f_frsize
+        if total == 0:
+            return 0
+        return int(used / total * 100)
+    except OSError:
+        return 0
+
+
+def _omega_home(home: str | Path | None) -> Path:
+    if home is not None:
+        return Path(home)
+    return Path(os.environ.get("OMEGA_HOME", str(Path.home() / "Omega")))
+
+
+# ---------------------------------------------------------------------------
+# Tier 1 — Regenerable system caches
+# ---------------------------------------------------------------------------
+
+
+_TIER1_PATHS: list[str] = [
+    "~/.npm/_npx",
+    "~/.npm/_cacache",
+    "~/.bun/install/cache",
+    "~/.cache/ms-playwright",
+    "~/.cache/uv",
+    "~/.cache/pip",
+    "~/.cache/bun",
+    "~/.cache/node-gyp",
+]
+
+
+def tier1_caches(*, dry_run: bool = False) -> TierResult:
+    """System-wide regenerable caches. Self-healing on next npm/uv/pip run."""
+    total_bytes = 0
+    items = 0
+    for spec in _TIER1_PATHS:
+        p = Path(spec).expanduser()
+        if not p.exists():
+            continue
+        sz = _rm_tree(p, dry_run=dry_run)
+        if sz > 0:
+            total_bytes += sz
+            items += 1
+    return TierResult(
+        tier="1 system caches",
+        items_removed=items,
+        bytes_freed=total_bytes,
+        dry_run=dry_run,
+        detail=f"paths={items}",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Tier 2 — OmegaOS scratch / temp scaffolds
+# ---------------------------------------------------------------------------
+
+
+def tier2_scratch(home: Path, *, dry_run: bool = False) -> TierResult:
+    """OmegaOS-tagged /tmp scaffolds + $OMEGA_HOME/Agentik_Extra/var/tmp."""
+    items = 0
+    bytes_freed = 0
+
+    # /tmp scaffolds tagged for OmegaOS / AISB
+    tmp = Path("/tmp")
+    if tmp.exists():
+        cutoff = time.time() - 86400  # >1d
+        for child in tmp.iterdir():
+            try:
+                name = child.name
+                if not (name.startswith("omega-") or name.startswith("aisb-")
+                        or name.startswith("omega_") or name.startswith("aisb_")):
+                    continue
+                if child.stat().st_mtime > cutoff:
+                    continue
+            except OSError:
+                continue
+            sz = _dir_size(child) if child.is_dir() else _safe_size(child)
+            if not dry_run:
+                if child.is_dir():
+                    shutil.rmtree(child, ignore_errors=True)
+                else:
+                    try:
+                        child.unlink()
+                    except OSError:
+                        pass
+            items += 1
+            bytes_freed += sz
+
+    # $OMEGA_HOME/Agentik_Extra/var/tmp/*  + var/cache/*
+    var_tmp = home / "Agentik_Extra" / "var" / "tmp"
+    var_cache = home / "Agentik_Extra" / "var" / "cache"
+    for root in (var_tmp, var_cache):
+        if not root.is_dir():
+            continue
+        cutoff = time.time() - 86400
+        for child in root.iterdir():
+            try:
+                if child.stat().st_mtime > cutoff:
+                    continue
+            except OSError:
+                continue
+            sz = _dir_size(child) if child.is_dir() else _safe_size(child)
+            if not dry_run:
+                if child.is_dir():
+                    shutil.rmtree(child, ignore_errors=True)
+                else:
+                    try:
+                        child.unlink()
+                    except OSError:
+                        pass
+            items += 1
+            bytes_freed += sz
+
+    return TierResult(
+        tier="2 scratch + var/tmp", items_removed=items,
+        bytes_freed=bytes_freed, dry_run=dry_run,
+        detail=">1d under /tmp/omega-* + Agentik_Extra/var/{tmp,cache}",
+    )
+
+
+def _safe_size(p: Path) -> int:
+    try:
+        return p.stat().st_size
+    except OSError:
+        return 0
+
+
+# ---------------------------------------------------------------------------
+# Tier 3 — OmegaOS runtime DBs + sessions (delegate to prune)
+# ---------------------------------------------------------------------------
+
+
+def tier3_runtime(
+    home: Path, *, older_than_days: int = 30, dry_run: bool = False,
+) -> TierResult:
+    """Delegate to omega.prune for events/audits/telegram/sessions."""
+    from omega_engine import prune as P
+
+    reports = [
+        P.prune_events(home, older_than_days=older_than_days, dry_run=dry_run),
+        P.prune_audits(home, older_than_days=older_than_days, dry_run=dry_run),
+        P.prune_telegram(home, older_than_days=older_than_days, dry_run=dry_run),
+        P.prune_sessions(home, older_than_days=older_than_days, dry_run=dry_run),
+    ]
+    items = sum(r.rows_deleted + r.files_deleted for r in reports)
+    bytes_freed = sum(r.bytes_freed for r in reports)
+    parts = [
+        f"{r.target}={r.rows_deleted or r.files_deleted}" for r in reports
+    ]
+    return TierResult(
+        tier="3 runtime DBs+sessions", items_removed=items,
+        bytes_freed=bytes_freed, dry_run=dry_run,
+        detail=", ".join(parts) + f" >={older_than_days}d",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Tier 4 — Stale tmux worker sessions
+# ---------------------------------------------------------------------------
+
+
+def tier4_tmux(
+    home: Path, *, older_than_hours: int = 24, dry_run: bool = False,
+) -> TierResult:
+    """Kill detached worker tmux sessions older than `older_than_hours`."""
+    try:
+        from omega_engine import tmux as TX
+    except ImportError:
+        return TierResult(tier="4 tmux workers", dry_run=dry_run,
+                          detail="tmux module unavailable")
+    res = TX.cleanup_stale(home, older_than_hours=older_than_hours,
+                           dry_run=dry_run)
+    items = len(res.get("candidates", []))
+    return TierResult(
+        tier="4 tmux workers", items_removed=items, dry_run=dry_run,
+        detail=f">={older_than_hours}h detached: "
+        f"{', '.join(res.get('candidates', []))[:80]}",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Tier 5 — Idle node_modules in OmegaOS coding tree
+# ---------------------------------------------------------------------------
+
+
+def tier5_node_modules(
+    home: Path, *, idle_days: int = 7, dry_run: bool = False,
+) -> TierResult:
+    """Remove node_modules in $OMEGA_HOME/Agentik_Coding/projects/<slug>/
+    when the *project* itself has had no edit in `idle_days` days.
+
+    We check the project root, not just `node_modules`, so a project that
+    was last touched 10d ago is safe to nuke even if `node_modules` mtime is
+    newer (e.g. a stale lockfile rewrite).
+    """
+    coding = home / "Agentik_Coding" / "projects"
+    if not coding.is_dir():
+        return TierResult(tier="5 node_modules", dry_run=dry_run,
+                          detail="no Agentik_Coding/projects")
+    cutoff = time.time() - idle_days * 86400
+    items = 0
+    bytes_freed = 0
+    skipped = []
+    for proj in coding.iterdir():
+        if not proj.is_dir():
+            continue
+        nm = proj / "node_modules"
+        if not nm.is_dir():
+            continue
+        # Newest edit anywhere under the project (excluding node_modules itself)
+        newest = _newest_mtime(proj, ignore_subdir="node_modules")
+        if newest >= cutoff:
+            skipped.append(proj.name)
+            continue
+        sz = _dir_size(nm)
+        if not dry_run:
+            shutil.rmtree(nm, ignore_errors=True)
+        items += 1
+        bytes_freed += sz
+    return TierResult(
+        tier="5 node_modules", items_removed=items, bytes_freed=bytes_freed,
+        dry_run=dry_run, skipped=skipped,
+        detail=f"idle>={idle_days}d  (kept {len(skipped)} active)",
+    )
+
+
+def _newest_mtime(root: Path, *, ignore_subdir: str | None = None) -> float:
+    """Newest mtime of any FILE under root (dirs are skipped — their mtime
+    bumps on child create/delete, which is noise for "is this project idle").
+    """
+    newest = 0.0
+    try:
+        for p in root.iterdir():
+            if ignore_subdir and p.name == ignore_subdir:
+                continue
+            try:
+                if p.is_dir():
+                    n = _newest_mtime(p)
+                else:
+                    n = p.stat().st_mtime
+                if n > newest:
+                    newest = n
+            except OSError:
+                continue
+    except OSError:
+        pass
+    # If no file anywhere — fall back to the root's own mtime to avoid
+    # nuking a brand-new but legitimately empty project.
+    if newest == 0.0:
+        try:
+            newest = root.stat().st_mtime
+        except OSError:
+            pass
+    return newest
+
+
+# ---------------------------------------------------------------------------
+# Tier 6 — Truncate giant logs
+# ---------------------------------------------------------------------------
+
+
+def tier6_logs(
+    home: Path, *, max_log_mb: int = 20, keep_tail_mb: int = 1,
+    dry_run: bool = False,
+) -> TierResult:
+    """Truncate (NOT delete) Agentik_Extra/var/logs/*.log when they exceed
+    `max_log_mb`. We keep the last `keep_tail_mb` bytes so context survives.
+    Daemons may hold the FD open, so truncate-in-place not unlink."""
+    logs = home / "Agentik_Extra" / "var" / "logs"
+    if not logs.is_dir():
+        return TierResult(tier="6 logs", dry_run=dry_run,
+                          detail="no var/logs")
+    max_bytes = max_log_mb * 1024 * 1024
+    keep_bytes = keep_tail_mb * 1024 * 1024
+    items = 0
+    freed = 0
+    for child in logs.glob("*.log"):
+        try:
+            sz = child.stat().st_size
+        except OSError:
+            continue
+        if sz <= max_bytes:
+            continue
+        if dry_run:
+            items += 1
+            freed += sz - keep_bytes
+            continue
+        try:
+            with child.open("rb") as f:
+                f.seek(-keep_bytes, os.SEEK_END)
+                tail = f.read()
+            with child.open("wb") as f:
+                f.write(tail)
+            items += 1
+            freed += sz - keep_bytes
+        except OSError:
+            continue
+    return TierResult(
+        tier="6 logs", items_removed=items, bytes_freed=freed,
+        dry_run=dry_run, detail=f"truncated >{max_log_mb}MB → {keep_tail_mb}MB tail",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Deep tier — staging, transcripts, VACUUM
+# ---------------------------------------------------------------------------
+
+
+def deep_staging(
+    home: Path, *, older_than_days: int = 14, dry_run: bool = False,
+) -> TierResult:
+    """Delete completed Smith insight proposals older than `older_than_days`.
+
+    Insights live under Agentik_Extra/staging/promotion/*.json — those are
+    OUTPUT of Smith reflection. Once they've been triaged (accepted/rejected)
+    they're safe to nuke. We use mtime as proxy for "triaged"."""
+    staging = home / "Agentik_Extra" / "staging" / "promotion"
+    if not staging.is_dir():
+        return TierResult(tier="D staging", dry_run=dry_run,
+                          detail="no staging dir")
+    cutoff = time.time() - older_than_days * 86400
+    items = 0
+    freed = 0
+    for child in staging.iterdir():
+        try:
+            if child.stat().st_mtime > cutoff:
+                continue
+            sz = _safe_size(child) if child.is_file() else _dir_size(child)
+        except OSError:
+            continue
+        if not dry_run:
+            if child.is_dir():
+                shutil.rmtree(child, ignore_errors=True)
+            else:
+                try:
+                    child.unlink()
+                except OSError:
+                    pass
+        items += 1
+        freed += sz
+    return TierResult(
+        tier="D staging insights", items_removed=items, bytes_freed=freed,
+        dry_run=dry_run, detail=f">={older_than_days}d in staging/promotion",
+    )
+
+
+def deep_transcripts(
+    home: Path, *, older_than_days: int = 7, dry_run: bool = False,
+) -> TierResult:
+    """Drop raw claude transcript blobs older than `older_than_days`.
+
+    Transcripts live as ``Agentik_Runtime/sessions/<task_id>/transcript.json``.
+    They're debug-only — the structured ``.done.json`` is the real artifact.
+    Only nuke transcripts for tasks that have a ``.done.json`` (i.e. completed)."""
+    sess_root = home / "Agentik_Runtime" / "sessions"
+    if not sess_root.is_dir():
+        return TierResult(tier="D transcripts", dry_run=dry_run,
+                          detail="no sessions dir")
+    cutoff = time.time() - older_than_days * 86400
+    items = 0
+    freed = 0
+    for child in sess_root.iterdir():
+        if not child.is_dir():
+            continue
+        done = child / ".done.json"
+        tr = child / "transcript.json"
+        if not (done.exists() and tr.exists()):
+            continue
+        try:
+            if tr.stat().st_mtime > cutoff:
+                continue
+            sz = _safe_size(tr)
+        except OSError:
+            continue
+        if not dry_run:
+            try:
+                tr.unlink()
+            except OSError:
+                continue
+        items += 1
+        freed += sz
+    return TierResult(
+        tier="D transcripts", items_removed=items, bytes_freed=freed,
+        dry_run=dry_run, detail=f">={older_than_days}d for completed tasks",
+    )
+
+
+def deep_vacuum(home: Path, *, dry_run: bool = False) -> TierResult:
+    """VACUUM the OmegaOS SQLite DBs to reclaim freelist pages after prune."""
+    dbs = [
+        home / "Agentik_Runtime" / "eventlog" / "omega.db",
+        home / "Agentik_Runtime" / "audits.db",
+        home / "Agentik_Runtime" / "telegram-history.db",
+        home / "Agentik_Runtime" / "memory" / "index.db",
+    ]
+    items = 0
+    freed = 0
+    for db in dbs:
+        if not db.exists():
+            continue
+        before = _safe_size(db)
+        if dry_run:
+            items += 1
+            continue
+        try:
+            conn = sqlite3.connect(str(db))
+            try:
+                conn.execute("VACUUM")
+                conn.commit()
+            finally:
+                conn.close()
+            after = _safe_size(db)
+            items += 1
+            freed += max(0, before - after)
+        except sqlite3.Error:
+            continue
+    return TierResult(
+        tier="D VACUUM DBs", items_removed=items, bytes_freed=freed,
+        dry_run=dry_run, detail="sqlite freelist reclaim",
+    )
+
+
+def deep_opensrc(*, dry_run: bool = False) -> TierResult:
+    """Wipe ~/.opensrc/repos — re-fetched on next opensrc CLI use.
+    Not OmegaOS-specific but cheap and commonly large."""
+    p = Path.home() / ".opensrc" / "repos"
+    sz = _rm_tree(p, dry_run=dry_run)
+    return TierResult(
+        tier="D opensrc cache", items_removed=1 if sz else 0,
+        bytes_freed=sz, dry_run=dry_run, detail="~/.opensrc/repos",
+    )
+
+
+def deep_git_gc(home: Path, *, dry_run: bool = False) -> TierResult:
+    """`git gc` on the engine + the 5 biggest project repos. Non-destructive."""
+    candidates: list[tuple[int, Path]] = []
+    coding = home / "Agentik_Coding" / "projects"
+    roots = [home]
+    if coding.is_dir():
+        roots.extend(p for p in coding.iterdir() if p.is_dir())
+    for r in roots:
+        git = r / ".git"
+        if git.is_dir():
+            candidates.append((_dir_size(git), r))
+    candidates.sort(key=lambda t: -t[0])
+    items = 0
+    freed = 0
+    for _size, root in candidates[:5]:
+        if dry_run:
+            items += 1
+            continue
+        try:
+            before = _dir_size(root / ".git")
+            subprocess.run(
+                ["git", "-C", str(root), "gc", "--auto", "--quiet"],
+                check=False, capture_output=True, timeout=120,
+            )
+            after = _dir_size(root / ".git")
+            items += 1
+            freed += max(0, before - after)
+        except (OSError, subprocess.SubprocessError):
+            continue
+    return TierResult(
+        tier="D git gc", items_removed=items, bytes_freed=freed,
+        dry_run=dry_run, detail="non-destructive repack",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Top-level — `omega cleanup` entry point
+# ---------------------------------------------------------------------------
+
+
+def cleanup(
+    home: str | Path | None = None,
+    *,
+    dry_run: bool = True,
+    deep: bool = False,
+    target_pct: int | None = None,
+    idle_days: int = 7,
+    runtime_days: int = 30,
+    tmux_hours: int = 24,
+) -> list[TierResult]:
+    """Run the full OmegaOS cleanup pipeline. Returns one report per tier.
+
+    Parameters
+    ----------
+    home          $OMEGA_HOME (defaults to env var, then ~/Omega)
+    dry_run       when True (default), no file is touched
+    deep          run the optional deep tiers (staging, transcripts, VACUUM,
+                  opensrc, git gc)
+    target_pct    stop early once `df` reports the FS for `home` is
+                  under `target_pct` percent full
+    idle_days     node_modules pruning threshold (tier 5)
+    runtime_days  runtime DB / sessions retention (tier 3)
+    tmux_hours    stale-worker tmux age (tier 4)
+    """
+    h = _omega_home(home)
+    out: list[TierResult] = []
+
+    def _maybe_stop() -> bool:
+        if target_pct is None or dry_run:
+            return False
+        return _disk_usage_pct(h) <= target_pct
+
+    out.append(tier1_caches(dry_run=dry_run))
+    if _maybe_stop():
+        return out
+    out.append(tier2_scratch(h, dry_run=dry_run))
+    if _maybe_stop():
+        return out
+    out.append(tier3_runtime(h, older_than_days=runtime_days, dry_run=dry_run))
+    if _maybe_stop():
+        return out
+    out.append(tier4_tmux(h, older_than_hours=tmux_hours, dry_run=dry_run))
+    if _maybe_stop():
+        return out
+    out.append(tier5_node_modules(h, idle_days=idle_days, dry_run=dry_run))
+    if _maybe_stop():
+        return out
+    out.append(tier6_logs(h, dry_run=dry_run))
+    if not deep or _maybe_stop():
+        return out
+
+    # Deep tiers — only on --deep
+    out.append(deep_staging(h, dry_run=dry_run))
+    out.append(deep_transcripts(h, dry_run=dry_run))
+    out.append(deep_vacuum(h, dry_run=dry_run))
+    out.append(deep_opensrc(dry_run=dry_run))
+    out.append(deep_git_gc(h, dry_run=dry_run))
+    return out
+
+
+def format_summary(reports: list[TierResult], home: Path) -> str:
+    """Pretty-print the report list — one line per tier + a final total."""
+    lines = ["=== OmegaOS cleanup ==="]
+    lines.append(f"home: {home}")
+    total = 0
+    for r in reports:
+        lines.append("  " + r.line)
+        total += r.bytes_freed
+    flag = " [dry-run]" if reports and reports[0].dry_run else ""
+    lines.append(f"-- total freed: {_human(total)}{flag}")
+    return "\n".join(lines)