@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/scripts/generate_summary_tiers.py

@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+"""Generate tiered summaries from MEMORY.md files.
+
+Creates Level 0/1/2 summary templates with token budgets:
+- Level 0 (Ultra-compact): ~200 tokens - key facts only
+- Level 1 (Working memory): ~1000 tokens - active context
+- Level 2 (Full context): ~3000 tokens - complete reference
+
+Usage:
+    python3 generate_summary_tiers.py <path> [--json] [--output-dir DIR]
+
+Part of claw-compactor. License: MIT.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from lib.tokens import estimate_tokens
+from lib.markdown import parse_sections
+from lib.exceptions import FileNotFoundError_
+
+logger = logging.getLogger(__name__)
+
+# Tier definitions
+TIERS = {
+    0: {"name": "Ultra-compact", "budget": 200, "description": "Key facts and critical decisions only"},
+    1: {"name": "Working memory", "budget": 1000, "description": "Active context for daily work"},
+    2: {"name": "Full context", "budget": 3000, "description": "Complete reference with details"},
+}
+
+# Section priority for compression (higher = keep more)
+SECTION_PRIORITIES = {
+    "decision": 10,
+    "critical": 10,
+    "important": 9,
+    "action": 8,
+    "todo": 8,
+    "config": 7,
+    "setup": 7,
+    "architecture": 7,
+    "preference": 6,
+    "convention": 6,
+    "lesson": 5,
+    "note": 4,
+    "log": 3,
+    "history": 2,
+    "archive": 1,
+}
+
+DEFAULT_PRIORITY = 5
+
+
+def _classify_section(header: str) -> int:
+    """Classify a section header by priority.
+
+    Returns a priority score (1-10). Higher = more important.
+    """
+    h = header.lower()
+    for keyword, priority in SECTION_PRIORITIES.items():
+        if keyword in h:
+            return priority
+    return DEFAULT_PRIORITY
+
+
+def _find_memory_files(target: str) -> List[Path]:
+    """Find memory files to process.
+
+    Raises FileNotFoundError_ if target doesn't exist.
+    """
+    p = Path(target)
+    if not p.exists():
+        raise FileNotFoundError_(f"Path not found: {target}")
+
+    if p.is_file():
+        return [p]
+
+    files = []
+    # Prioritize MEMORY.md
+    mem = p / "MEMORY.md"
+    if mem.exists():
+        files.append(mem)
+    # Add other root .md files
+    for f in sorted(p.glob("*.md")):
+        if f.name != "MEMORY.md" and f not in files:
+            files.append(f)
+    # memory/ subdirectory
+    mem_dir = p / "memory"
+    if mem_dir.is_dir():
+        for f in sorted(mem_dir.glob("*.md")):
+            files.append(f)
+
+    return files
+
+
+def generate_tiers(files: List[Path]) -> Dict[str, Any]:
+    """Generate tier analysis from memory files.
+
+    Returns a dict with total_tokens, total_sections, and per-tier info.
+    """
+    # Collect all sections with priorities
+    all_sections: List[Dict[str, Any]] = []
+    total_tokens = 0
+
+    for f in files:
+        text = f.read_text(encoding="utf-8", errors="replace")
+        tokens = estimate_tokens(text)
+        total_tokens += tokens
+        sections = parse_sections(text)
+        for header, body, level in sections:
+            sec_tokens = estimate_tokens(header + '\n' + body) if (header or body) else 0
+            priority = _classify_section(header) if header else DEFAULT_PRIORITY
+            all_sections.append({
+                "header": header,
+                "body": body,
+                "level": level,
+                "tokens": sec_tokens,
+                "priority": priority,
+                "file": str(f),
+            })
+
+    # Sort by priority descending, then by token count ascending
+    all_sections.sort(key=lambda s: (-s["priority"], s["tokens"]))
+
+    # Build tiers
+    tiers: Dict[int, Dict[str, Any]] = {}
+    for tier_level, tier_def in TIERS.items():
+        budget = tier_def["budget"]
+        selected: List[Dict[str, Any]] = []
+        used = 0
+        for sec in all_sections:
+            if used + sec["tokens"] <= budget:
+                selected.append(sec)
+                used += sec["tokens"]
+        tiers[tier_level] = {
+            "name": tier_def["name"],
+            "budget": budget,
+            "description": tier_def["description"],
+            "sections_included": len(selected),
+            "tokens_used": used,
+            "sections": selected,
+        }
+
+    return {
+        "total_tokens": total_tokens,
+        "total_sections": len(all_sections),
+        "tiers": tiers,
+    }
+
+
+def format_tier_template(result: Dict[str, Any], level: int) -> str:
+    """Format a tier as a markdown template."""
+    tier = result["tiers"][level]
+    lines = [
+        f"# Level {level} — {tier['name']}",
+        f"Budget: {tier['budget']} tokens | Used: {tier['tokens_used']}",
+        f"Sections: {tier['sections_included']}",
+        "",
+    ]
+    for sec in tier["sections"]:
+        if sec["header"]:
+            lines.append(f"## {sec['header']}")
+        if sec["body"]:
+            lines.append(sec["body"])
+        lines.append("")
+
+    return '\n'.join(lines)
+
+
+def format_human(result: Dict[str, Any]) -> str:
+    """Format tier analysis as a human-readable report."""
+    lines = [
+        "=== Summary Tier Analysis ===",
+        f"Total tokens: {result['total_tokens']:,}",
+        f"Total sections: {result['total_sections']}",
+        "",
+    ]
+    for level in range(3):
+        tier = result["tiers"][level]
+        lines.append(f"Level {level} ({tier['name']}):")
+        lines.append(f"  Budget: {tier['budget']} tokens")
+        lines.append(f"  Used: {tier['tokens_used']} tokens")
+        lines.append(f"  Sections: {tier['sections_included']}")
+        lines.append("")
+
+    return '\n'.join(lines)
+
+
+def extract_key_facts(text: str) -> List[str]:
+    """Extract key facts from markdown text.
+
+    Identifies lines with key:value patterns, important markers, and
+    critical information. Returns deduplicated list of fact strings.
+    """
+    if not text:
+        return []
+
+    facts: List[str] = []
+    seen: set = set()
+
+    for line in text.split('\n'):
+        line = line.strip()
+        if not line:
+            continue
+        # Skip headers
+        if line.startswith('#'):
+            continue
+        # Strip bullet prefix
+        clean = line.lstrip('- *+').strip()
+        if not clean:
+            continue
+
+        # Key:value patterns, important markers
+        is_fact = (
+            ':' in clean
+            or any(m in line for m in ['⚠️', 'Critical', 'Important', 'IMPORTANT', 'WARNING'])
+            or any(c.isdigit() for c in clean)  # Contains numbers
+        )
+
+        if is_fact and clean not in seen:
+            seen.add(clean)
+            facts.append(clean)
+
+    return facts
+
+
+def generate_auto_summary(
+    files: List[Path],
+    budget: int = 200,
+) -> str:
+    """Generate an automatic summary from memory files within token budget.
+
+    Extracts key facts and fills up to budget tokens.
+    """
+    all_facts: List[str] = []
+    for f in files:
+        text = f.read_text(encoding="utf-8", errors="replace")
+        all_facts.extend(extract_key_facts(text))
+
+    lines = ["# Auto Summary", ""]
+    used_tokens = estimate_tokens('\n'.join(lines))
+
+    for fact in all_facts:
+        fact_line = f"- {fact}"
+        fact_tokens = estimate_tokens(fact_line)
+        if used_tokens + fact_tokens > budget:
+            break
+        lines.append(fact_line)
+        used_tokens += fact_tokens
+
+    return '\n'.join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate tiered summaries")
+    parser.add_argument("path", help="File or directory")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    parser.add_argument("--output-dir", help="Write tier files to this directory")
+    args = parser.parse_args()
+
+    files = _find_memory_files(args.path)
+    result = generate_tiers(files)
+
+    if args.json:
+        # Make JSON-serializable (remove section bodies for brevity)
+        output = {
+            "total_tokens": result["total_tokens"],
+            "total_sections": result["total_sections"],
+            "tiers": {
+                k: {kk: vv for kk, vv in v.items() if kk != "sections"}
+                for k, v in result["tiers"].items()
+            },
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        print(format_human(result))
+
+    if args.output_dir:
+        out = Path(args.output_dir)
+        out.mkdir(parents=True, exist_ok=True)
+        for level in range(3):
+            (out / f"MEMORY-L{level}.md").write_text(
+                format_tier_template(result, level), encoding="utf-8"
+            )
+
+
+if __name__ == "__main__":
+    main()

package/skills/claw-compactor/scripts/lib/__init__.py

@@ -0,0 +1,34 @@
+"""claw-compactor shared library.
+
+Core utilities for token estimation, markdown parsing, deduplication,
+dictionary encoding, run-length encoding, and format optimization.
+
+Layer 6 — Engram Observational Memory engine:
+    EngramEngine      — main public API
+    EngramStorage     — file-system storage backend
+    OBSERVER_SYSTEM_PROMPT   — Observer LLM system prompt
+    REFLECTOR_SYSTEM_PROMPT  — Reflector LLM system prompt
+
+Part of claw-compactor. License: MIT.
+"""
+
+# Expose Engram classes at package level for convenient imports:
+#   from scripts.lib import EngramEngine
+#   from scripts.lib import EngramStorage
+from lib.engram import EngramEngine  # noqa: F401
+from lib.engram_storage import EngramStorage  # noqa: F401
+from lib.engram_prompts import (  # noqa: F401
+    OBSERVER_SYSTEM_PROMPT,
+    REFLECTOR_SYSTEM_PROMPT,
+    OBSERVER_USER_TEMPLATE,
+    REFLECTOR_USER_TEMPLATE,
+)
+
+__all__ = [
+    "EngramEngine",
+    "EngramStorage",
+    "OBSERVER_SYSTEM_PROMPT",
+    "REFLECTOR_SYSTEM_PROMPT",
+    "OBSERVER_USER_TEMPLATE",
+    "REFLECTOR_USER_TEMPLATE",
+]

package/skills/claw-compactor/scripts/lib/config.py

@@ -0,0 +1,289 @@
+"""Configuration management for claw-compactor.
+
+Loads settings from claw-compactor-config.json in the workspace root,
+falling back to sensible defaults.
+
+Also provides load_engram_config() for the Engram (Layer 6) subsystem,
+reading engram.yaml (or engram.json as fallback) with env-var overrides
+and .env file fallback.
+
+Part of claw-compactor. License: MIT.
+"""
+
+import json
+import logging
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+logger = logging.getLogger("claw-compactor.config")
+
+DEFAULT_CONFIG: Dict[str, Any] = {
+    "chars_per_token": 4,
+    "level0_max_tokens": 200,
+    "level1_max_tokens": 500,
+    "dedup_similarity_threshold": 0.6,
+    "dedup_shingle_size": 3,
+    "dedup_max_results": 50,
+    "dedup_min_line_length": 20,
+    "compress_min_tokens": 50,
+    "compress_target_ratio": 0.4,
+    "date_format": "%Y-%m-%d",
+    "memory_dir": "memory",
+    "memory_file": "MEMORY.md",
+    "summary_tiers_file": "memory/summary-tiers.md",
+    "compressed_suffix": ".compressed.md",
+    "log_level": "INFO",
+}
+
+CONFIG_FILENAME = "claw-compactor-config.json"
+
+
+@dataclass
+class MemCompressConfig:
+    """Runtime configuration for claw-compactor."""
+
+    chars_per_token: int = DEFAULT_CONFIG["chars_per_token"]
+    level0_max_tokens: int = DEFAULT_CONFIG["level0_max_tokens"]
+    level1_max_tokens: int = DEFAULT_CONFIG["level1_max_tokens"]
+    dedup_similarity_threshold: float = DEFAULT_CONFIG["dedup_similarity_threshold"]
+    dedup_shingle_size: int = DEFAULT_CONFIG["dedup_shingle_size"]
+    dedup_max_results: int = DEFAULT_CONFIG["dedup_max_results"]
+    dedup_min_line_length: int = DEFAULT_CONFIG["dedup_min_line_length"]
+    compress_min_tokens: int = DEFAULT_CONFIG["compress_min_tokens"]
+    compress_target_ratio: float = DEFAULT_CONFIG["compress_target_ratio"]
+    date_format: str = DEFAULT_CONFIG["date_format"]
+    memory_dir: str = DEFAULT_CONFIG["memory_dir"]
+    memory_file: str = DEFAULT_CONFIG["memory_file"]
+    summary_tiers_file: str = DEFAULT_CONFIG["summary_tiers_file"]
+    compressed_suffix: str = DEFAULT_CONFIG["compressed_suffix"]
+    log_level: str = DEFAULT_CONFIG["log_level"]
+
+
+def _deep_merge(base: dict, override: dict) -> dict:
+    """Recursively merge *override* into *base* (non-destructive)."""
+    result = dict(base)
+    for k, v in override.items():
+        if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+            result[k] = _deep_merge(result[k], v)
+        else:
+            result[k] = v
+    return result
+
+
+def _load_dotenv(env_path: Path) -> None:
+    """Load KEY=VALUE pairs from *env_path* into os.environ (skips if exists)."""
+    if not env_path.exists():
+        return
+    try:
+        for line in env_path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            key, _, val = line.partition("=")
+            os.environ.setdefault(key.strip(), val.strip())
+    except OSError:
+        pass
+
+
+def load_engram_config(config_path: Optional[Path] = None) -> Dict[str, Any]:
+    """Load Engram configuration with priority: env vars > YAML/JSON > .env > defaults.
+
+    Search order for the config file (when *config_path* is not given):
+      1. $ENGRAM_CONFIG env var
+      2. ./engram.yaml  (relative to claw-compactor root, auto-detected)
+      3. ./engram.json
+
+    Returns a flat-ish dict that EngramEngine / engram_auto.py can consume.
+    All path values are expanded (~ / $HOME).
+
+    Env-var overrides (any of these set → take precedence over YAML):
+      ENGRAM_MODEL         → llm.model
+      ENGRAM_MAX_TOKENS    → llm.max_tokens
+      OPENAI_BASE_URL      → llm.base_url  (for openai-compatible provider)
+      ENGRAM_PROVIDER      → llm.provider
+      ENGRAM_OBSERVER_THRESHOLD   → threads.default.observer_threshold
+      ENGRAM_REFLECTOR_THRESHOLD  → threads.default.reflector_threshold
+      ENGRAM_MAX_WORKERS   → concurrency.max_workers
+      ENGRAM_SCAN_DIR      → sessions.scan_dir
+      ENGRAM_STORAGE_DIR   → storage.base_dir
+    """
+    # ------------------------------------------------------------------ #
+    # 0. Locate claw-compactor root                                       #
+    # ------------------------------------------------------------------ #
+    here = Path(__file__).resolve().parent.parent.parent  # scripts/lib → root
+
+    # ------------------------------------------------------------------ #
+    # 1. Load .env first (lowest priority, sets env vars for everything)  #
+    # ------------------------------------------------------------------ #
+    _load_dotenv(here / ".env")
+
+    # ------------------------------------------------------------------ #
+    # 2. Defaults                                                         #
+    # ------------------------------------------------------------------ #
+    defaults: Dict[str, Any] = {
+        "llm": {
+            "provider": "openai-compatible",
+            "base_url": "https://api.openai.com",
+            "api_key_env": "OPENAI_API_KEY",
+            "model": "gpt-4o",
+            "max_tokens": 4096,
+        },
+        "threads": {
+            "default": {
+                "observer_threshold": 30000,
+                "reflector_threshold": 40000,
+            }
+        },
+        "sessions": {
+            "scan_dir": "~/.openclaw/agents/main/sessions",
+            "max_age_hours": 48,
+        },
+        "storage": {
+            "base_dir": "~/.openclaw/workspace/memory/engram",
+        },
+        "concurrency": {
+            "max_workers": 4,
+        },
+        "logging": {
+            "level": "INFO",
+        },
+    }
+
+    # ------------------------------------------------------------------ #
+    # 3. Find and load config file                                        #
+    # ------------------------------------------------------------------ #
+    cfg: Dict[str, Any] = dict(defaults)
+
+    if config_path is None:
+        env_cfg = os.environ.get("ENGRAM_CONFIG", "")
+        if env_cfg:
+            config_path = Path(env_cfg).expanduser()
+        else:
+            for candidate in [here / "engram.yaml", here / "engram.json"]:
+                if candidate.exists():
+                    config_path = candidate
+                    break
+
+    if config_path and config_path.exists():
+        try:
+            text = config_path.read_text(encoding="utf-8")
+            if config_path.suffix in (".yaml", ".yml"):
+                try:
+                    import yaml  # type: ignore[import]
+                    file_cfg = yaml.safe_load(text) or {}
+                except ImportError:
+                    logger.warning(
+                        "PyYAML not available; ignoring %s. "
+                        "Install pyyaml or rename to engram.json.",
+                        config_path,
+                    )
+                    file_cfg = {}
+            else:
+                file_cfg = json.loads(text) if text.strip() else {}
+
+            if isinstance(file_cfg, dict):
+                cfg = _deep_merge(defaults, file_cfg)
+                logger.debug("Engram config loaded from %s", config_path)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to load Engram config %s: %s — using defaults", config_path, exc)
+    else:
+        logger.debug("No engram.yaml / engram.json found; using defaults + env vars")
+
+    # ------------------------------------------------------------------ #
+    # 4. Env-var overrides (highest priority)                             #
+    # ------------------------------------------------------------------ #
+    _env = os.environ
+
+    if _env.get("ENGRAM_PROVIDER"):
+        cfg["llm"]["provider"] = _env["ENGRAM_PROVIDER"]
+    if _env.get("OPENAI_BASE_URL"):
+        cfg["llm"]["base_url"] = _env["OPENAI_BASE_URL"]
+    if _env.get("ENGRAM_MODEL"):
+        cfg["llm"]["model"] = _env["ENGRAM_MODEL"]
+    if _env.get("ENGRAM_MAX_TOKENS"):
+        try:
+            cfg["llm"]["max_tokens"] = int(_env["ENGRAM_MAX_TOKENS"])
+        except ValueError:
+            pass
+    if _env.get("ENGRAM_OBSERVER_THRESHOLD"):
+        try:
+            cfg["threads"]["default"]["observer_threshold"] = int(_env["ENGRAM_OBSERVER_THRESHOLD"])
+        except ValueError:
+            pass
+    if _env.get("ENGRAM_REFLECTOR_THRESHOLD"):
+        try:
+            cfg["threads"]["default"]["reflector_threshold"] = int(_env["ENGRAM_REFLECTOR_THRESHOLD"])
+        except ValueError:
+            pass
+    if _env.get("ENGRAM_MAX_WORKERS"):
+        try:
+            cfg["concurrency"]["max_workers"] = int(_env["ENGRAM_MAX_WORKERS"])
+        except ValueError:
+            pass
+    if _env.get("ENGRAM_SCAN_DIR"):
+        cfg["sessions"]["scan_dir"] = _env["ENGRAM_SCAN_DIR"]
+    if _env.get("ENGRAM_STORAGE_DIR"):
+        cfg["storage"]["base_dir"] = _env["ENGRAM_STORAGE_DIR"]
+
+    # Expand ~ in paths
+    cfg["sessions"]["scan_dir"] = str(Path(cfg["sessions"]["scan_dir"]).expanduser())
+    cfg["storage"]["base_dir"] = str(Path(cfg["storage"]["base_dir"]).expanduser())
+
+    return cfg
+
+
+def engram_engine_kwargs(engram_cfg: Dict[str, Any]) -> Dict[str, Any]:
+    """Convert load_engram_config() output into EngramEngine constructor kwargs."""
+    llm = engram_cfg.get("llm", {})
+    thread_defaults = engram_cfg.get("threads", {}).get("default", {})
+
+    # Resolve API key from the named env var
+    api_key_env = llm.get("api_key_env", "OPENAI_API_KEY")
+    api_key = os.environ.get(api_key_env, "")
+
+    provider = llm.get("provider", "openai-compatible")
+    if provider == "anthropic":
+        anthropic_key = api_key or os.environ.get("ANTHROPIC_API_KEY", "")
+        openai_key = ""
+        openai_url = ""
+    else:
+        anthropic_key = os.environ.get("ANTHROPIC_API_KEY", "")
+        openai_key = api_key or os.environ.get("OPENAI_API_KEY", "")
+        openai_url = llm.get("base_url", "")
+
+    return {
+        "observer_threshold": thread_defaults.get("observer_threshold", 30000),
+        "reflector_threshold": thread_defaults.get("reflector_threshold", 40000),
+        "model": llm.get("model"),
+        "max_tokens": llm.get("max_tokens", 4096),
+        "anthropic_api_key": anthropic_key,
+        "openai_api_key": openai_key,
+        "openai_base_url": openai_url,
+    }
+
+
+def load_config(workspace: Path) -> MemCompressConfig:
+    """Load configuration from *workspace*/claw-compactor-config.json.
+
+    Returns default config if the file is missing, empty, or invalid.
+    """
+    config_path = workspace / CONFIG_FILENAME
+    if not config_path.exists():
+        return MemCompressConfig()
+    try:
+        text = config_path.read_text(encoding="utf-8").strip()
+        if not text:
+            return MemCompressConfig()
+        data = json.loads(text)
+        if not isinstance(data, dict):
+            logger.warning("Config is not a JSON object, using defaults")
+            return MemCompressConfig()
+        # Filter to known fields only
+        known = {f.name for f in MemCompressConfig.__dataclass_fields__.values()}
+        filtered = {k: v for k, v in data.items() if k in known}
+        return MemCompressConfig(**filtered)
+    except (json.JSONDecodeError, TypeError, ValueError) as exc:
+        logger.warning("Invalid config %s: %s — using defaults", config_path, exc)
+        return MemCompressConfig()