@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/scripts/compressed_context.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+"""Compressed Context Protocol -- compress text for expensive model consumption.
+
+Generates ultra-compressed context + decompression instructions for system prompts.
+Three compression levels: ultra, medium, light.
+
+Usage:
+    python3 compressed_context.py <file> [--level ultra|medium|light] [--output FILE]
+
+Part of claw-compactor. License: MIT.
+"""
+
+import argparse
+import json
+import logging
+import re
+import sys
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from lib.tokens import estimate_tokens
+
+logger = logging.getLogger(__name__)
+
+# Decompression instructions to prepend to system prompt
+DECOMPRESS_INSTRUCTIONS = {
+    "ultra": (
+        "Compressed notation: key:val=attribute, loc:X+Y=locations, "
+        "Ny+=N+ years, slash-separated=alternatives. "
+        "Expand naturally when responding."
+    ),
+    "medium": (
+        "Text uses abbreviated notation: key:value pairs, "
+        "condensed lists, minimal punctuation. Read as natural language."
+    ),
+    "light": (
+        "Text is lightly condensed. Read normally."
+    ),
+}
+
+# Common words to abbreviate in ultra mode
+ULTRA_ABBREVS = {
+    "experience": "exp",
+    "management": "mgmt",
+    "development": "dev",
+    "approximately": "~",
+    "application": "app",
+    "applications": "apps",
+    "configuration": "config",
+    "information": "info",
+    "environment": "env",
+    "infrastructure": "infra",
+    "architecture": "arch",
+    "implementation": "impl",
+    "performance": "perf",
+    "operations": "ops",
+    "production": "prod",
+    "repository": "repo",
+    "repositories": "repos",
+    "documentation": "docs",
+    "communication": "comms",
+    "organization": "org",
+    "technology": "tech",
+    "technologies": "tech",
+    "cryptocurrency": "crypto",
+    "quantitative": "quant",
+    "distributed": "dist",
+    "international": "intl",
+    "professional": "pro",
+    "certificate": "cert",
+    "authentication": "auth",
+    "authorization": "authz",
+    "database": "db",
+    "kubernetes": "k8s",
+    "continuous": "cont",
+    "integration": "integ",
+    "deployment": "deploy",
+    "monitoring": "mon",
+    "notification": "notif",
+    "requirements": "reqs",
+    "specification": "spec",
+    "administrator": "admin",
+    "description": "desc",
+    "transaction": "tx",
+    "transactions": "txs",
+    "currently": "curr",
+    "previously": "prev",
+    "following": "foll",
+    "including": "incl",
+    "especially": "esp",
+    "engineering": "eng",
+    "university": "univ",
+    "founded": "founder",
+    "established": "est",
+    "headquarters": "HQ",
+    "years of": "y+",
+    "based in": "loc:",
+    "located in": "loc:",
+    "offices in": "offices:",
+    "founder of": "founder:",
+    "CEO of": "CEO:",
+    "CTO of": "CTO:",
+}
+
+# Filler phrases to remove in ultra mode
+ULTRA_FILLERS = [
+    "In addition,", "Furthermore,", "Moreover,", "Additionally,",
+    "It is worth noting that", "It should be noted that",
+    "As a matter of fact,", "In fact,", "Actually,",
+    "Basically,", "Essentially,", "In other words,",
+    "That being said,", "Having said that,",
+    "At the end of the day,", "When it comes to",
+    "In terms of", "With regard to", "With respect to",
+    "As mentioned earlier,", "As previously stated,",
+    "It is important to note that", "Please note that",
+    "In conclusion,", "To summarize,", "To sum up,",
+    "extensive experience", "extensive experience in",
+    "He has", "She has", "They have",
+    "which is", "that is", "who is",
+    "a wide range of", "a variety of",
+]
+
+# Medium-mode abbreviations (less aggressive)
+MEDIUM_ABBREVS = {
+    "configuration": "config",
+    "application": "app",
+    "environment": "env",
+    "infrastructure": "infra",
+    "implementation": "impl",
+    "documentation": "docs",
+    "database": "db",
+    "kubernetes": "k8s",
+}
+
+
+def compress_ultra(text: str) -> str:
+    """Apply ultra compression -- aggressive abbreviation and filler removal."""
+    if not text:
+        return ""
+
+    result = text
+
+    # Remove fillers
+    for filler in ULTRA_FILLERS:
+        result = result.replace(filler, "")
+
+    # Apply abbreviations (case-insensitive for the word, preserve surrounding)
+    for word, abbrev in ULTRA_ABBREVS.items():
+        # Replace whole words
+        result = re.sub(r'\b' + re.escape(word) + r'\b', abbrev, result, flags=re.IGNORECASE)
+
+    # Remove articles and common short fillers
+    result = re.sub(r'\b(?:the|a|an|is|are|was|were|has|have|had|been|being)\b\s*', '', result, flags=re.IGNORECASE)
+    # Remove "of" in common phrases but keep meaningful ones
+    result = re.sub(r'\bof\b\s+', ' ', result)
+    # Remove "and" → "+"
+    result = re.sub(r'\band\b', '+', result)
+    # Remove "with" → "w/"
+    result = re.sub(r'\bwith\b', 'w/', result)
+    # Remove "for" → "4"
+    result = re.sub(r'\bfor\b', '4', result)
+    # "in" → "in" (keep, too short to abbreviate)
+
+    # Clean up spacing
+    result = re.sub(r'  +', ' ', result)
+    result = re.sub(r'\n{3,}', '\n\n', result)
+    result = re.sub(r'^\s+', '', result, flags=re.MULTILINE)
+
+    return result.strip()
+
+
+def compress_medium(text: str) -> str:
+    """Apply medium compression -- moderate abbreviation."""
+    if not text:
+        return ""
+
+    result = text
+
+    # Apply medium abbreviations only
+    for word, abbrev in MEDIUM_ABBREVS.items():
+        result = re.sub(r'\b' + re.escape(word) + r'\b', abbrev, result, flags=re.IGNORECASE)
+
+    # Remove some fillers
+    for filler in ULTRA_FILLERS[:5]:  # Only the most common
+        result = result.replace(filler, "")
+
+    # Clean up
+    result = re.sub(r'  +', ' ', result)
+    result = re.sub(r'\n{3,}', '\n\n', result)
+
+    return result.strip()
+
+
+def compress_light(text: str) -> str:
+    """Apply light compression -- just cleanup."""
+    if not text:
+        return ""
+
+    result = text
+    result = re.sub(r'  +', ' ', result)
+    result = re.sub(r'\n{3,}', '\n\n', result)
+    return result.strip()
+
+
+def compress(text: str, level: str) -> Dict[str, str]:
+    """Compress text at the specified level.
+
+    Returns dict with compressed text, instructions, and level.
+    Raises ValueError for invalid level.
+    """
+    if level not in DECOMPRESS_INSTRUCTIONS:
+        raise ValueError(f"Invalid compression level: {level}. Use: ultra, medium, light")
+
+    compressors = {
+        "ultra": compress_ultra,
+        "medium": compress_medium,
+        "light": compress_light,
+    }
+
+    compressed = compressors[level](text)
+    return {
+        "compressed": compressed,
+        "instructions": DECOMPRESS_INSTRUCTIONS[level],
+        "level": level,
+    }
+
+
+def compress_with_stats(text: str, level: str) -> Dict[str, Any]:
+    """Compress text and return statistics.
+
+    Returns dict with compressed text, token counts, and reduction percentage.
+    """
+    result = compress(text, level)
+    orig_tokens = estimate_tokens(text)
+    comp_tokens = estimate_tokens(result["compressed"])
+    inst_tokens = estimate_tokens(result["instructions"])
+
+    # Net includes instruction overhead
+    net_tokens = comp_tokens + inst_tokens
+    reduction = ((orig_tokens - comp_tokens) / orig_tokens * 100) if orig_tokens > 0 else 0.0
+
+    return {
+        "compressed": result["compressed"],
+        "instructions": result["instructions"],
+        "level": level,
+        "original_tokens": orig_tokens,
+        "compressed_tokens": comp_tokens,
+        "instruction_tokens": inst_tokens,
+        "net_tokens": net_tokens,
+        "reduction_pct": round(reduction, 1),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compressed Context Protocol")
+    parser.add_argument("file", help="File to compress")
+    parser.add_argument("--level", default="ultra", choices=["ultra", "medium", "light"])
+    parser.add_argument("--output", help="Output file")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    text = Path(args.file).read_text(encoding="utf-8")
+    stats = compress_with_stats(text, args.level)
+
+    if args.output:
+        Path(args.output).write_text(stats["compressed"], encoding="utf-8")
+
+    if args.json:
+        print(json.dumps(stats, indent=2))
+    else:
+        pct = stats["reduction_pct"]
+        print(f"Level: {args.level}")
+        print(f"Original: {stats['original_tokens']} tokens")
+        print(f"Compressed: {stats['compressed_tokens']} tokens ({pct:.1f}% reduction)")
+        print(f"Instructions: {stats['instruction_tokens']} tokens")
+
+
+if __name__ == "__main__":
+    main()

package/skills/claw-compactor/scripts/dedup_memory.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Find and merge near-duplicate entries across memory files.
+
+Uses shingle hashing for efficient similarity detection without O(n^2) comparison.
+
+Usage:
+    python3 dedup_memory.py <path> [--json] [--auto-merge] [--threshold 0.6]
+
+Part of claw-compactor. License: MIT.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Dict, List, Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from lib.tokens import estimate_tokens
+from lib.markdown import parse_sections, strip_markdown_redundancy
+from lib.dedup import find_duplicates, merge_duplicates, SIMILARITY_THRESHOLD
+from lib.exceptions import FileNotFoundError_
+
+logger = logging.getLogger(__name__)
+
+
+def _collect_entries(target: str) -> List[Dict[str, Any]]:
+    """Collect bullet/paragraph entries from markdown files at *target*.
+
+    Returns a list of dicts with 'text', 'source', and 'section' keys.
+    """
+    path = Path(target)
+    if not path.exists():
+        raise FileNotFoundError_(f"Path not found: {target}")
+
+    files = [path] if path.is_file() else sorted(path.rglob("*.md"))
+    entries: List[Dict[str, Any]] = []
+
+    for f in files:
+        text = f.read_text(encoding="utf-8")
+        if not text.strip():
+            continue
+        sections = parse_sections(text)
+        for header, body, level in sections:
+            if not body.strip():
+                continue
+            # Split body into bullet lines or paragraphs
+            for line in body.split('\n'):
+                line = line.strip()
+                if line and len(line) > 10:  # Skip very short lines
+                    entries.append({
+                        "text": line,
+                        "source": str(f),
+                        "section": header,
+                    })
+
+    return entries
+
+
+def run_dedup(
+    target: str,
+    threshold: float = SIMILARITY_THRESHOLD,
+    auto_merge: bool = False,
+) -> Dict[str, Any]:
+    """Run deduplication on *target* (file or directory).
+
+    Returns a dict with statistics and duplicate groups.
+    """
+    entries = _collect_entries(target)
+    texts = [e["text"] for e in entries]
+
+    tokens_before = estimate_tokens('\n'.join(texts))
+    groups = find_duplicates(texts, threshold=threshold)
+
+    result: Dict[str, Any] = {
+        "total_entries": len(entries),
+        "duplicate_groups": groups,
+        "duplicate_group_count": len(groups),
+        "entries_removed": 0,
+        "tokens_before": tokens_before,
+    }
+
+    if auto_merge and groups:
+        merged = merge_duplicates(texts, groups)
+        tokens_after = estimate_tokens('\n'.join(merged))
+        result["entries_removed"] = len(texts) - len(merged)
+        result["tokens_after"] = tokens_after
+        result["tokens_saved"] = tokens_before - tokens_after
+
+    if groups:
+        result["groups"] = []
+        for g in groups:
+            group_entries = [entries[i] for i in g["indices"]]
+            result["groups"].append({
+                "similarity": g["similarity"],
+                "entries": [e["text"][:100] for e in group_entries],
+                "sources": list(set(e["source"] for e in group_entries)),
+            })
+
+    return result
+
+
+def format_human(result: Dict[str, Any]) -> str:
+    """Format dedup results as a human-readable report."""
+    lines = ["# Deduplication Report", ""]
+    lines.append(f"Total entries scanned: {result['total_entries']}")
+    groups = result['duplicate_groups']
+    num_groups = len(groups) if isinstance(groups, list) else groups
+    lines.append(f"Duplicate groups found: {num_groups}")
+
+    if not num_groups:
+        lines.append("\nNo duplicates found.")
+        return '\n'.join(lines)
+
+    lines.append(f"Entries removed: {result.get('entries_removed', 0)}")
+    if "tokens_saved" in result:
+        lines.append(f"Tokens saved: {result['tokens_saved']}")
+
+    if "groups" in result:
+        lines.append("\n## Groups")
+        for i, g in enumerate(result["groups"]):
+            lines.append(f"\n### Group {i + 1} (similarity: {g['similarity']:.2f})")
+            for entry in g["entries"]:
+                lines.append(f"  - {entry}")
+
+    return '\n'.join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Find near-duplicate memory entries")
+    parser.add_argument("path", help="File or directory to scan")
+    parser.add_argument("--json", action="store_true")
+    parser.add_argument("--auto-merge", action="store_true")
+    parser.add_argument("--threshold", type=float, default=SIMILARITY_THRESHOLD)
+    args = parser.parse_args()
+
+    result = run_dedup(args.path, threshold=args.threshold, auto_merge=args.auto_merge)
+
+    if args.json:
+        print(json.dumps(result, indent=2, ensure_ascii=False))
+    else:
+        print(format_human(result))
+
+
+if __name__ == "__main__":
+    main()

package/skills/claw-compactor/scripts/dictionary_compress.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Dictionary-based compression for workspace memory files.
+
+Learns high-frequency phrases from the workspace, builds a codebook,
+and applies lossless substitution compression.
+
+Usage:
+    python3 dictionary_compress.py <workspace> --build       # Scan + generate codebook
+    python3 dictionary_compress.py <workspace> --compress     # Apply codebook
+    python3 dictionary_compress.py <workspace> --decompress   # Expand codes back
+    python3 dictionary_compress.py <workspace> --stats        # Show compression effect
+
+Part of claw-compactor. License: MIT.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+from typing import Dict, List, Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from lib.dictionary import (
+    build_codebook, compress_text, decompress_text,
+    save_codebook, load_codebook, compression_stats,
+)
+from lib.tokens import estimate_tokens
+from lib.exceptions import FileNotFoundError_, MemCompressError
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_CODEBOOK_PATH = "memory/.codebook.json"
+
+
+def _collect_md_files(workspace: Path) -> List[Path]:
+    """Collect all markdown files in workspace."""
+    files: List[Path] = []
+    for name in ["MEMORY.md", "TOOLS.md", "AGENTS.md", "SOUL.md", "USER.md"]:
+        p = workspace / name
+        if p.exists():
+            files.append(p)
+    mem_dir = workspace / "memory"
+    if mem_dir.is_dir():
+        for f in sorted(mem_dir.glob("*.md")):
+            if not f.name.startswith('.'):
+                files.append(f)
+    return files
+
+
+def _read_texts(files: List[Path]) -> List[str]:
+    """Read all files into a list of strings."""
+    return [f.read_text(encoding="utf-8", errors="replace") for f in files]
+
+
+def cmd_build(
+    workspace: Path,
+    codebook_path: Path,
+    min_freq: int = 3,
+    max_entries: int = 200,
+) -> Dict[str, Any]:
+    """Scan workspace and build codebook."""
+    files = _collect_md_files(workspace)
+    texts = _read_texts(files)
+    cb = build_codebook(texts, min_freq=min_freq, max_entries=max_entries)
+    save_codebook(cb, codebook_path)
+    return {
+        "codebook_entries": len(cb),
+        "codebook_path": str(codebook_path),
+        "files_scanned": len(files),
+    }
+
+
+def cmd_compress(
+    workspace: Path,
+    codebook_path: Path,
+    dry_run: bool = False,
+) -> Dict[str, Any]:
+    """Apply codebook compression to all workspace files."""
+    cb = load_codebook(codebook_path)
+    files = _collect_md_files(workspace)
+    total_before = 0
+    total_after = 0
+
+    for f in files:
+        text = f.read_text(encoding="utf-8", errors="replace")
+        before = estimate_tokens(text)
+        compressed = compress_text(text, cb)
+        after = estimate_tokens(compressed)
+        total_before += before
+        total_after += after
+        if not dry_run:
+            f.write_text(compressed, encoding="utf-8")
+
+    return {
+        "files": len(files),
+        "tokens_before": total_before,
+        "tokens_after": total_after,
+        "tokens_saved": total_before - total_after,
+        "dry_run": dry_run,
+    }
+
+
+def cmd_decompress(
+    workspace: Path,
+    codebook_path: Path,
+    dry_run: bool = False,
+) -> Dict[str, Any]:
+    """Expand codebook codes back to original phrases."""
+    cb = load_codebook(codebook_path)
+    files = _collect_md_files(workspace)
+
+    for f in files:
+        text = f.read_text(encoding="utf-8", errors="replace")
+        decompressed = decompress_text(text, cb)
+        if not dry_run:
+            f.write_text(decompressed, encoding="utf-8")
+
+    return {"files": len(files), "dry_run": dry_run}
+
+
+def cmd_stats(
+    workspace: Path,
+    codebook_path: Path,
+) -> Dict[str, Any]:
+    """Show compression statistics."""
+    cb = load_codebook(codebook_path)
+    files = _collect_md_files(workspace)
+    texts = _read_texts(files)
+    combined = '\n'.join(texts)
+    compressed = compress_text(combined, cb)
+    stats = compression_stats(combined, compressed, cb)
+    stats["files"] = len(files)
+    return stats
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Dictionary-based compression")
+    parser.add_argument("workspace", help="Workspace directory")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--build", action="store_true", help="Build codebook")
+    group.add_argument("--compress", action="store_true", help="Apply compression")
+    group.add_argument("--decompress", action="store_true", help="Expand codes")
+    group.add_argument("--stats", action="store_true", help="Show stats")
+    parser.add_argument("--codebook", default=None, help="Codebook path")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    ws = Path(args.workspace)
+    cb_path = Path(args.codebook) if args.codebook else ws / DEFAULT_CODEBOOK_PATH
+
+    if args.build:
+        result = cmd_build(ws, cb_path)
+    elif args.compress:
+        result = cmd_compress(ws, cb_path, dry_run=args.dry_run)
+    elif args.decompress:
+        result = cmd_decompress(ws, cb_path, dry_run=args.dry_run)
+    else:
+        result = cmd_stats(ws, cb_path)
+
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        for k, v in result.items():
+            print(f"{k}: {v}")
+
+
+if __name__ == "__main__":
+    main()