@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/scripts/lib/markdown.py

@@ -0,0 +1,312 @@
+"""Markdown parsing and manipulation utilities.
+
+Part of claw-compactor. License: MIT.
+"""
+
+import re
+import logging
+from difflib import SequenceMatcher
+from typing import List, Tuple, Dict, Optional
+
+logger = logging.getLogger(__name__)
+
+# Chinese punctuation -> English equivalents (saves tokens)
+_ZH_PUNCT_MAP: Dict[str, str] = {
+    '\uFF0C': ',', '\u3002': '.', '\uFF1B': ';', '\uFF1A': ':', '\uFF01': '!', '\uFF1F': '?',
+    '\u201C': '"', '\u201D': '"', '\u2018': "'", '\u2019': "'",
+    '\uFF08': '(', '\uFF09': ')', '\u3010': '[', '\u3011': ']',
+    '\u3001': ',', '\u2026': '...', '\u2014\u2014': '--', '\uFF5E': '~',
+}
+_ZH_PUNCT_RE = re.compile('|'.join(re.escape(k) for k in _ZH_PUNCT_MAP))
+
+# Emoji pattern (broad: emoticons, symbols, pictographs, etc.)
+_EMOJI_RE = re.compile(
+    '[\U0001F600-\U0001F64F'   # emoticons
+    '\U0001F300-\U0001F5FF'    # symbols & pictographs
+    '\U0001F680-\U0001F6FF'    # transport & map
+    '\U0001F1E0-\U0001F1FF'    # flags
+    '\U00002702-\U000027B0'    # dingbats
+    '\U0001F900-\U0001F9FF'    # supplemental symbols
+    '\U0001FA00-\U0001FA6F'    # chess symbols
+    '\U0001FA70-\U0001FAFF'    # symbols extended-A
+    '\U00002600-\U000026FF'    # misc symbols
+    ']+', re.UNICODE
+)
+
+# Header regex
+_HEADER_RE = re.compile(r'^(#{1,6})\s+(.*)', re.MULTILINE)
+
+# Table separator line
+_TABLE_SEP_RE = re.compile(r'^[\s|:\-]+$')
+
+
+def parse_sections(text: str) -> List[Tuple[str, str, int]]:
+    """Parse *text* into sections delimited by markdown headers.
+
+    Returns a list of (header, body, level) tuples.
+    A preamble (text before the first header) is returned with header=''.
+    """
+    if not text:
+        return []
+
+    sections: List[Tuple[str, str, int]] = []
+    lines = text.split('\n')
+    current_header = ''
+    current_level = 0
+    current_body_lines: List[str] = []
+
+    for line in lines:
+        m = _HEADER_RE.match(line)
+        if m:
+            # Save previous section
+            body = '\n'.join(current_body_lines).strip()
+            if current_header or body:
+                sections.append((current_header, body, current_level))
+            current_header = m.group(2).strip()
+            current_level = len(m.group(1))
+            current_body_lines = []
+        else:
+            current_body_lines.append(line)
+
+    # Last section
+    body = '\n'.join(current_body_lines).strip()
+    if current_header or body:
+        sections.append((current_header, body, current_level))
+
+    return sections
+
+
+def strip_markdown_redundancy(text: str) -> str:
+    """Remove excessive blank lines and trailing whitespace."""
+    if not text:
+        return ""
+    # Collapse 3+ consecutive blank lines to 2
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    # Strip trailing whitespace per line
+    lines = [line.rstrip() for line in text.split('\n')]
+    return '\n'.join(lines).strip()
+
+
+def remove_duplicate_lines(text: str) -> str:
+    """Remove exact duplicate non-blank lines, preserving order."""
+    if not text:
+        return ""
+    seen = set()
+    result = []
+    for line in text.split('\n'):
+        stripped = line.strip()
+        if not stripped:
+            # Preserve blank lines
+            result.append(line)
+            continue
+        if stripped in seen:
+            continue
+        seen.add(stripped)
+        result.append(line)
+    return '\n'.join(result)
+
+
+def normalize_chinese_punctuation(text: str) -> str:
+    """Replace Chinese fullwidth punctuation with ASCII equivalents."""
+    if not text:
+        return ""
+    # Handle the double-char em-dash first
+    text = text.replace('\u2014\u2014', '--')
+    return _ZH_PUNCT_RE.sub(lambda m: _ZH_PUNCT_MAP.get(m.group(), m.group()), text)
+
+
+def strip_emoji(text: str) -> str:
+    """Remove emoji characters from *text*."""
+    if not text:
+        return ""
+    result = _EMOJI_RE.sub('', text)
+    # Collapse multiple spaces left by emoji removal
+    result = re.sub(r'  +', ' ', result)
+    return result
+
+
+def remove_empty_sections(text: str) -> str:
+    """Remove markdown sections that have no meaningful body content."""
+    if not text:
+        return ""
+    sections = parse_sections(text)
+    if not sections:
+        return text
+
+    # Determine which sections have children (a deeper section follows)
+    has_child = [False] * len(sections)
+    for idx, (header, body, level) in enumerate(sections):
+        if level > 0:
+            # Look backwards for a parent
+            for pidx in range(idx - 1, -1, -1):
+                _, _, plevel = sections[pidx]
+                if plevel > 0 and plevel < level:
+                    has_child[pidx] = True
+                    break
+
+    result_lines: List[str] = []
+    for idx, (header, body, level) in enumerate(sections):
+        if not header and not body:
+            continue
+        if header and not body.strip() and not has_child[idx]:
+            continue  # Empty section with no children
+        if header:
+            result_lines.append('#' * level + ' ' + header)
+        if body.strip():
+            result_lines.append(body)
+        result_lines.append('')  # Blank line between sections
+
+    return '\n'.join(result_lines).strip()
+
+
+def compress_markdown_table(text: str) -> str:
+    """Convert markdown tables to compact key:value notation.
+
+    A 2-column table becomes ``Key: Value`` lines.
+    Multi-column tables become ``Col1 | Col2 | ...`` lines (no header row / separator).
+    """
+    if not text:
+        return ""
+
+    lines = text.split('\n')
+    result: List[str] = []
+    i = 0
+
+    while i < len(lines):
+        line = lines[i]
+        # Detect a table: line with | ... | followed by separator |---|
+        if '|' in line and i + 1 < len(lines) and _TABLE_SEP_RE.match(lines[i + 1].strip()):
+            # Parse header row
+            headers = [c.strip() for c in line.strip().strip('|').split('|')]
+            i += 2  # skip header + separator
+            rows: List[List[str]] = []
+            while i < len(lines) and '|' in lines[i] and lines[i].strip():
+                cells = [c.strip() for c in lines[i].strip().strip('|').split('|')]
+                rows.append(cells)
+                i += 1
+
+            if len(headers) >= 5:
+                # Wide tables: preserve as-is but without header/separator
+                for row in rows:
+                    result.append('| ' + ' | '.join(row) + ' |')
+            elif len(headers) == 2:
+                # 2-column: key: value format
+                for row in rows:
+                    k = row[0] if len(row) > 0 else ''
+                    v = row[1] if len(row) > 1 else ''
+                    if k or v:
+                        result.append(f"- {k}: {v}")
+            else:
+                # Multi-column: compact format using headers as labels
+                for row in rows:
+                    parts = []
+                    for ci, cell in enumerate(row):
+                        if ci == 0:
+                            parts.append(cell)
+                        elif ci < len(headers):
+                            parts.append(f"{headers[ci]}={cell}")
+                        else:
+                            parts.append(cell)
+                    result.append(', '.join(parts))
+        else:
+            result.append(line)
+            i += 1
+
+    return '\n'.join(result)
+
+
+def merge_similar_bullets(text: str, threshold: float = 0.80) -> str:
+    """Merge bullet lines with high similarity.
+
+    Uses SequenceMatcher ratio. When two bullets exceed *threshold*,
+    keep the longer one.
+    """
+    if not text:
+        return ""
+
+    lines = text.split('\n')
+    bullet_re = re.compile(r'^(\s*[-*+]\s+)(.*)')
+    result: List[str] = []
+    bullets: List[Tuple[str, str, str]] = []  # (prefix, content, full_line)
+
+    def flush_bullets():
+        if not bullets:
+            return
+        kept = list(bullets)
+        merged_out: List[bool] = [False] * len(kept)
+        for i in range(len(kept)):
+            if merged_out[i]:
+                continue
+            for j in range(i + 1, len(kept)):
+                if merged_out[j]:
+                    continue
+                ratio = SequenceMatcher(None, kept[i][1], kept[j][1]).ratio()
+                if ratio >= threshold:
+                    # Keep the longer one
+                    if len(kept[j][1]) > len(kept[i][1]):
+                        merged_out[i] = True
+                        break
+                    else:
+                        merged_out[j] = True
+        for idx, (prefix, content, full_line) in enumerate(kept):
+            if not merged_out[idx]:
+                result.append(full_line)
+        bullets.clear()
+
+    for line in lines:
+        m = bullet_re.match(line)
+        if m:
+            bullets.append((m.group(1), m.group(2), line))
+        else:
+            flush_bullets()
+            result.append(line)
+
+    flush_bullets()
+    return '\n'.join(result)
+
+
+def merge_short_bullets(text: str, max_words: int = 3, max_merge: int = 10) -> str:
+    """Combine consecutive short bullet points into comma-separated form.
+
+    Bullets with <= *max_words* words are candidates. Up to *max_merge*
+    consecutive short bullets are joined into one line.
+    """
+    if not text:
+        return ""
+
+    bullet_re = re.compile(r'^(\s*[-*+]\s+)(.*)')
+    lines = text.split('\n')
+    result: List[str] = []
+    short_bullets: List[str] = []
+    prefix = '- '
+
+    def flush_short():
+        nonlocal prefix
+        if not short_bullets:
+            return
+        if len(short_bullets) <= 2:
+            for sb in short_bullets:
+                result.append(prefix + sb)
+        else:
+            # Merge into one line
+            result.append(prefix + ', '.join(short_bullets))
+        short_bullets.clear()
+
+    for line in lines:
+        m = bullet_re.match(line)
+        if m:
+            content = m.group(2).strip()
+            prefix = m.group(1)
+            if len(content.split()) <= max_words:
+                short_bullets.append(content)
+                if len(short_bullets) >= max_merge:
+                    flush_short()
+            else:
+                flush_short()
+                result.append(line)
+        else:
+            flush_short()
+            result.append(line)
+
+    flush_short()
+    return '\n'.join(result)

package/skills/claw-compactor/scripts/lib/rle.py

@@ -0,0 +1,165 @@
+"""Run-Length Encoding for structured data patterns.
+
+Detects and compresses structured repetitive patterns:
+- IP address families → common prefix extraction
+- File paths → $WS/ shorthand
+- Enumeration lists → compact format
+- Repeated section headers
+
+Part of claw-compactor. License: MIT.
+"""
+
+import re
+import logging
+from collections import Counter
+from typing import Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+# Default workspace path to shorten
+DEFAULT_WS_PATHS = [
+    "/home/user/workspace",
+]
+
+# IP pattern
+_IP_RE = re.compile(r'\b(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\b')
+
+
+def compress_paths(text: str, workspace_paths: Optional[List[str]] = None) -> str:
+    """Replace long workspace paths with $WS shorthand."""
+    if not text:
+        return ""
+    paths = workspace_paths or DEFAULT_WS_PATHS
+    result = text
+    for ws in sorted(paths, key=len, reverse=True):
+        result = result.replace(ws, "$WS")
+    return result
+
+
+def decompress_paths(text: str, workspace_path: str) -> str:
+    """Expand $WS back to the full workspace path."""
+    if not text:
+        return ""
+    return text.replace("$WS", workspace_path)
+
+
+def compress_ip_families(text: str, min_occurrences: int = 2) -> Tuple[str, Dict[str, str]]:
+    """Group IPs by common prefix and compress families.
+
+    Returns (compressed_text, prefix_map) where prefix_map maps
+    $IPn labels to the common prefix.
+    Only compresses families with min_occurrences+ IPs sharing a 3-octet prefix.
+    """
+    if not text:
+        return "", {}
+
+    ips = _IP_RE.findall(text)
+    if not ips:
+        return text, {}
+
+    # Group by first 3 octets
+    families: Dict[str, List[str]] = {}
+    for ip in ips:
+        parts = ip.split('.')
+        prefix = '.'.join(parts[:3]) + '.'
+        families.setdefault(prefix, []).append(ip)
+
+    # Only compress families with min_occurrences+ members
+    prefix_map: Dict[str, str] = {}
+    result = text
+    idx = 0
+    for prefix, members in sorted(families.items(), key=lambda x: -len(x[1])):
+        if len(members) < min_occurrences:
+            continue
+        label = f"$IP{idx}" if idx > 0 else "$IP"
+        prefix_map[label] = prefix
+        for ip in set(members):
+            parts = ip.split('.')
+            suffix = parts[3]
+            result = result.replace(ip, f"{label}.{suffix}")
+        idx += 1
+
+    return result, prefix_map
+
+
+def decompress_ip_families(text: str, prefix_map: Dict[str, str]) -> str:
+    """Expand compressed IP references back to full IPs."""
+    if not text or not prefix_map:
+        return text
+    result = text
+    for label, prefix in prefix_map.items():
+        # Match $IPn.suffix patterns
+        pattern = re.compile(re.escape(label) + r'\.(\d{1,3})')
+        result = pattern.sub(lambda m: prefix + m.group(1), result)
+    return result
+
+
+def compress_enumerations(text: str) -> str:
+    """Compress comma-separated lists of ALL-CAPS short codes.
+
+    Only compresses lists with 4+ items that are all uppercase short tokens.
+    E.g. "BTC, ETH, SOL, BNB, DOGE" → "[BTC,ETH,SOL,BNB,DOGE]"
+    """
+    if not text:
+        return ""
+
+    # Match comma-separated uppercase tokens
+    pattern = re.compile(r'((?:[A-Z][A-Z0-9]{1,6})(?:\s*,\s*(?:[A-Z][A-Z0-9]{1,6})){3,})')
+
+    def _compact(m: re.Match) -> str:
+        items = [s.strip() for s in m.group(0).split(',')]
+        return '[' + ','.join(items) + ']'
+
+    return pattern.sub(_compact, text)
+
+
+def compress_repeated_headers(text: str) -> str:
+    """Compress repeated identical section headers.
+
+    When the same header text appears multiple times, keep only the first
+    and merge contents.
+    """
+    if not text:
+        return ""
+    lines = text.split('\n')
+    seen_headers: Dict[str, int] = {}
+    result: List[str] = []
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        # Check if this is a header
+        if line.startswith('#'):
+            header_text = line.lstrip('#').strip()
+            if header_text in seen_headers:
+                # Skip this header, but keep its body content
+                i += 1
+                while i < len(lines) and not lines[i].startswith('#'):
+                    if lines[i].strip():
+                        result.append(lines[i])
+                    i += 1
+                continue
+            else:
+                seen_headers[header_text] = len(result)
+        result.append(line)
+        i += 1
+    return '\n'.join(result)
+
+
+def compress(text: str, workspace_paths: Optional[List[str]] = None) -> str:
+    """Apply all RLE-style compressions to *text*."""
+    if not text:
+        return ""
+    result = compress_paths(text, workspace_paths)
+    result, _ = compress_ip_families(result)
+    result = compress_enumerations(result)
+    return result
+
+
+def decompress(text: str, workspace_path: str, ip_prefix_map: Optional[Dict[str, str]] = None) -> str:
+    """Reverse all RLE-style compressions."""
+    if not text:
+        return ""
+    result = decompress_paths(text, workspace_path)
+    if ip_prefix_map:
+        result = decompress_ip_families(result, ip_prefix_map)
+    return result

package/skills/claw-compactor/scripts/lib/tokenizer_optimizer.py

@@ -0,0 +1,188 @@
+"""Token-level format optimization.
+
+Applies encoding-aware transformations that reduce token count while
+preserving all semantic information. Each transformation targets
+specific tokenizer inefficiencies in cl100k_base / o200k_base.
+
+Key insight: the same information can be encoded in fewer tokens
+by choosing formats the tokenizer handles more efficiently.
+
+Part of claw-compactor. License: MIT.
+"""
+
+import re
+import logging
+from typing import List, Tuple
+
+logger = logging.getLogger(__name__)
+
+# Chinese full-width punctuation → half-width (each saves ~1 token)
+_ZH_PUNCT_MAP = {
+    '，': ',', '。': '.', '；': ';', '：': ':', '！': '!', '？': '?',
+    '"': '"', '"': '"', ''': "'", ''': "'",
+    '（': '(', '）': ')', '【': '[', '】': ']',
+    '、': ',', '…': '...', '——': '--', '～': '~',
+}
+_ZH_PUNCT_RE = re.compile('|'.join(re.escape(k) for k in _ZH_PUNCT_MAP))
+
+# Bold/italic markdown decorators
+_BOLD_RE = re.compile(r'\*\*(.+?)\*\*')
+_ITALIC_RE = re.compile(r'(?<!\*)\*([^*]+?)\*(?!\*)')
+
+# Inline code that's just a plain word (not actual code)
+_TRIVIAL_CODE_RE = re.compile(r'`([a-zA-Z0-9_.-]+)`')
+
+# Markdown table detection
+_TABLE_SEP_RE = re.compile(r'^[\s|:\-]+$')
+
+# Bullet patterns
+_BULLET_RE = re.compile(r'^(\s*)([-*+])\s+', re.MULTILINE)
+
+# Multiple spaces / excessive indentation
+_MULTI_SPACE_RE = re.compile(r'  +')
+_LEADING_SPACES_RE = re.compile(r'^( {4,})', re.MULTILINE)
+
+
+def strip_bold_italic(text: str) -> str:
+    """Remove **bold** and *italic* markdown decorators."""
+    if not text:
+        return ""
+    text = _BOLD_RE.sub(r'\1', text)
+    text = _ITALIC_RE.sub(r'\1', text)
+    return text
+
+
+def normalize_punctuation(text: str) -> str:
+    """Replace Chinese fullwidth punctuation with ASCII equivalents."""
+    if not text:
+        return ""
+    text = text.replace('——', '--')
+    return _ZH_PUNCT_RE.sub(lambda m: _ZH_PUNCT_MAP.get(m.group(), m.group()), text)
+
+
+def strip_trivial_backticks(text: str) -> str:
+    """Remove backticks around simple words (not real code).
+
+    Keeps backticks when content contains spaces or special chars.
+    """
+    if not text:
+        return ""
+    return _TRIVIAL_CODE_RE.sub(r'\1', text)
+
+
+def minimize_whitespace(text: str) -> str:
+    """Reduce multiple spaces and excessive indentation."""
+    if not text:
+        return ""
+    # Reduce multiple spaces to single
+    text = _MULTI_SPACE_RE.sub(' ', text)
+    # Cap leading indentation at 4 spaces
+    text = _LEADING_SPACES_RE.sub('    ', text)
+    # Collapse 3+ consecutive newlines to 2
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    return text
+
+
+def compact_bullets(text: str) -> str:
+    """Remove bullet prefixes from long consecutive bullet lists (3+).
+
+    Short lists (1-2 items) keep their bullets.
+    """
+    if not text:
+        return ""
+    lines = text.split('\n')
+    result: List[str] = []
+    bullet_run: List[str] = []
+    bullet_re = re.compile(r'^(\s*[-*+])\s+(.*)')
+
+    def flush():
+        if len(bullet_run) >= 3:
+            # Strip bullet prefix
+            for content in bullet_run:
+                result.append(content)
+        else:
+            # Keep original bullets
+            for content in bullet_run:
+                result.append('- ' + content)
+        bullet_run.clear()
+
+    for line in lines:
+        m = bullet_re.match(line)
+        if m:
+            bullet_run.append(m.group(2))
+        else:
+            flush()
+            result.append(line)
+    flush()
+    return '\n'.join(result)
+
+
+def compress_table_to_kv(text: str) -> str:
+    """Convert markdown tables to compact key:value or compact format."""
+    if not text:
+        return ""
+
+    lines = text.split('\n')
+    result: List[str] = []
+    i = 0
+
+    while i < len(lines):
+        line = lines[i]
+        if '|' in line and i + 1 < len(lines) and _TABLE_SEP_RE.match(lines[i + 1].strip()):
+            headers = [c.strip() for c in line.strip().strip('|').split('|')]
+            i += 2
+            rows: List[List[str]] = []
+            while i < len(lines) and '|' in lines[i] and lines[i].strip():
+                cells = [c.strip() for c in lines[i].strip().strip('|').split('|')]
+                rows.append(cells)
+                i += 1
+
+            if len(headers) == 2:
+                for row in rows:
+                    k = row[0] if len(row) > 0 else ''
+                    v = row[1] if len(row) > 1 else ''
+                    if k or v:
+                        result.append(f"{k}: {v}")
+            else:
+                for row in rows:
+                    result.append(' | '.join(row))
+        else:
+            result.append(line)
+            i += 1
+
+    return '\n'.join(result)
+
+
+def optimize_tokens(text: str, aggressive: bool = False) -> str:
+    """Apply all token-saving optimizations.
+
+    Args:
+        text: Input text.
+        aggressive: If True, apply more aggressive transformations
+                    (strip bold/italic, compact bullets, strip backticks).
+    """
+    if not text:
+        return ""
+    result = normalize_punctuation(text)
+    result = compress_table_to_kv(result)
+    result = minimize_whitespace(result)
+    if aggressive:
+        result = strip_bold_italic(result)
+        result = strip_trivial_backticks(result)
+        result = compact_bullets(result)
+    return result
+
+
+def estimate_savings(original: str, optimized: str) -> dict:
+    """Calculate token savings between original and optimized text."""
+    from lib.tokens import estimate_tokens
+    orig_tokens = estimate_tokens(original)
+    opt_tokens = estimate_tokens(optimized)
+    reduction = ((orig_tokens - opt_tokens) / orig_tokens * 100) if orig_tokens else 0.0
+    return {
+        "original_tokens": orig_tokens,
+        "optimized_tokens": opt_tokens,
+        "original_chars": len(original),
+        "optimized_chars": len(optimized),
+        "token_reduction_pct": round(reduction, 2),
+    }