claude-agent-skills 1.3.0

package/bundled-skills/caveman-compress/scripts/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

package/bundled-skills/caveman-compress/scripts/benchmark.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import sys
+
+# Support both direct execution and module import
+try:
+    from .validate import validate
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).parent))
+    from validate import validate
+
+try:
+    import tiktoken
+    _enc = tiktoken.get_encoding("o200k_base")
+except ImportError:
+    _enc = None
+
+
+def count_tokens(text):
+    if _enc is None:
+        return len(text.split())  # fallback: word count
+    return len(_enc.encode(text))
+
+
+def benchmark_pair(orig_path: Path, comp_path: Path):
+    orig_text = orig_path.read_text()
+    comp_text = comp_path.read_text()
+
+    orig_tokens = count_tokens(orig_text)
+    comp_tokens = count_tokens(comp_text)
+    saved = 100 * (orig_tokens - comp_tokens) / orig_tokens if orig_tokens > 0 else 0.0
+    result = validate(orig_path, comp_path)
+
+    return (comp_path.name, orig_tokens, comp_tokens, saved, result.is_valid)
+
+
+def print_table(rows):
+    print("\n| File | Original | Compressed | Saved % | Valid |")
+    print("|------|----------|------------|---------|-------|")
+    for r in rows:
+        print(f"| {r[0]} | {r[1]} | {r[2]} | {r[3]:.1f}% | {'✅' if r[4] else '❌'} |")
+
+
+def main():
+    # Direct file pair: python3 benchmark.py original.md compressed.md
+    if len(sys.argv) == 3:
+        orig = Path(sys.argv[1]).resolve()
+        comp = Path(sys.argv[2]).resolve()
+        if not orig.exists():
+            print(f"❌ Not found: {orig}")
+            sys.exit(1)
+        if not comp.exists():
+            print(f"❌ Not found: {comp}")
+            sys.exit(1)
+        print_table([benchmark_pair(orig, comp)])
+        return
+
+    # Glob mode: repo_root/tests/caveman-compress/
+    # __file__ lives at <repo_root>/skills/caveman-compress/scripts/benchmark.py
+    # Walk up four dirs: scripts → caveman-compress → skills → repo_root.
+    tests_dir = Path(__file__).resolve().parents[3] / "tests" / "caveman-compress"
+    if not tests_dir.exists():
+        print(f"❌ Tests dir not found: {tests_dir}")
+        sys.exit(1)
+
+    rows = []
+    for orig in sorted(tests_dir.glob("*.original.md")):
+        comp = orig.with_name(orig.stem.removesuffix(".original") + ".md")
+        if comp.exists():
+            rows.append(benchmark_pair(orig, comp))
+
+    if not rows:
+        print("No compressed file pairs found.")
+        return
+
+    print_table(rows)
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/caveman-compress/scripts/cli.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+Caveman Compress CLI
+
+Usage:
+    caveman <filepath>
+"""
+
+import sys
+
+# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
+# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
+# masking the real error and leaving the user with a half-compressed file.
+for _stream in (sys.stdout, sys.stderr):
+    reconfigure = getattr(_stream, "reconfigure", None)
+    if callable(reconfigure):
+        try:
+            reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+
+from pathlib import Path
+
+from .compress import backup_dir_for, compress_file
+from .detect import detect_file_type, should_compress
+
+
+def print_usage():
+    print("Usage: caveman <filepath>")
+
+
+def main():
+    if len(sys.argv) != 2:
+        print_usage()
+        sys.exit(1)
+
+    filepath = Path(sys.argv[1])
+
+    # Check file exists
+    if not filepath.exists():
+        print(f"❌ File not found: {filepath}")
+        sys.exit(1)
+
+    if not filepath.is_file():
+        print(f"❌ Not a file: {filepath}")
+        sys.exit(1)
+
+    filepath = filepath.resolve()
+
+    # Detect file type
+    file_type = detect_file_type(filepath)
+
+    print(f"Detected: {file_type}")
+
+    # Check if compressible
+    if not should_compress(filepath):
+        print("Skipping: file is not natural language (code/config)")
+        sys.exit(0)
+
+    print("Starting caveman compression...\n")
+
+    try:
+        success = compress_file(filepath)
+
+        if success:
+            print("\nCompression completed successfully")
+            backup_path = backup_dir_for(filepath) / (filepath.stem + ".original.md")
+            print(f"Compressed: {filepath}")
+            print(f"Original:   {backup_path}")
+            sys.exit(0)
+        else:
+            print("\n❌ Compression failed after retries")
+            sys.exit(2)
+
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(130)
+
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/bundled-skills/caveman-compress/scripts/compress.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+"""
+Caveman Memory Compression Orchestrator
+
+Usage:
+    python scripts/compress.py <filepath>
+"""
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import List
+
+OUTER_FENCE_REGEX = re.compile(
+    r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL
+)
+
+# YAML frontmatter: starts at file start with --- on its own line, ends with --- on its own line.
+# Captures the entire block (including delimiters and trailing newline) and the body after.
+FRONTMATTER_REGEX = re.compile(
+    r"\A(---\r?\n.*?\r?\n---\r?\n)(.*)", re.DOTALL
+)
+
+
+def split_frontmatter(text: str):
+    """Split YAML frontmatter from body. Returns (frontmatter, body).
+
+    Memory files (and many other markdown docs) start with a YAML frontmatter
+    block delimited by `---` lines. The compression LLM has a habit of stripping
+    or rewriting these despite preserve-structure rules in the prompt — so we
+    surgically remove the frontmatter before compression and prepend it back
+    verbatim to the output. Files without frontmatter pass through unchanged.
+    """
+    m = FRONTMATTER_REGEX.match(text)
+    if m:
+        return m.group(1), m.group(2)
+    return "", text
+
+# Filenames and paths that almost certainly hold secrets or PII. Compressing
+# them ships raw bytes to the Anthropic API — a third-party data boundary that
+# developers on sensitive codebases cannot cross. detect.py already skips .env
+# by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
+# slip through the natural-language filter. This is a hard refuse before read.
+SENSITIVE_BASENAME_REGEX = re.compile(
+    r"(?ix)^("
+    r"\.env(\..+)?"
+    r"|\.netrc"
+    r"|credentials(\..+)?"
+    r"|secrets?(\..+)?"
+    r"|passwords?(\..+)?"
+    r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
+    r"|authorized_keys"
+    r"|known_hosts"
+    r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
+    r")$"
+)
+
+SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
+
+SENSITIVE_NAME_TOKENS = (
+    "secret", "credential", "password", "passwd",
+    "apikey", "accesskey", "token", "privatekey",
+)
+
+
+def backup_dir_for(filepath: Path) -> Path:
+    """Resolve the out-of-tree backup directory for a given source file.
+
+    Backups must live OUTSIDE the source directory so skill auto-loaders
+    (Claude Code rules/, opencode instructions/, etc.) stop re-ingesting the
+    `.original.md` copies as live files. Base dir is platform-aware:
+      - Windows: %LOCALAPPDATA%\\caveman-compress\\backups
+      - else:    $XDG_DATA_HOME/caveman-compress/backups if set,
+                 else ~/.local/share/caveman-compress/backups
+
+    The source file's parent-dir name is mirrored under the base to reduce
+    cross-project collisions (e.g. two `task.md` files in different repos).
+    """
+    if os.name == "nt" or sys.platform == "win32":
+        local_appdata = os.environ.get("LOCALAPPDATA")
+        base = Path(local_appdata) if local_appdata else Path.home() / "AppData" / "Local"
+        base = base / "caveman-compress" / "backups"
+    else:
+        xdg = os.environ.get("XDG_DATA_HOME")
+        base = Path(xdg) if xdg else Path.home() / ".local" / "share"
+        base = base / "caveman-compress" / "backups"
+    return base / filepath.parent.name
+
+
+def is_sensitive_path(filepath: Path) -> bool:
+    """Heuristic denylist for files that must never be shipped to a third-party API."""
+    name = filepath.name
+    if SENSITIVE_BASENAME_REGEX.match(name):
+        return True
+    lowered_parts = {p.lower() for p in filepath.parts}
+    if lowered_parts & SENSITIVE_PATH_COMPONENTS:
+        return True
+    # Normalize separators so "api-key" and "api_key" both match "apikey".
+    lower = re.sub(r"[_\-\s.]", "", name.lower())
+    return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
+
+
+def strip_llm_wrapper(text: str) -> str:
+    """Strip outer ```markdown ... ``` fence when it wraps the entire output."""
+    m = OUTER_FENCE_REGEX.match(text)
+    if m:
+        return m.group(2)
+    return text
+
+from .detect import should_compress
+from .validate import validate
+
+MAX_RETRIES = 2
+
+
+# ---------- Claude Calls ----------
+
+
+def call_claude(prompt: str) -> str:
+    """Send a prompt to Claude.
+
+    Prefers the Anthropic SDK when ANTHROPIC_API_KEY is set; otherwise falls
+    back to the ``claude --print`` CLI (which handles desktop auth).
+
+    On Windows the CLI subprocess decoding defaults to the system codepage
+    (cp1251 / cp1252) and crashes on UTF-8 output — see issue #152. Pinning
+    ``encoding="utf-8"`` with ``errors="replace"`` matches the CLI's actual
+    native I/O and prevents the UnicodeDecodeError before validation can
+    report. Windows users with non-ASCII content can also set
+    ``ANTHROPIC_API_KEY`` to route through the SDK and skip the subprocess.
+    """
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if api_key:
+        try:
+            import anthropic
+
+            client = anthropic.Anthropic(api_key=api_key)
+            msg = client.messages.create(
+                model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
+                max_tokens=8192,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return strip_llm_wrapper(msg.content[0].text.strip())
+        except ImportError:
+            pass  # anthropic not installed, fall back to CLI
+    # Fallback: use claude CLI (handles desktop auth).
+    # Resolve binary via shutil.which so Windows .cmd/.bat shims (e.g.
+    # %APPDATA%\npm\claude.CMD) work without shell=True. On POSIX,
+    # shutil.which returns the same absolute path as the implicit lookup,
+    # so this is a no-op there. Falls back to bare "claude" if not found
+    # on PATH so subprocess raises a clear FileNotFoundError.
+    claude_bin = shutil.which("claude") or "claude"
+    try:
+        result = subprocess.run(
+            [claude_bin, "--print"],
+            input=prompt,
+            text=True,
+            capture_output=True,
+            check=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+        return strip_llm_wrapper(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Claude call failed:\n{e.stderr}")
+
+
+def build_compress_prompt(original: str) -> str:
+    return f"""
+Compress this markdown into caveman format.
+
+STRICT RULES:
+- Do NOT modify anything inside ``` code blocks
+- Do NOT modify anything inside inline backticks
+- Preserve ALL URLs exactly
+- Preserve ALL headings exactly
+- Preserve file paths and commands
+- Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
+
+Only compress natural language.
+
+TEXT:
+{original}
+"""
+
+
+def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str:
+    errors_str = "\n".join(f"- {e}" for e in errors)
+    return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
+
+CRITICAL RULES:
+- DO NOT recompress or rephrase the file
+- ONLY fix the listed errors — leave everything else exactly as-is
+- The ORIGINAL is provided as reference only (to restore missing content)
+- Preserve caveman style in all untouched sections
+
+ERRORS TO FIX:
+{errors_str}
+
+HOW TO FIX:
+- Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
+- Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
+- Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
+- Do not touch any section not mentioned in the errors
+
+ORIGINAL (reference only):
+{original}
+
+COMPRESSED (fix this):
+{compressed}
+
+Return ONLY the fixed compressed file. No explanation.
+"""
+
+
+# ---------- Core Logic ----------
+
+
+def compress_file(filepath: Path) -> bool:
+    # Resolve and validate path
+    filepath = filepath.resolve()
+    MAX_FILE_SIZE = 500_000  # 500KB
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {filepath}")
+    if filepath.stat().st_size > MAX_FILE_SIZE:
+        raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
+
+    # Refuse files that look like they contain secrets or PII. Compressing ships
+    # the raw bytes to the Anthropic API — a third-party boundary — so we fail
+    # loudly rather than silently exfiltrate credentials or keys. Override is
+    # intentional: the user must rename the file if the heuristic is wrong.
+    if is_sensitive_path(filepath):
+        raise ValueError(
+            f"Refusing to compress {filepath}: filename looks sensitive "
+            "(credentials, keys, secrets, or known private paths). "
+            "Compression sends file contents to the Anthropic API. "
+            "Rename the file if this is a false positive."
+        )
+
+    print(f"Processing: {filepath}")
+
+    if not should_compress(filepath):
+        print("Skipping (not natural language)")
+        return False
+
+    original_text = filepath.read_text(errors="ignore")
+    # Store backup outside the source directory so skill auto-loaders don't
+    # re-ingest the `.original.md` copy as a live file. Mirror the source's
+    # parent-dir name + stem under a platform-aware base to reduce collisions.
+    backup_dir = backup_dir_for(filepath)
+    backup_dir.mkdir(parents=True, exist_ok=True)
+    backup_path = backup_dir / (filepath.stem + ".original.md")
+
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
+    # Check if backup already exists to prevent accidental overwriting
+    if backup_path.exists():
+        print(f"⚠️ Backup file already exists: {backup_path}")
+        print("The original backup may contain important content.")
+        print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
+        return False
+
+    # Split YAML frontmatter off before compression. Claude tends to strip or
+    # rewrite frontmatter despite preserve-structure rules; we keep it verbatim
+    # by removing it from the input and re-prepending it to the output.
+    frontmatter, body = split_frontmatter(original_text)
+    if frontmatter:
+        print(f"Detected YAML frontmatter ({len(frontmatter)} chars) — preserving verbatim")
+
+    if not body.strip():
+        print("❌ Refusing to compress: body is empty after frontmatter removal.")
+        return False
+
+    # Step 1: Compress (body only, frontmatter excluded)
+    print("Compressing with Claude...")
+    compressed_body = call_claude(build_compress_prompt(body))
+
+    if compressed_body is None or not compressed_body.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    # Compare the BODY (not the whole file) — frontmatter is preserved verbatim
+    # and would never change, so identity must be judged on the compressible part.
+    if compressed_body.strip() == body.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Reassemble: frontmatter (verbatim) + compressed body
+    compressed = frontmatter + compressed_body
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
+    backup_path.write_text(original_text)
+    backup_readback = backup_path.read_text(errors="ignore")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
+    filepath.write_text(compressed)
+
+    # Step 2: Validate + Retry
+    for attempt in range(MAX_RETRIES):
+        print(f"\nValidation attempt {attempt + 1}")
+
+        result = validate(backup_path, filepath)
+
+        if result.is_valid:
+            print("Validation passed")
+            break
+
+        print("❌ Validation failed:")
+        for err in result.errors:
+            print(f"   - {err}")
+
+        if attempt == MAX_RETRIES - 1:
+            # Restore original on failure
+            filepath.write_text(original_text)
+            backup_path.unlink(missing_ok=True)
+            print("❌ Failed after retries — original restored")
+            return False
+
+        print("Fixing with Claude...")
+        compressed = call_claude(
+            build_fix_prompt(original_text, compressed, result.errors)
+        )
+        filepath.write_text(compressed)
+
+    return True

package/bundled-skills/caveman-compress/scripts/detect.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Detect whether a file is natural language (compressible) or code/config (skip)."""
+
+import json
+import re
+from pathlib import Path
+
+# Extensions that are natural language and compressible
+COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
+
+# Extensions that are code/config and should be skipped
+SKIP_EXTENSIONS = {
+    ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
+    ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
+    ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
+    ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
+    ".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
+}
+
+# Patterns that indicate a line is code
+CODE_PATTERNS = [
+    re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
+    re.compile(r"^\s*(def |class |function |async function |export )"),
+    re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
+    re.compile(r"^\s*[\}\]\);]+\s*$"),  # closing braces/brackets
+    re.compile(r"^\s*@\w+"),  # decorators/annotations
+    re.compile(r'^\s*"[^"]+"\s*:\s*'),  # JSON-like key-value
+    re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"),  # assignment with literal
+]
+
+
+def _is_code_line(line: str) -> bool:
+    """Check if a line looks like code."""
+    return any(p.match(line) for p in CODE_PATTERNS)
+
+
+def _is_json_content(text: str) -> bool:
+    """Check if content is valid JSON."""
+    try:
+        json.loads(text)
+        return True
+    except (json.JSONDecodeError, ValueError):
+        return False
+
+
+def _is_yaml_content(lines: list[str]) -> bool:
+    """Heuristic: check if content looks like YAML."""
+    yaml_indicators = 0
+    for line in lines[:30]:
+        stripped = line.strip()
+        if stripped.startswith("---"):
+            yaml_indicators += 1
+        elif re.match(r"^\w[\w\s]*:\s", stripped):
+            yaml_indicators += 1
+        elif stripped.startswith("- ") and ":" in stripped:
+            yaml_indicators += 1
+    # If most non-empty lines look like YAML
+    non_empty = sum(1 for l in lines[:30] if l.strip())
+    return non_empty > 0 and yaml_indicators / non_empty > 0.6
+
+
+def detect_file_type(filepath: Path) -> str:
+    """Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
+
+    Returns:
+        One of: 'natural_language', 'code', 'config', 'unknown'
+    """
+    ext = filepath.suffix.lower()
+
+    # Extension-based classification
+    if ext in COMPRESSIBLE_EXTENSIONS:
+        return "natural_language"
+    if ext in SKIP_EXTENSIONS:
+        return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
+
+    # Extensionless files (like CLAUDE.md, TODO) — check content
+    if not ext:
+        try:
+            text = filepath.read_text(errors="ignore")
+        except (OSError, PermissionError):
+            return "unknown"
+
+        lines = text.splitlines()[:50]
+
+        if _is_json_content(text[:10000]):
+            return "config"
+        if _is_yaml_content(lines):
+            return "config"
+
+        code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
+        non_empty = sum(1 for l in lines if l.strip())
+        if non_empty > 0 and code_lines / non_empty > 0.4:
+            return "code"
+
+        return "natural_language"
+
+    return "unknown"
+
+
+def should_compress(filepath: Path) -> bool:
+    """Return True if the file is natural language and should be compressed."""
+    if not filepath.is_file():
+        return False
+    # Skip backup files
+    if filepath.name.endswith(".original.md"):
+        return False
+    return detect_file_type(filepath) == "natural_language"
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python detect.py <file1> [file2] ...")
+        sys.exit(1)
+
+    for path_str in sys.argv[1:]:
+        p = Path(path_str).resolve()
+        file_type = detect_file_type(p)
+        compress = should_compress(p)
+        print(f"  {p.name:30s} type={file_type:20s} compress={compress}")