@kulapard/pi-caveman 0.1.0 → 0.3.0

package/skills/caveman-compress/scripts/compress.py

@@ -1,341 +0,0 @@
-#!/usr/bin/env python3
-"""
-Caveman Memory Compression Orchestrator
-
-Usage:
-    python scripts/compress.py <filepath>
-"""
-
-import os
-import re
-import shutil
-import subprocess
-import sys
-from pathlib import Path
-
-OUTER_FENCE_REGEX = re.compile(
-    r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL
-)
-
-# YAML frontmatter: starts at file start with --- on its own line, ends with --- on its own line.
-# Captures the entire block (including delimiters and trailing newline) and the body after.
-FRONTMATTER_REGEX = re.compile(
-    r"\A(---\r?\n.*?\r?\n---\r?\n)(.*)", re.DOTALL
-)
-
-
-def split_frontmatter(text: str):
-    """Split YAML frontmatter from body. Returns (frontmatter, body).
-
-    Memory files (and many other markdown docs) start with a YAML frontmatter
-    block delimited by `---` lines. The compression LLM has a habit of stripping
-    or rewriting these despite preserve-structure rules in the prompt — so we
-    surgically remove the frontmatter before compression and prepend it back
-    verbatim to the output. Files without frontmatter pass through unchanged.
-    """
-    m = FRONTMATTER_REGEX.match(text)
-    if m:
-        return m.group(1), m.group(2)
-    return "", text
-
-# Filenames and paths that almost certainly hold secrets or PII. Compressing
-# them ships raw bytes to the Anthropic API — a third-party data boundary that
-# developers on sensitive codebases cannot cross. detect.py already skips .env
-# by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
-# slip through the natural-language filter. This is a hard refuse before read.
-SENSITIVE_BASENAME_REGEX = re.compile(
-    r"(?ix)^("
-    r"\.env(\..+)?"
-    r"|\.netrc"
-    r"|credentials(\..+)?"
-    r"|secrets?(\..+)?"
-    r"|passwords?(\..+)?"
-    r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
-    r"|authorized_keys"
-    r"|known_hosts"
-    r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
-    r")$"
-)
-
-SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
-
-SENSITIVE_NAME_TOKENS = (
-    "secret", "credential", "password", "passwd",
-    "apikey", "accesskey", "token", "privatekey",
-)
-
-
-def backup_dir_for(filepath: Path) -> Path:
-    """Resolve the out-of-tree backup directory for a given source file.
-
-    Backups must live OUTSIDE the source directory so skill auto-loaders
-    (Claude Code rules/, opencode instructions/, etc.) stop re-ingesting the
-    `.original.md` copies as live files. Base dir is platform-aware:
-      - Windows: %LOCALAPPDATA%\\caveman-compress\\backups
-      - else:    $XDG_DATA_HOME/caveman-compress/backups if set,
-                 else ~/.local/share/caveman-compress/backups
-
-    The source file's parent-dir name is mirrored under the base to reduce
-    cross-project collisions (e.g. two `task.md` files in different repos).
-    """
-    if os.name == "nt" or sys.platform == "win32":
-        local_appdata = os.environ.get("LOCALAPPDATA")
-        base = Path(local_appdata) if local_appdata else Path.home() / "AppData" / "Local"
-        base = base / "caveman-compress" / "backups"
-    else:
-        xdg = os.environ.get("XDG_DATA_HOME")
-        base = Path(xdg) if xdg else Path.home() / ".local" / "share"
-        base = base / "caveman-compress" / "backups"
-    return base / filepath.parent.name
-
-
-def is_sensitive_path(filepath: Path) -> bool:
-    """Heuristic denylist for files that must never be shipped to a third-party API."""
-    name = filepath.name
-    if SENSITIVE_BASENAME_REGEX.match(name):
-        return True
-    lowered_parts = {p.lower() for p in filepath.parts}
-    if lowered_parts & SENSITIVE_PATH_COMPONENTS:
-        return True
-    # Normalize separators so "api-key" and "api_key" both match "apikey".
-    lower = re.sub(r"[_\-\s.]", "", name.lower())
-    return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
-
-
-def strip_llm_wrapper(text: str) -> str:
-    """Strip outer ```markdown ... ``` fence when it wraps the entire output."""
-    m = OUTER_FENCE_REGEX.match(text)
-    if m:
-        return m.group(2)
-    return text
-
-from .detect import should_compress
-from .validate import validate
-
-MAX_RETRIES = 2
-
-
-# ---------- Claude Calls ----------
-
-
-def call_claude(prompt: str) -> str:
-    """Send a prompt to Claude.
-
-    Prefers the Anthropic SDK when ANTHROPIC_API_KEY is set; otherwise falls
-    back to the ``claude --print`` CLI (which handles desktop auth).
-
-    On Windows the CLI subprocess decoding defaults to the system codepage
-    (cp1251 / cp1252) and crashes on UTF-8 output — see issue #152. Pinning
-    ``encoding="utf-8"`` with ``errors="replace"`` matches the CLI's actual
-    native I/O and prevents the UnicodeDecodeError before validation can
-    report. Windows users with non-ASCII content can also set
-    ``ANTHROPIC_API_KEY`` to route through the SDK and skip the subprocess.
-    """
-    api_key = os.environ.get("ANTHROPIC_API_KEY")
-    if api_key:
-        try:
-            import anthropic
-
-            client = anthropic.Anthropic(api_key=api_key)
-            msg = client.messages.create(
-                model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
-                max_tokens=8192,
-                messages=[{"role": "user", "content": prompt}],
-            )
-            return strip_llm_wrapper(msg.content[0].text.strip())
-        except ImportError:
-            pass  # anthropic not installed, fall back to CLI
-    # Fallback: use claude CLI (handles desktop auth).
-    # Resolve binary via shutil.which so Windows .cmd/.bat shims (e.g.
-    # %APPDATA%\npm\claude.CMD) work without shell=True. On POSIX,
-    # shutil.which returns the same absolute path as the implicit lookup,
-    # so this is a no-op there. Falls back to bare "claude" if not found
-    # on PATH so subprocess raises a clear FileNotFoundError.
-    claude_bin = shutil.which("claude") or "claude"
-    try:
-        result = subprocess.run(
-            [claude_bin, "--print"],
-            input=prompt,
-            text=True,
-            capture_output=True,
-            check=True,
-            encoding="utf-8",
-            errors="replace",
-        )
-        return strip_llm_wrapper(result.stdout.strip())
-    except subprocess.CalledProcessError as e:
-        raise RuntimeError(f"Claude call failed:\n{e.stderr}")
-
-
-def build_compress_prompt(original: str) -> str:
-    return f"""
-Compress this markdown into caveman format.
-
-STRICT RULES:
-- Do NOT modify anything inside ``` code blocks
-- Do NOT modify anything inside inline backticks
-- Preserve ALL URLs exactly
-- Preserve ALL headings exactly
-- Preserve file paths and commands
-- Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
-
-Only compress natural language.
-
-TEXT:
-{original}
-"""
-
-
-def build_fix_prompt(original: str, compressed: str, errors: list[str]) -> str:
-    errors_str = "\n".join(f"- {e}" for e in errors)
-    return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
-
-CRITICAL RULES:
-- DO NOT recompress or rephrase the file
-- ONLY fix the listed errors — leave everything else exactly as-is
-- The ORIGINAL is provided as reference only (to restore missing content)
-- Preserve caveman style in all untouched sections
-
-ERRORS TO FIX:
-{errors_str}
-
-HOW TO FIX:
-- Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
-- Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
-- Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
-- Do not touch any section not mentioned in the errors
-
-ORIGINAL (reference only):
-{original}
-
-COMPRESSED (fix this):
-{compressed}
-
-Return ONLY the fixed compressed file. No explanation.
-"""
-
-
-# ---------- Core Logic ----------
-
-
-def compress_file(filepath: Path) -> bool:
-    # Resolve and validate path
-    filepath = filepath.resolve()
-    MAX_FILE_SIZE = 500_000  # 500KB
-    if not filepath.exists():
-        raise FileNotFoundError(f"File not found: {filepath}")
-    if filepath.stat().st_size > MAX_FILE_SIZE:
-        raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
-
-    # Refuse files that look like they contain secrets or PII. Compressing ships
-    # the raw bytes to the Anthropic API — a third-party boundary — so we fail
-    # loudly rather than silently exfiltrate credentials or keys. Override is
-    # intentional: the user must rename the file if the heuristic is wrong.
-    if is_sensitive_path(filepath):
-        raise ValueError(
-            f"Refusing to compress {filepath}: filename looks sensitive "
-            "(credentials, keys, secrets, or known private paths). "
-            "Compression sends file contents to the Anthropic API. "
-            "Rename the file if this is a false positive."
-        )
-
-    print(f"Processing: {filepath}")
-
-    if not should_compress(filepath):
-        print("Skipping (not natural language)")
-        return False
-
-    original_text = filepath.read_text(errors="ignore")
-    # Store backup outside the source directory so skill auto-loaders don't
-    # re-ingest the `.original.md` copy as a live file. Mirror the source's
-    # parent-dir name + stem under a platform-aware base to reduce collisions.
-    backup_dir = backup_dir_for(filepath)
-    backup_dir.mkdir(parents=True, exist_ok=True)
-    backup_path = backup_dir / (filepath.stem + ".original.md")
-
-    if not original_text.strip():
-        print("❌ Refusing to compress: file is empty or whitespace-only.")
-        return False
-
-    # Check if backup already exists to prevent accidental overwriting
-    if backup_path.exists():
-        print(f"⚠️ Backup file already exists: {backup_path}")
-        print("The original backup may contain important content.")
-        print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
-        return False
-
-    # Split YAML frontmatter off before compression. Claude tends to strip or
-    # rewrite frontmatter despite preserve-structure rules; we keep it verbatim
-    # by removing it from the input and re-prepending it to the output.
-    frontmatter, body = split_frontmatter(original_text)
-    if frontmatter:
-        print(f"Detected YAML frontmatter ({len(frontmatter)} chars) — preserving verbatim")
-
-    if not body.strip():
-        print("❌ Refusing to compress: body is empty after frontmatter removal.")
-        return False
-
-    # Step 1: Compress (body only, frontmatter excluded)
-    print("Compressing with Claude...")
-    compressed_body = call_claude(build_compress_prompt(body))
-
-    if compressed_body is None or not compressed_body.strip():
-        print("❌ Compression aborted: Claude returned an empty response.")
-        print("   Original file is untouched (no backup created).")
-        return False
-
-    # Compare the BODY (not the whole file) — frontmatter is preserved verbatim
-    # and would never change, so identity must be judged on the compressible part.
-    if compressed_body.strip() == body.strip():
-        print("❌ Compression aborted: output is identical to input.")
-        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
-        print("   already in caveman form. Original file is untouched (no backup created).")
-        return False
-
-    # Reassemble: frontmatter (verbatim) + compressed body
-    compressed = frontmatter + compressed_body
-
-    # Save original as backup, then verify the backup readback before
-    # touching the input file. If the filesystem dropped bytes (encoding,
-    # antivirus, disk full), unlink the bad backup and abort instead of
-    # leaving the user with a corrupt backup + compressed primary.
-    backup_path.write_text(original_text)
-    backup_readback = backup_path.read_text(errors="ignore")
-    if backup_readback != original_text:
-        print(f"❌ Backup write verification failed: {backup_path}")
-        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
-        try:
-            backup_path.unlink()
-        except OSError:
-            pass
-        return False
-    filepath.write_text(compressed)
-
-    # Step 2: Validate + Retry
-    for attempt in range(MAX_RETRIES):
-        print(f"\nValidation attempt {attempt + 1}")
-
-        result = validate(backup_path, filepath)
-
-        if result.is_valid:
-            print("Validation passed")
-            break
-
-        print("❌ Validation failed:")
-        for err in result.errors:
-            print(f"   - {err}")
-
-        if attempt == MAX_RETRIES - 1:
-            # Restore original on failure
-            filepath.write_text(original_text)
-            backup_path.unlink(missing_ok=True)
-            print("❌ Failed after retries — original restored")
-            return False
-
-        print("Fixing with Claude...")
-        compressed = call_claude(
-            build_fix_prompt(original_text, compressed, result.errors)
-        )
-        filepath.write_text(compressed)
-
-    return True

package/skills/caveman-compress/scripts/detect.py

@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-"""Detect whether a file is natural language (compressible) or code/config (skip)."""
-
-import json
-import re
-from pathlib import Path
-
-# Extensions that are natural language and compressible
-COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
-
-# Extensions that are code/config and should be skipped
-SKIP_EXTENSIONS = {
-    ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
-    ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
-    ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
-    ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
-    ".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
-}
-
-# The subset of SKIP_EXTENSIONS that is configuration (not source code). Used to
-# decide whether a skipped file reports as "config" vs "code". Must stay a subset
-# of SKIP_EXTENSIONS (asserted below) so the two sets cannot silently drift.
-CONFIG_EXTENSIONS = {
-    ".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env",
-}
-assert CONFIG_EXTENSIONS <= SKIP_EXTENSIONS, (
-    "CONFIG_EXTENSIONS must be a subset of SKIP_EXTENSIONS: "
-    f"{CONFIG_EXTENSIONS - SKIP_EXTENSIONS} not in SKIP_EXTENSIONS"
-)
-
-# Real-world extensionless config/build filenames whose classification is NOT
-# already covered by the SKIP_EXTENSIONS fallback below. These have an empty
-# `Path.suffix`, so the SKIP_EXTENSIONS check never matches them and they would
-# otherwise be content-sniffed as natural language and offered up for
-# compression to a third-party API. Map the lowercased full filename to its
-# classification so a bare `Dockerfile`/`Makefile`/`.gitignore` is handled like
-# its dotted-extension cousins would be.
-#
-# Names that are themselves SKIP_EXTENSIONS entries (e.g. ".env") are
-# deliberately omitted here: the `name in SKIP_EXTENSIONS` fallback in
-# detect_file_type already classifies them identically, so listing them in both
-# places would be redundant and require hand-syncing.
-SKIP_FILENAMES = {
-    "dockerfile": "code",
-    "makefile": "code",
-    "gnumakefile": "code",
-    ".gitignore": "config",
-    ".gitattributes": "config",
-    ".dockerignore": "config",
-    ".editorconfig": "config",
-    ".npmrc": "config",
-    ".prettierrc": "config",
-    ".eslintrc": "config",
-    ".babelrc": "config",
-}
-
-# Patterns that indicate a line is code
-CODE_PATTERNS = [
-    re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
-    re.compile(r"^\s*(def |class |function |async function |export )"),
-    re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
-    re.compile(r"^\s*[\}\]\);]+\s*$"),  # closing braces/brackets
-    re.compile(r"^\s*@\w+"),  # decorators/annotations
-    re.compile(r'^\s*"[^"]+"\s*:\s*'),  # JSON-like key-value
-    re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"),  # assignment with literal
-]
-
-
-def _is_code_line(line: str) -> bool:
-    """Check if a line looks like code."""
-    return any(p.match(line) for p in CODE_PATTERNS)
-
-
-def _is_json_content(text: str) -> bool:
-    """Check if content is valid JSON."""
-    try:
-        json.loads(text)
-        return True
-    except (json.JSONDecodeError, ValueError):
-        return False
-
-
-def _is_yaml_content(lines: list[str]) -> bool:
-    """Heuristic: check if content looks like YAML."""
-    yaml_indicators = 0
-    for line in lines[:30]:
-        stripped = line.strip()
-        if stripped.startswith("---"):
-            yaml_indicators += 1
-        elif re.match(r"^\w[\w\s]*:\s", stripped):
-            yaml_indicators += 1
-        elif stripped.startswith("- ") and ":" in stripped:
-            yaml_indicators += 1
-    # If most non-empty lines look like YAML
-    non_empty = sum(1 for line in lines[:30] if line.strip())
-    return non_empty > 0 and yaml_indicators / non_empty > 0.6
-
-
-def detect_file_type(filepath: Path) -> str:
-    """Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
-
-    Returns:
-        One of: 'natural_language', 'code', 'config', 'unknown'
-    """
-    ext = filepath.suffix.lower()
-
-    # Extension-based classification
-    if ext in COMPRESSIBLE_EXTENSIONS:
-        return "natural_language"
-    if ext in SKIP_EXTENSIONS:
-        return "config" if ext in CONFIG_EXTENSIONS else "code"
-
-    # Extensionless files (like CLAUDE.md, TODO) — check content.
-    if not ext:
-        # Leading-dot / build files (".env", "Dockerfile", "Makefile",
-        # ".gitignore") have an empty suffix, so the SKIP_EXTENSIONS check above
-        # never matched them and they would be content-sniffed as natural
-        # language. Classify them by full filename first so they are never
-        # offered up for compression to a third-party API.
-        name = filepath.name.lower()
-        if name in SKIP_FILENAMES:
-            return SKIP_FILENAMES[name]
-        if name in SKIP_EXTENSIONS:
-            return "config" if name in CONFIG_EXTENSIONS else "code"
-
-        try:
-            text = filepath.read_text(errors="ignore")
-        except (OSError, PermissionError):
-            return "unknown"
-
-        lines = text.splitlines()[:50]
-
-        if _is_json_content(text[:10000]):
-            return "config"
-        if _is_yaml_content(lines):
-            return "config"
-
-        code_lines = sum(1 for line in lines if line.strip() and _is_code_line(line))
-        non_empty = sum(1 for line in lines if line.strip())
-        if non_empty > 0 and code_lines / non_empty > 0.4:
-            return "code"
-
-        return "natural_language"
-
-    return "unknown"
-
-
-def should_compress(filepath: Path) -> bool:
-    """Return True if the file is natural language and should be compressed."""
-    if not filepath.is_file():
-        return False
-    # Skip backup files
-    if filepath.name.endswith(".original.md"):
-        return False
-    return detect_file_type(filepath) == "natural_language"
-
-
-if __name__ == "__main__":
-    import sys
-
-    if len(sys.argv) < 2:
-        print("Usage: python detect.py <file1> [file2] ...")
-        sys.exit(1)
-
-    for path_str in sys.argv[1:]:
-        p = Path(path_str).resolve()
-        file_type = detect_file_type(p)
-        compress = should_compress(p)
-        print(f"  {p.name:30s} type={file_type:20s} compress={compress}")