@abarcenas/ai-workflow-template 1.2.0 → 1.13.0

package/.agents/skills/compress/scripts/compress.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""
+Caveman Memory Compression Orchestrator
+
+Usage:
+    python scripts/compress.py <filepath>
+"""
+
+import os
+import re
+import subprocess
+from pathlib import Path
+from typing import List
+
+OUTER_FENCE_REGEX = re.compile(
+    r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL
+)
+
+# Filenames and paths that almost certainly hold secrets or PII. Compressing
+# them ships raw bytes to the Anthropic API — a third-party data boundary that
+# developers on sensitive codebases cannot cross. detect.py already skips .env
+# by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
+# slip through the natural-language filter. This is a hard refuse before read.
+SENSITIVE_BASENAME_REGEX = re.compile(
+    r"(?ix)^("
+    r"\.env(\..+)?"
+    r"|\.netrc"
+    r"|credentials(\..+)?"
+    r"|secrets?(\..+)?"
+    r"|passwords?(\..+)?"
+    r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
+    r"|authorized_keys"
+    r"|known_hosts"
+    r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
+    r")$"
+)
+
+SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
+
+SENSITIVE_NAME_TOKENS = (
+    "secret", "credential", "password", "passwd",
+    "apikey", "accesskey", "token", "privatekey",
+)
+
+
+def is_sensitive_path(filepath: Path) -> bool:
+    """Heuristic denylist for files that must never be shipped to a third-party API."""
+    name = filepath.name
+    if SENSITIVE_BASENAME_REGEX.match(name):
+        return True
+    lowered_parts = {p.lower() for p in filepath.parts}
+    if lowered_parts & SENSITIVE_PATH_COMPONENTS:
+        return True
+    # Normalize separators so "api-key" and "api_key" both match "apikey".
+    lower = re.sub(r"[_\-\s.]", "", name.lower())
+    return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
+
+
+def strip_llm_wrapper(text: str) -> str:
+    """Strip outer ```markdown ... ``` fence when it wraps the entire output."""
+    m = OUTER_FENCE_REGEX.match(text)
+    if m:
+        return m.group(2)
+    return text
+
+from .detect import should_compress
+from .validate import validate
+
+MAX_RETRIES = 2
+
+
+# ---------- Claude Calls ----------
+
+
+def call_claude(prompt: str) -> str:
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if api_key:
+        try:
+            import anthropic
+
+            client = anthropic.Anthropic(api_key=api_key)
+            msg = client.messages.create(
+                model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
+                max_tokens=8192,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return strip_llm_wrapper(msg.content[0].text.strip())
+        except ImportError:
+            pass  # anthropic not installed, fall back to CLI
+    # Fallback: use claude CLI (handles desktop auth)
+    try:
+        result = subprocess.run(
+            ["claude", "--print"],
+            input=prompt,
+            text=True,
+            capture_output=True,
+            check=True,
+        )
+        return strip_llm_wrapper(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Claude call failed:\n{e.stderr}")
+
+
+def build_compress_prompt(original: str) -> str:
+    return f"""
+Compress this markdown into caveman format.
+
+STRICT RULES:
+- Do NOT modify anything inside ``` code blocks
+- Do NOT modify anything inside inline backticks
+- Preserve ALL URLs exactly
+- Preserve ALL headings exactly
+- Preserve file paths and commands
+- Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
+
+Only compress natural language.
+
+TEXT:
+{original}
+"""
+
+
+def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str:
+    errors_str = "\n".join(f"- {e}" for e in errors)
+    return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
+
+CRITICAL RULES:
+- DO NOT recompress or rephrase the file
+- ONLY fix the listed errors — leave everything else exactly as-is
+- The ORIGINAL is provided as reference only (to restore missing content)
+- Preserve caveman style in all untouched sections
+
+ERRORS TO FIX:
+{errors_str}
+
+HOW TO FIX:
+- Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
+- Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
+- Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
+- Do not touch any section not mentioned in the errors
+
+ORIGINAL (reference only):
+{original}
+
+COMPRESSED (fix this):
+{compressed}
+
+Return ONLY the fixed compressed file. No explanation.
+"""
+
+
+# ---------- Core Logic ----------
+
+
+def compress_file(filepath: Path) -> bool:
+    # Resolve and validate path
+    filepath = filepath.resolve()
+    MAX_FILE_SIZE = 500_000  # 500KB
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {filepath}")
+    if filepath.stat().st_size > MAX_FILE_SIZE:
+        raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
+
+    # Refuse files that look like they contain secrets or PII. Compressing ships
+    # the raw bytes to the Anthropic API — a third-party boundary — so we fail
+    # loudly rather than silently exfiltrate credentials or keys. Override is
+    # intentional: the user must rename the file if the heuristic is wrong.
+    if is_sensitive_path(filepath):
+        raise ValueError(
+            f"Refusing to compress {filepath}: filename looks sensitive "
+            "(credentials, keys, secrets, or known private paths). "
+            "Compression sends file contents to the Anthropic API. "
+            "Rename the file if this is a false positive."
+        )
+
+    print(f"Processing: {filepath}")
+
+    if not should_compress(filepath):
+        print("Skipping (not natural language)")
+        return False
+
+    original_text = filepath.read_text(errors="ignore")
+    backup_path = filepath.with_name(filepath.stem + ".original.md")
+
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
+    # Check if backup already exists to prevent accidental overwriting
+    if backup_path.exists():
+        print(f"⚠️ Backup file already exists: {backup_path}")
+        print("The original backup may contain important content.")
+        print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
+        return False
+
+    # Step 1: Compress
+    print("Compressing with Claude...")
+    compressed = call_claude(build_compress_prompt(original_text))
+
+    if compressed is None or not compressed.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    if compressed.strip() == original_text.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
+    backup_path.write_text(original_text)
+    backup_readback = backup_path.read_text(errors="ignore")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
+    filepath.write_text(compressed)
+
+    # Step 2: Validate + Retry
+    for attempt in range(MAX_RETRIES):
+        print(f"\nValidation attempt {attempt + 1}")
+
+        result = validate(backup_path, filepath)
+
+        if result.is_valid:
+            print("Validation passed")
+            break
+
+        print("❌ Validation failed:")
+        for err in result.errors:
+            print(f"   - {err}")
+
+        if attempt == MAX_RETRIES - 1:
+            # Restore original on failure
+            filepath.write_text(original_text)
+            backup_path.unlink(missing_ok=True)
+            print("❌ Failed after retries — original restored")
+            return False
+
+        print("Fixing with Claude...")
+        compressed = call_claude(
+            build_fix_prompt(original_text, compressed, result.errors)
+        )
+        filepath.write_text(compressed)
+
+    return True

package/.agents/skills/compress/scripts/detect.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Detect whether a file is natural language (compressible) or code/config (skip)."""
+
+import json
+import re
+from pathlib import Path
+
+# Extensions that are natural language and compressible
+COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
+
+# Extensions that are code/config and should be skipped
+SKIP_EXTENSIONS = {
+    ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
+    ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
+    ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
+    ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
+    ".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
+}
+
+# Patterns that indicate a line is code
+CODE_PATTERNS = [
+    re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
+    re.compile(r"^\s*(def |class |function |async function |export )"),
+    re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
+    re.compile(r"^\s*[\}\]\);]+\s*$"),  # closing braces/brackets
+    re.compile(r"^\s*@\w+"),  # decorators/annotations
+    re.compile(r'^\s*"[^"]+"\s*:\s*'),  # JSON-like key-value
+    re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"),  # assignment with literal
+]
+
+
+def _is_code_line(line: str) -> bool:
+    """Check if a line looks like code."""
+    return any(p.match(line) for p in CODE_PATTERNS)
+
+
+def _is_json_content(text: str) -> bool:
+    """Check if content is valid JSON."""
+    try:
+        json.loads(text)
+        return True
+    except (json.JSONDecodeError, ValueError):
+        return False
+
+
+def _is_yaml_content(lines: list[str]) -> bool:
+    """Heuristic: check if content looks like YAML."""
+    yaml_indicators = 0
+    for line in lines[:30]:
+        stripped = line.strip()
+        if stripped.startswith("---"):
+            yaml_indicators += 1
+        elif re.match(r"^\w[\w\s]*:\s", stripped):
+            yaml_indicators += 1
+        elif stripped.startswith("- ") and ":" in stripped:
+            yaml_indicators += 1
+    # If most non-empty lines look like YAML
+    non_empty = sum(1 for l in lines[:30] if l.strip())
+    return non_empty > 0 and yaml_indicators / non_empty > 0.6
+
+
+def detect_file_type(filepath: Path) -> str:
+    """Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
+
+    Returns:
+        One of: 'natural_language', 'code', 'config', 'unknown'
+    """
+    ext = filepath.suffix.lower()
+
+    # Extension-based classification
+    if ext in COMPRESSIBLE_EXTENSIONS:
+        return "natural_language"
+    if ext in SKIP_EXTENSIONS:
+        return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
+
+    # Extensionless files (like CLAUDE.md, TODO) — check content
+    if not ext:
+        try:
+            text = filepath.read_text(errors="ignore")
+        except (OSError, PermissionError):
+            return "unknown"
+
+        lines = text.splitlines()[:50]
+
+        if _is_json_content(text[:10000]):
+            return "config"
+        if _is_yaml_content(lines):
+            return "config"
+
+        code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
+        non_empty = sum(1 for l in lines if l.strip())
+        if non_empty > 0 and code_lines / non_empty > 0.4:
+            return "code"
+
+        return "natural_language"
+
+    return "unknown"
+
+
+def should_compress(filepath: Path) -> bool:
+    """Return True if the file is natural language and should be compressed."""
+    if not filepath.is_file():
+        return False
+    # Skip backup files
+    if filepath.name.endswith(".original.md"):
+        return False
+    return detect_file_type(filepath) == "natural_language"
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python detect.py <file1> [file2] ...")
+        sys.exit(1)
+
+    for path_str in sys.argv[1:]:
+        p = Path(path_str).resolve()
+        file_type = detect_file_type(p)
+        compress = should_compress(p)
+        print(f"  {p.name:30s} type={file_type:20s} compress={compress}")

package/.agents/skills/compress/scripts/validate.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+import re
+from collections import Counter
+from pathlib import Path
+
+URL_REGEX = re.compile(r"https?://[^\s)]+")
+FENCE_OPEN_REGEX = re.compile(r"^(\s{0,3})(`{3,}|~{3,})(.*)$")
+HEADING_REGEX = re.compile(r"^(#{1,6})\s+(.*)", re.MULTILINE)
+BULLET_REGEX = re.compile(r"^\s*[-*+]\s+", re.MULTILINE)
+
+# crude but effective path detection
+# Requires either a path prefix (./ ../ / or drive letter) or a slash/backslash within the match
+PATH_REGEX = re.compile(r"(?:\./|\.\./|/|[A-Za-z]:\\)[\w\-/\\\.]+|[\w\-\.]+[/\\][\w\-/\\\.]+")
+
+
+class ValidationResult:
+    def __init__(self):
+        self.is_valid = True
+        self.errors = []
+        self.warnings = []
+
+    def add_error(self, msg):
+        self.is_valid = False
+        self.errors.append(msg)
+
+    def add_warning(self, msg):
+        self.warnings.append(msg)
+
+
+def read_file(path: Path) -> str:
+    return path.read_text(errors="ignore")
+
+
+# ---------- Extractors ----------
+
+
+def extract_headings(text):
+    return [(level, title.strip()) for level, title in HEADING_REGEX.findall(text)]
+
+
+def extract_code_blocks(text):
+    """Line-based fenced code block extractor.
+
+    Handles ``` and ~~~ fences with variable length (CommonMark: closing
+    fence must use same char and be at least as long as opening). Supports
+    nested fences (e.g. an outer 4-backtick block wrapping inner 3-backtick
+    content).
+    """
+    blocks = []
+    lines = text.split("\n")
+    i = 0
+    n = len(lines)
+    while i < n:
+        m = FENCE_OPEN_REGEX.match(lines[i])
+        if not m:
+            i += 1
+            continue
+        fence_char = m.group(2)[0]
+        fence_len = len(m.group(2))
+        open_line = lines[i]
+        block_lines = [open_line]
+        i += 1
+        closed = False
+        while i < n:
+            close_m = FENCE_OPEN_REGEX.match(lines[i])
+            if (
+                close_m
+                and close_m.group(2)[0] == fence_char
+                and len(close_m.group(2)) >= fence_len
+                and close_m.group(3).strip() == ""
+            ):
+                block_lines.append(lines[i])
+                closed = True
+                i += 1
+                break
+            block_lines.append(lines[i])
+            i += 1
+        if closed:
+            blocks.append("\n".join(block_lines))
+        # Unclosed fences are silently skipped — they indicate malformed markdown
+        # and including them would cause false-positive validation failures.
+    return blocks
+
+
+def extract_urls(text):
+    return set(URL_REGEX.findall(text))
+
+
+def extract_paths(text):
+    return set(PATH_REGEX.findall(text))
+
+
+def count_bullets(text):
+    return len(BULLET_REGEX.findall(text))
+
+
+def extract_inline_codes(text):
+    text_without_fences = re.sub(r"^```[\s\S]*?^```", "", text, flags=re.MULTILINE)
+    text_without_fences = re.sub(r"^~~~[\s\S]*?^~~~", "", text_without_fences, flags=re.MULTILINE)
+    return re.findall(r"`([^`]+)`", text_without_fences)
+
+
+# ---------- Validators ----------
+
+
+def validate_headings(orig, comp, result):
+    h1 = extract_headings(orig)
+    h2 = extract_headings(comp)
+
+    if len(h1) != len(h2):
+        result.add_error(f"Heading count mismatch: {len(h1)} vs {len(h2)}")
+
+    if h1 != h2:
+        result.add_warning("Heading text/order changed")
+
+
+def validate_code_blocks(orig, comp, result):
+    c1 = extract_code_blocks(orig)
+    c2 = extract_code_blocks(comp)
+
+    if c1 != c2:
+        result.add_error("Code blocks not preserved exactly")
+
+
+def validate_urls(orig, comp, result):
+    u1 = extract_urls(orig)
+    u2 = extract_urls(comp)
+
+    if u1 != u2:
+        result.add_error(f"URL mismatch: lost={u1 - u2}, added={u2 - u1}")
+
+
+def validate_paths(orig, comp, result):
+    p1 = extract_paths(orig)
+    p2 = extract_paths(comp)
+
+    if p1 != p2:
+        result.add_warning(f"Path mismatch: lost={p1 - p2}, added={p2 - p1}")
+
+
+def validate_bullets(orig, comp, result):
+    b1 = count_bullets(orig)
+    b2 = count_bullets(comp)
+
+    if b1 == 0:
+        return
+
+    diff = abs(b1 - b2) / b1
+
+    if diff > 0.15:
+        result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
+
+
+def validate_inline_codes(orig, comp, result):
+    c1 = Counter(extract_inline_codes(orig))
+    c2 = Counter(extract_inline_codes(comp))
+
+    if c1 != c2:
+        lost = set(c1.keys()) - set(c2.keys())
+        added = set(c2.keys()) - set(c1.keys())
+        for code, count in c1.items():
+            if code in c2 and c2[code] < count:
+                lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
+        if lost:
+            result.add_error(f"Inline code lost: {lost}")
+        if added:
+            result.add_warning(f"Inline code added: {added}")
+
+
+# ---------- Main ----------
+
+
+def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
+    result = ValidationResult()
+
+    orig = read_file(original_path)
+    comp = read_file(compressed_path)
+
+    validate_headings(orig, comp, result)
+    validate_code_blocks(orig, comp, result)
+    validate_urls(orig, comp, result)
+    validate_paths(orig, comp, result)
+    validate_bullets(orig, comp, result)
+    validate_inline_codes(orig, comp, result)
+
+    return result
+
+
+# ---------- CLI ----------
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 3:
+        print("Usage: python validate.py <original> <compressed>")
+        sys.exit(1)
+
+    orig = Path(sys.argv[1]).resolve()
+    comp = Path(sys.argv[2]).resolve()
+
+    res = validate(orig, comp)
+
+    print(f"\nValid: {res.is_valid}")
+
+    if res.errors:
+        print("\nErrors:")
+        for e in res.errors:
+            print(f"  - {e}")
+
+    if res.warnings:
+        print("\nWarnings:")
+        for w in res.warnings:
+            print(f"  - {w}")