@izhimu/pi-caveman 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +68 -0
  2. package/README.md +60 -74
  3. package/extensions/caveman.ts +452 -346
  4. package/package.json +37 -6
  5. package/skills/caveman/README.md +68 -48
  6. package/skills/caveman/SKILL.md +57 -56
  7. package/skills/megacave/README.md +11 -0
  8. package/skills/megacave/SKILL.md +65 -0
  9. package/skills/ultracave/README.md +13 -0
  10. package/skills/ultracave/SKILL.md +63 -0
  11. package/skills/cavecrew/README.md +0 -61
  12. package/skills/cavecrew/SKILL.md +0 -82
  13. package/skills/caveman-commit/README.md +0 -44
  14. package/skills/caveman-commit/SKILL.md +0 -65
  15. package/skills/caveman-compress/README.md +0 -163
  16. package/skills/caveman-compress/SECURITY.md +0 -31
  17. package/skills/caveman-compress/SKILL.md +0 -111
  18. package/skills/caveman-compress/scripts/__init__.py +0 -9
  19. package/skills/caveman-compress/scripts/__main__.py +0 -3
  20. package/skills/caveman-compress/scripts/benchmark.py +0 -80
  21. package/skills/caveman-compress/scripts/cli.py +0 -85
  22. package/skills/caveman-compress/scripts/compress.py +0 -414
  23. package/skills/caveman-compress/scripts/detect.py +0 -139
  24. package/skills/caveman-compress/scripts/validate.py +0 -221
  25. package/skills/caveman-discover/SKILL.md +0 -118
  26. package/skills/caveman-evidence-review/SKILL.md +0 -144
  27. package/skills/caveman-explore/SKILL.md +0 -42
  28. package/skills/caveman-explore/package.json +0 -12
  29. package/skills/caveman-explore/tests/skill-file.test.mjs +0 -44
  30. package/skills/caveman-help/README.md +0 -38
  31. package/skills/caveman-help/SKILL.md +0 -63
  32. package/skills/caveman-learn/CLAUDE.md +0 -32
  33. package/skills/caveman-learn/README.md +0 -29
  34. package/skills/caveman-learn/SKILL.md +0 -69
  35. package/skills/caveman-learn/package.json +0 -12
  36. package/skills/caveman-learn/tests/skill-file.test.mjs +0 -44
  37. package/skills/caveman-manage/SKILL.md +0 -114
  38. package/skills/caveman-optimize/SKILL.md +0 -114
  39. package/skills/caveman-review/README.md +0 -33
  40. package/skills/caveman-review/SKILL.md +0 -55
  41. package/skills/caveman-setup/SKILL.md +0 -224
  42. package/skills/caveman-stats/README.md +0 -36
  43. package/skills/caveman-stats/SKILL.md +0 -12
  44. package/skills/investigate-first/SKILL.md +0 -16
  45. package/skills/investigate-first/agents/openai.yaml +0 -4
  46. package/skills/lean-build/SKILL.md +0 -18
  47. package/skills/lean-build/agents/openai.yaml +0 -4
  48. package/skills/migration/SKILL.md +0 -17
  49. package/skills/migration/agents/openai.yaml +0 -4
  50. package/skills/safe-refactor/SKILL.md +0 -16
  51. package/skills/safe-refactor/agents/openai.yaml +0 -4
  52. package/skills/surgical-patch/SKILL.md +0 -16
  53. package/skills/surgical-patch/agents/openai.yaml +0 -4
  54. package/skills/verify-and-stop/SKILL.md +0 -16
  55. package/skills/verify-and-stop/agents/openai.yaml +0 -4
@@ -1,414 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Caveman Memory Compression Orchestrator
4
-
5
- Usage:
6
- python scripts/compress.py <filepath>
7
- """
8
-
9
- import os
10
- import re
11
- import shutil
12
- import stat
13
- import subprocess
14
- import sys
15
- import tempfile
16
- from pathlib import Path
17
- from typing import List
18
-
19
- OUTER_FENCE_REGEX = re.compile(
20
- r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL
21
- )
22
-
23
- # YAML frontmatter: starts at file start with --- on its own line, ends with --- on its own line.
24
- # Captures the entire block (including delimiters and trailing newline) and the body after.
25
- FRONTMATTER_REGEX = re.compile(
26
- r"\A(---\r?\n.*?\r?\n---\r?\n)(.*)", re.DOTALL
27
- )
28
-
29
-
30
- def split_frontmatter(text: str):
31
- """Split YAML frontmatter from body. Returns (frontmatter, body).
32
-
33
- Memory files (and many other markdown docs) start with a YAML frontmatter
34
- block delimited by `---` lines. The compression LLM has a habit of stripping
35
- or rewriting these despite preserve-structure rules in the prompt — so we
36
- surgically remove the frontmatter before compression and prepend it back
37
- verbatim to the output. Files without frontmatter pass through unchanged.
38
- """
39
- m = FRONTMATTER_REGEX.match(text)
40
- if m:
41
- return m.group(1), m.group(2)
42
- return "", text
43
-
44
- # Filenames and paths that almost certainly hold secrets or PII. Compressing
45
- # them ships raw bytes to the Anthropic API — a third-party data boundary that
46
- # developers on sensitive codebases cannot cross. detect.py already skips .env
47
- # by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
48
- # slip through the natural-language filter. This is a hard refuse before read.
49
- SENSITIVE_BASENAME_REGEX = re.compile(
50
- r"(?ix)^("
51
- r"\.env(\..+)?"
52
- r"|\.netrc"
53
- r"|credentials(\..+)?"
54
- r"|secrets?(\..+)?"
55
- r"|passwords?(\..+)?"
56
- r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
57
- r"|authorized_keys"
58
- r"|known_hosts"
59
- r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
60
- r")$"
61
- )
62
-
63
- SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
64
-
65
- SENSITIVE_NAME_TOKENS = (
66
- "secret", "credential", "password", "passwd",
67
- "apikey", "accesskey", "token", "privatekey",
68
- )
69
-
70
-
71
- def backup_dir_for(filepath: Path) -> Path:
72
- """Resolve the out-of-tree backup directory for a given source file.
73
-
74
- Backups must live OUTSIDE the source directory so skill auto-loaders
75
- (Claude Code rules/, opencode instructions/, etc.) stop re-ingesting the
76
- `.original.md` copies as live files. Base dir is platform-aware:
77
- - Windows: %LOCALAPPDATA%\\caveman-compress\\backups
78
- - else: $XDG_DATA_HOME/caveman-compress/backups if set,
79
- else ~/.local/share/caveman-compress/backups
80
-
81
- The source file's parent-dir name is mirrored under the base to reduce
82
- cross-project collisions (e.g. two `task.md` files in different repos).
83
- """
84
- if os.name == "nt" or sys.platform == "win32":
85
- local_appdata = os.environ.get("LOCALAPPDATA")
86
- base = Path(local_appdata) if local_appdata else Path.home() / "AppData" / "Local"
87
- base = base / "caveman-compress" / "backups"
88
- else:
89
- xdg = os.environ.get("XDG_DATA_HOME")
90
- base = Path(xdg) if xdg else Path.home() / ".local" / "share"
91
- base = base / "caveman-compress" / "backups"
92
- return base / filepath.parent.name
93
-
94
-
95
- def is_sensitive_path(filepath: Path) -> bool:
96
- """Heuristic denylist for files that must never be shipped to a third-party API."""
97
- name = filepath.name
98
- if SENSITIVE_BASENAME_REGEX.match(name):
99
- return True
100
- lowered_parts = {p.lower() for p in filepath.parts}
101
- if lowered_parts & SENSITIVE_PATH_COMPONENTS:
102
- return True
103
- # Normalize separators so "api-key" and "api_key" both match "apikey".
104
- lower = re.sub(r"[_\-\s.]", "", name.lower())
105
- return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
106
-
107
-
108
- def strip_llm_wrapper(text: str) -> str:
109
- """Strip outer ```markdown ... ``` fence when it wraps the entire output."""
110
- m = OUTER_FENCE_REGEX.match(text)
111
- if m:
112
- return m.group(2)
113
- return text
114
-
115
-
116
- def write_text_atomic(path: Path, text: str) -> None:
117
- """Write ``text`` to ``path`` atomically as UTF-8.
118
-
119
- Path.write_text() truncates the destination before encoding the string —
120
- a UnicodeEncodeError (or any other failure) partway through leaves a
121
- 0-byte file, destroying whatever was there before (issue #655). Encode
122
- first, write the bytes to a sibling temp file, fsync, then os.replace()
123
- so the destination only ever moves from one complete, valid file to
124
- another. Preserves the original file's permission bits across the swap.
125
- """
126
- data = text.encode("utf-8")
127
- fd, tmp_name = tempfile.mkstemp(
128
- dir=str(path.parent), prefix=path.name + ".", suffix=".tmp"
129
- )
130
- tmp_path = Path(tmp_name)
131
- try:
132
- with os.fdopen(fd, "wb") as f:
133
- f.write(data)
134
- f.flush()
135
- os.fsync(f.fileno())
136
- if path.exists():
137
- os.chmod(tmp_path, stat.S_IMODE(path.stat().st_mode))
138
- os.replace(tmp_path, path)
139
- except Exception:
140
- try:
141
- tmp_path.unlink()
142
- except OSError:
143
- pass
144
- raise
145
-
146
-
147
- def first_nonblank_line(text: str) -> str:
148
- """Return the first non-blank line, stripped — used to detect a prose
149
- preamble smuggled in ahead of the real content (issue #588)."""
150
- for line in text.splitlines():
151
- if line.strip():
152
- return line.strip()
153
- return ""
154
-
155
-
156
- def _write_target(filepath: Path, text: str, backup_path: Path) -> None:
157
- """Write to the target file, surfacing the backup location if the write
158
- itself fails. write_text_atomic already leaves the target untouched on
159
- failure, but the caller still needs to know where the pre-compression
160
- original lives instead of being left to guess (issue #652)."""
161
- try:
162
- write_text_atomic(filepath, text)
163
- except Exception:
164
- print(f"❌ Write to {filepath} failed. Original preserved at backup: {backup_path}")
165
- raise
166
-
167
-
168
- from .detect import should_compress
169
- from .validate import validate
170
-
171
- MAX_RETRIES = 2
172
-
173
-
174
- # ---------- Claude Calls ----------
175
-
176
-
177
- def call_claude(prompt: str) -> str:
178
- """Send a prompt to Claude.
179
-
180
- Prefers the Anthropic SDK when ANTHROPIC_API_KEY is set; otherwise falls
181
- back to the ``claude --print`` CLI (which handles desktop auth).
182
-
183
- On Windows the CLI subprocess decoding defaults to the system codepage
184
- (cp1251 / cp1252) and crashes on UTF-8 output — see issue #152. Pinning
185
- ``encoding="utf-8"`` with ``errors="replace"`` matches the CLI's actual
186
- native I/O and prevents the UnicodeDecodeError before validation can
187
- report. Windows users with non-ASCII content can also set
188
- ``ANTHROPIC_API_KEY`` to route through the SDK and skip the subprocess.
189
- """
190
- api_key = os.environ.get("ANTHROPIC_API_KEY")
191
- if api_key:
192
- try:
193
- import anthropic
194
-
195
- client = anthropic.Anthropic(api_key=api_key)
196
- msg = client.messages.create(
197
- model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
198
- max_tokens=8192,
199
- messages=[{"role": "user", "content": prompt}],
200
- )
201
- return strip_llm_wrapper(msg.content[0].text.strip())
202
- except ImportError:
203
- pass # anthropic not installed, fall back to CLI
204
- # Fallback: use claude CLI (handles desktop auth).
205
- # Resolve binary via shutil.which so Windows .cmd/.bat shims (e.g.
206
- # %APPDATA%\npm\claude.CMD) work without shell=True. On POSIX,
207
- # shutil.which returns the same absolute path as the implicit lookup,
208
- # so this is a no-op there. Falls back to bare "claude" if not found
209
- # on PATH so subprocess raises a clear FileNotFoundError.
210
- claude_bin = shutil.which("claude") or "claude"
211
- try:
212
- result = subprocess.run(
213
- [claude_bin, "--print"],
214
- input=prompt,
215
- text=True,
216
- capture_output=True,
217
- check=True,
218
- encoding="utf-8",
219
- errors="replace",
220
- )
221
- return strip_llm_wrapper(result.stdout.strip())
222
- except subprocess.CalledProcessError as e:
223
- raise RuntimeError(f"Claude call failed:\n{e.stderr}")
224
-
225
-
226
- def build_compress_prompt(original: str) -> str:
227
- return f"""
228
- Compress this markdown into caveman format.
229
-
230
- STRICT RULES:
231
- - Do NOT modify anything inside ``` code blocks
232
- - Do NOT modify anything inside inline backticks
233
- - Preserve ALL URLs exactly
234
- - Preserve ALL headings exactly
235
- - Preserve file paths and commands
236
- - Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
237
-
238
- Only compress natural language.
239
-
240
- TEXT:
241
- {original}
242
- """
243
-
244
-
245
- def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str:
246
- errors_str = "\n".join(f"- {e}" for e in errors)
247
- return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
248
-
249
- CRITICAL RULES:
250
- - DO NOT recompress or rephrase the file
251
- - ONLY fix the listed errors — leave everything else exactly as-is
252
- - The ORIGINAL is provided as reference only (to restore missing content)
253
- - Preserve caveman style in all untouched sections
254
-
255
- ERRORS TO FIX:
256
- {errors_str}
257
-
258
- HOW TO FIX:
259
- - Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
260
- - Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
261
- - Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
262
- - Do not touch any section not mentioned in the errors
263
-
264
- ORIGINAL (reference only):
265
- {original}
266
-
267
- COMPRESSED (fix this):
268
- {compressed}
269
-
270
- Return ONLY the fixed compressed file. No explanation.
271
- """
272
-
273
-
274
- # ---------- Core Logic ----------
275
-
276
-
277
- def compress_file(filepath: Path) -> bool:
278
- # Resolve and validate path
279
- filepath = filepath.resolve()
280
- MAX_FILE_SIZE = 500_000 # 500KB
281
- if not filepath.exists():
282
- raise FileNotFoundError(f"File not found: {filepath}")
283
- if filepath.stat().st_size > MAX_FILE_SIZE:
284
- raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
285
-
286
- # Refuse files that look like they contain secrets or PII. Compressing ships
287
- # the raw bytes to the Anthropic API — a third-party boundary — so we fail
288
- # loudly rather than silently exfiltrate credentials or keys. Override is
289
- # intentional: the user must rename the file if the heuristic is wrong.
290
- if is_sensitive_path(filepath):
291
- raise ValueError(
292
- f"Refusing to compress {filepath}: filename looks sensitive "
293
- "(credentials, keys, secrets, or known private paths). "
294
- "Compression sends file contents to the Anthropic API. "
295
- "Rename the file if this is a false positive."
296
- )
297
-
298
- print(f"Processing: {filepath}")
299
-
300
- if not should_compress(filepath):
301
- print("Skipping (not natural language)")
302
- return False
303
-
304
- original_text = filepath.read_text(encoding="utf-8", errors="ignore")
305
- # Store backup outside the source directory so skill auto-loaders don't
306
- # re-ingest the `.original.md` copy as a live file. Mirror the source's
307
- # parent-dir name + stem under a platform-aware base to reduce collisions.
308
- backup_dir = backup_dir_for(filepath)
309
- backup_path = backup_dir / (filepath.stem + ".original.md")
310
-
311
- if not original_text.strip():
312
- print("❌ Refusing to compress: file is empty or whitespace-only.")
313
- return False
314
-
315
- # Check if backup already exists to prevent accidental overwriting
316
- if backup_path.exists():
317
- print(f"⚠️ Backup file already exists: {backup_path}")
318
- print("The original backup may contain important content.")
319
- print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
320
- return False
321
-
322
- # Split YAML frontmatter off before compression. Claude tends to strip or
323
- # rewrite frontmatter despite preserve-structure rules; we keep it verbatim
324
- # by removing it from the input and re-prepending it to the output.
325
- frontmatter, body = split_frontmatter(original_text)
326
- if frontmatter:
327
- print(f"Detected YAML frontmatter ({len(frontmatter)} chars) — preserving verbatim")
328
-
329
- if not body.strip():
330
- print("❌ Refusing to compress: body is empty after frontmatter removal.")
331
- return False
332
-
333
- # Step 1: Compress (body only, frontmatter excluded)
334
- print("Compressing with Claude...")
335
- compressed_body = call_claude(build_compress_prompt(body))
336
-
337
- if compressed_body is None or not compressed_body.strip():
338
- print("❌ Compression aborted: Claude returned an empty response.")
339
- print(" Original file is untouched (no backup created).")
340
- return False
341
-
342
- # Compare the BODY (not the whole file) — frontmatter is preserved verbatim
343
- # and would never change, so identity must be judged on the compressible part.
344
- if compressed_body.strip() == body.strip():
345
- print("❌ Compression aborted: output is identical to input.")
346
- print(" Likely causes: Claude refused, returned the prompt verbatim, or the file is")
347
- print(" already in caveman form. Original file is untouched (no backup created).")
348
- return False
349
-
350
- # Reassemble: frontmatter (verbatim) + compressed body
351
- compressed = frontmatter + compressed_body
352
-
353
- # Save original as backup, then verify the backup readback before
354
- # touching the input file. If the filesystem dropped bytes (encoding,
355
- # antivirus, disk full), unlink the bad backup and abort instead of
356
- # leaving the user with a corrupt backup + compressed primary.
357
- backup_dir.mkdir(parents=True, exist_ok=True)
358
- write_text_atomic(backup_path, original_text)
359
- backup_readback = backup_path.read_text(encoding="utf-8", errors="ignore")
360
- if backup_readback != original_text:
361
- print(f"❌ Backup write verification failed: {backup_path}")
362
- print(" In-memory original differs from on-disk backup. Aborting before touching the input file.")
363
- try:
364
- backup_path.unlink()
365
- except OSError:
366
- pass
367
- return False
368
- _write_target(filepath, compressed, backup_path)
369
-
370
- # Step 2: Validate + Retry
371
- for attempt in range(MAX_RETRIES):
372
- print(f"\nValidation attempt {attempt + 1}")
373
-
374
- result = validate(backup_path, filepath)
375
-
376
- if result.is_valid:
377
- print("Validation passed")
378
- break
379
-
380
- print("❌ Validation failed:")
381
- for err in result.errors:
382
- print(f" - {err}")
383
-
384
- if attempt == MAX_RETRIES - 1:
385
- # Restore original on failure
386
- _write_target(filepath, original_text, backup_path)
387
- backup_path.unlink(missing_ok=True)
388
- print("❌ Failed after retries — original restored")
389
- return False
390
-
391
- print("Fixing with Claude...")
392
- compressed = call_claude(
393
- build_fix_prompt(original_text, compressed, result.errors)
394
- )
395
-
396
- if compressed is None or not compressed.strip():
397
- print("❌ Fix attempt aborted: Claude returned an empty response.")
398
- print(" Skipping this attempt.")
399
- continue
400
-
401
- # Guard against a prose preamble smuggled in ahead of the real fixed
402
- # content (issue #588). Only enforced when the original starts with a
403
- # structural anchor (frontmatter `---` or a heading) — plain-prose
404
- # first lines get legitimately rewritten by compression, and requiring
405
- # them verbatim would reject every valid fix.
406
- anchor = first_nonblank_line(original_text)
407
- if anchor.startswith(("---", "#")) and first_nonblank_line(compressed) != anchor:
408
- print("❌ Fix attempt aborted: output does not start with the original's first line.")
409
- print(" Possible preamble leak. Skipping this attempt.")
410
- continue
411
-
412
- _write_target(filepath, compressed, backup_path)
413
-
414
- return True
@@ -1,139 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Detect whether a file is natural language (compressible) or code/config (skip)."""
3
-
4
- import json
5
- import re
6
- from pathlib import Path
7
-
8
- # Extensions that are natural language and compressible
9
- COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
10
-
11
- # Extensions that are code/config and should be skipped
12
- SKIP_EXTENSIONS = {
13
- ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
14
- ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
15
- ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
16
- ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
17
- ".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
18
- }
19
-
20
- # Well-known build/config files that carry no (or a misleading) extension —
21
- # `Dockerfile` has no suffix so `.dockerfile` above never matches it, and
22
- # `CMakeLists.txt` would ride the compressible `.txt` rule. Checked by
23
- # basename before any extension rule.
24
- KNOWN_CODE_FILENAMES = {
25
- "dockerfile", "makefile", "gnumakefile", "jenkinsfile", "vagrantfile",
26
- "rakefile", "gemfile", "justfile", "procfile", "brewfile",
27
- "cmakelists.txt",
28
- }
29
-
30
- # Patterns that indicate a line is code
31
- CODE_PATTERNS = [
32
- re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
33
- re.compile(r"^\s*(def |class |function |async function |export )"),
34
- re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
35
- re.compile(r"^\s*[\}\]\);]+\s*$"), # closing braces/brackets
36
- re.compile(r"^\s*@\w+"), # decorators/annotations
37
- re.compile(r'^\s*"[^"]+"\s*:\s*'), # JSON-like key-value
38
- re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"), # assignment with literal
39
- ]
40
-
41
-
42
- def _is_code_line(line: str) -> bool:
43
- """Check if a line looks like code."""
44
- return any(p.match(line) for p in CODE_PATTERNS)
45
-
46
-
47
- def _is_json_content(text: str) -> bool:
48
- """Check if content is valid JSON."""
49
- try:
50
- json.loads(text)
51
- return True
52
- except (json.JSONDecodeError, ValueError):
53
- return False
54
-
55
-
56
- def _is_yaml_content(lines: list[str]) -> bool:
57
- """Heuristic: check if content looks like YAML."""
58
- yaml_indicators = 0
59
- for line in lines[:30]:
60
- stripped = line.strip()
61
- if stripped.startswith("---"):
62
- yaml_indicators += 1
63
- elif re.match(r"^\w[\w\s]*:\s", stripped):
64
- yaml_indicators += 1
65
- elif stripped.startswith("- ") and ":" in stripped:
66
- yaml_indicators += 1
67
- # If most non-empty lines look like YAML
68
- non_empty = sum(1 for l in lines[:30] if l.strip())
69
- return non_empty > 0 and yaml_indicators / non_empty > 0.6
70
-
71
-
72
- def detect_file_type(filepath: Path) -> str:
73
- """Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
74
-
75
- Returns:
76
- One of: 'natural_language', 'code', 'config', 'unknown'
77
- """
78
- ext = filepath.suffix.lower()
79
-
80
- # Known code filenames win over any extension rule
81
- if filepath.name.lower() in KNOWN_CODE_FILENAMES:
82
- return "code"
83
-
84
- # Extension-based classification
85
- if ext in COMPRESSIBLE_EXTENSIONS:
86
- return "natural_language"
87
- if ext in SKIP_EXTENSIONS:
88
- return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
89
-
90
- # Extensionless files (like CLAUDE.md, TODO) — check content
91
- if not ext:
92
- try:
93
- text = filepath.read_text(encoding="utf-8", errors="ignore")
94
- except (OSError, PermissionError):
95
- return "unknown"
96
-
97
- lines = text.splitlines()[:50]
98
-
99
- # Shebang means executable script, never prose
100
- if text.startswith("#!"):
101
- return "code"
102
-
103
- if _is_json_content(text[:10000]):
104
- return "config"
105
- if _is_yaml_content(lines):
106
- return "config"
107
-
108
- code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
109
- non_empty = sum(1 for l in lines if l.strip())
110
- if non_empty > 0 and code_lines / non_empty > 0.4:
111
- return "code"
112
-
113
- return "natural_language"
114
-
115
- return "unknown"
116
-
117
-
118
- def should_compress(filepath: Path) -> bool:
119
- """Return True if the file is natural language and should be compressed."""
120
- if not filepath.is_file():
121
- return False
122
- # Skip backup files
123
- if filepath.name.endswith(".original.md"):
124
- return False
125
- return detect_file_type(filepath) == "natural_language"
126
-
127
-
128
- if __name__ == "__main__":
129
- import sys
130
-
131
- if len(sys.argv) < 2:
132
- print("Usage: python detect.py <file1> [file2] ...")
133
- sys.exit(1)
134
-
135
- for path_str in sys.argv[1:]:
136
- p = Path(path_str).resolve()
137
- file_type = detect_file_type(p)
138
- compress = should_compress(p)
139
- print(f" {p.name:30s} type={file_type:20s} compress={compress}")