@izhimu/pi-caveman 0.2.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,505 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Caveman Memory Compression Orchestrator
4
-
5
- Usage:
6
- python scripts/compress.py <filepath>
7
- """
8
-
9
- import os
10
- import re
11
- import shutil
12
- import stat
13
- import subprocess
14
- import sys
15
- import tempfile
16
- from pathlib import Path
17
- from typing import List
18
-
19
- # Windows consoles default to cp1252, which cannot encode the emoji glyphs in
20
- # our status lines; replace unencodable characters instead of crashing.
21
- for _stream in (sys.stdout, sys.stderr):
22
- try:
23
- _stream.reconfigure(errors="replace")
24
- except Exception:
25
- pass
26
-
27
- # A fence marker at the start of a line, at CommonMark's 0-3 space indent.
28
- FENCE_LINE_REGEX = re.compile(r"^\s{0,3}(`{3,}|~{3,})")
29
-
30
- # YAML frontmatter: starts at file start with --- on its own line, ends with --- on its own line.
31
- # Captures the entire block (including delimiters and trailing newline) and the body after.
32
- FRONTMATTER_REGEX = re.compile(
33
- r"\A(---\r?\n.*?\r?\n---\r?\n)(.*)", re.DOTALL
34
- )
35
-
36
-
37
- def split_frontmatter(text: str):
38
- """Split YAML frontmatter from body. Returns (frontmatter, body).
39
-
40
- Memory files (and many other markdown docs) start with a YAML frontmatter
41
- block delimited by `---` lines. The compression LLM has a habit of stripping
42
- or rewriting these despite preserve-structure rules in the prompt — so we
43
- surgically remove the frontmatter before compression and prepend it back
44
- verbatim to the output. Files without frontmatter pass through unchanged.
45
- """
46
- m = FRONTMATTER_REGEX.match(text)
47
- if m:
48
- return m.group(1), m.group(2)
49
- return "", text
50
-
51
- # Filenames and paths that almost certainly hold secrets or PII. Compressing
52
- # them ships raw bytes to the Anthropic API — a third-party data boundary that
53
- # developers on sensitive codebases cannot cross. detect.py already skips .env
54
- # by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
55
- # slip through the natural-language filter. This is a hard refuse before read.
56
- SENSITIVE_BASENAME_REGEX = re.compile(
57
- r"(?ix)^("
58
- r"\.env(\..+)?"
59
- r"|\.netrc"
60
- r"|credentials(\..+)?"
61
- r"|secrets?(\..+)?"
62
- r"|passwords?(\..+)?"
63
- r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
64
- r"|authorized_keys"
65
- r"|known_hosts"
66
- r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
67
- r")$"
68
- )
69
-
70
- SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
71
-
72
- SENSITIVE_NAME_TOKENS = (
73
- "secret", "credential", "password", "passwd",
74
- "apikey", "accesskey", "token", "privatekey",
75
- )
76
-
77
-
78
- def backup_dir_for(filepath: Path) -> Path:
79
- """Resolve the out-of-tree backup directory for a given source file.
80
-
81
- Backups must live OUTSIDE the source directory so skill auto-loaders
82
- (Claude Code rules/, opencode instructions/, etc.) stop re-ingesting the
83
- `.original.md` copies as live files. Base dir is platform-aware:
84
- - Windows: %LOCALAPPDATA%\\caveman-compress\\backups
85
- - else: $XDG_DATA_HOME/caveman-compress/backups if set,
86
- else ~/.local/share/caveman-compress/backups
87
-
88
- The source file's parent-dir name is mirrored under the base to reduce
89
- cross-project collisions (e.g. two `task.md` files in different repos).
90
- """
91
- if os.name == "nt" or sys.platform == "win32":
92
- local_appdata = os.environ.get("LOCALAPPDATA")
93
- base = Path(local_appdata) if local_appdata else Path.home() / "AppData" / "Local"
94
- base = base / "caveman-compress" / "backups"
95
- else:
96
- xdg = os.environ.get("XDG_DATA_HOME")
97
- base = Path(xdg) if xdg else Path.home() / ".local" / "share"
98
- base = base / "caveman-compress" / "backups"
99
- return base / filepath.parent.name
100
-
101
-
102
- def is_sensitive_path(filepath: Path) -> bool:
103
- """Heuristic denylist for files that must never be shipped to a third-party API."""
104
- name = filepath.name
105
- if SENSITIVE_BASENAME_REGEX.match(name):
106
- return True
107
- lowered_parts = {p.lower() for p in filepath.parts}
108
- if lowered_parts & SENSITIVE_PATH_COMPONENTS:
109
- return True
110
- # Normalize separators so "api-key" and "api_key" both match "apikey".
111
- lower = re.sub(r"[_\-\s.]", "", name.lower())
112
- return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
113
-
114
-
115
- def strip_llm_wrapper(text: str) -> str:
116
- r"""Strip an outer ```markdown ... ``` fence when it wraps the ENTIRE output.
117
-
118
- The wrapper is only real when the first and last fence lines are the SAME
119
- block. The old regex (``\A\s*(fence)[^\n]*\n(.*)\n\1\s*\Z`` with DOTALL and
120
- a greedy ``.*``) never checked that: it matched any document that merely
121
- STARTS and ENDS with a fence line. An ordinary README section —
122
- ```bash npm install``` , prose, ```bash npm test``` — came back with its
123
- first and last fence markers deleted and its two code blocks merged into
124
- prose, so validation failed on both the compress and the fix path and the
125
- section was permanently uncompressible after three paid API calls.
126
- """
127
- lines = text.split("\n")
128
- first, last = 0, len(lines) - 1
129
- while first < len(lines) and not lines[first].strip():
130
- first += 1
131
- while last > first and not lines[last].strip():
132
- last -= 1
133
- if first >= last:
134
- return text
135
- opener = FENCE_LINE_REGEX.match(lines[first])
136
- closer = FENCE_LINE_REGEX.match(lines[last])
137
- if not opener or not closer:
138
- return text
139
- marker = opener.group(1)
140
- # Closing fence: same character, at least as long, and nothing else on the line.
141
- if closer.group(1)[0] != marker[0] or len(closer.group(1)) < len(marker):
142
- return text
143
- if lines[last].strip() != closer.group(1):
144
- return text
145
- # Any fence of the same kind in between means these two are not one block.
146
- for line in lines[first + 1:last]:
147
- inner = FENCE_LINE_REGEX.match(line)
148
- if inner and inner.group(1)[0] == marker[0] and len(inner.group(1)) >= len(marker):
149
- return text
150
- return "\n".join(lines[first + 1:last])
151
-
152
-
153
- def write_text_atomic(path: Path, text: str, newline: str = "\n") -> None:
154
- """Write ``text`` to ``path`` atomically as UTF-8.
155
-
156
- Path.write_text() truncates the destination before encoding the string —
157
- a UnicodeEncodeError (or any other failure) partway through leaves a
158
- 0-byte file, destroying whatever was there before (issue #655). Encode
159
- first, write the bytes to a sibling temp file, fsync, then os.replace()
160
- so the destination only ever moves from one complete, valid file to
161
- another. Preserves the original file's permission bits across the swap.
162
-
163
- ``newline`` is the line terminator to emit. Callers pass the terminator
164
- read_source() found in the source file so a CRLF document stays CRLF —
165
- text-mode writes translating LF to the platform default rewrote every
166
- line ending in every file the tool touched (issue #762), and reading the
167
- bytes ourselves means nothing translates them back.
168
- """
169
- if newline != "\n":
170
- # Normalise first: model output can already carry CRLF, and a bare
171
- # "\n" -> "\r\n" replace would turn those into "\r\r\n".
172
- text = text.replace("\r\n", "\n").replace("\n", newline)
173
- write_bytes_atomic(path, text.encode("utf-8"))
174
-
175
-
176
- def write_bytes_atomic(path: Path, data: bytes) -> None:
177
- """Write ``data`` to ``path`` atomically, preserving permission bits."""
178
- fd, tmp_name = tempfile.mkstemp(
179
- dir=str(path.parent), prefix=path.name + ".", suffix=".tmp"
180
- )
181
- tmp_path = Path(tmp_name)
182
- try:
183
- with os.fdopen(fd, "wb") as f:
184
- f.write(data)
185
- f.flush()
186
- os.fsync(f.fileno())
187
- if path.exists():
188
- os.chmod(tmp_path, stat.S_IMODE(path.stat().st_mode))
189
- os.replace(tmp_path, path)
190
- except Exception:
191
- try:
192
- tmp_path.unlink()
193
- except OSError:
194
- pass
195
- raise
196
-
197
-
198
- def read_source(filepath: Path) -> tuple[str, str, bytes]:
199
- """Read a source file as UTF-8, returning (text, line_terminator, raw_bytes).
200
-
201
- Decodes strictly. The old errors="ignore" silently DROPPED every byte that
202
- was not valid UTF-8 — a cp1252-authored file holding `\xe9` for "e-acute"
203
- lost that byte, the mangled text was what got written to the backup, the
204
- backup readback compared mangled-to-mangled so verification passed, and
205
- then the original was overwritten. The bytes were unrecoverable and
206
- nothing reported a problem (the destructive form of issue #686). A file we
207
- cannot read exactly is a file we must not rewrite.
208
-
209
- Line endings are detected from the raw bytes and returned to the caller
210
- rather than being universal-newline'd away, so write_text_atomic can put
211
- back what was there (issue #762). A mixed-ending file takes the terminator
212
- the majority of its lines use — presence of one CRLF is not a mandate to
213
- rewrite every LF in the document. The raw bytes come back too, so the
214
- backup can be a byte-for-byte copy rather than a re-rendering.
215
- """
216
- raw = filepath.read_bytes()
217
- try:
218
- text = raw.decode("utf-8")
219
- except UnicodeDecodeError as e:
220
- raise ValueError(
221
- f"Refusing to compress {filepath}: not valid UTF-8 "
222
- f"(byte 0x{raw[e.start]:02x} at offset {e.start}). "
223
- "Compression rewrites the file in place, and any byte this tool "
224
- "cannot decode would be destroyed by the round trip. "
225
- "Convert the file to UTF-8 first."
226
- ) from None
227
- crlf = text.count("\r\n")
228
- newline = "\r\n" if crlf * 2 > text.count("\n") else "\n"
229
- return text.replace("\r\n", "\n").replace("\r", "\n"), newline, raw
230
-
231
-
232
- def first_nonblank_line(text: str) -> str:
233
- """Return the first non-blank line, stripped — used to detect a prose
234
- preamble smuggled in ahead of the real content (issue #588)."""
235
- for line in text.splitlines():
236
- if line.strip():
237
- return line.strip()
238
- return ""
239
-
240
-
241
- def _write_target(filepath: Path, text: str | bytes, backup_path: Path, newline: str = "\n") -> None:
242
- """Write to the target file, surfacing the backup location if the write
243
- itself fails. write_text_atomic already leaves the target untouched on
244
- failure, but the caller still needs to know where the pre-compression
245
- original lives instead of being left to guess (issue #652).
246
-
247
- ``bytes`` restore the source verbatim; ``str`` is model output that still
248
- has to be rendered with the document's line terminator."""
249
- try:
250
- if isinstance(text, bytes):
251
- write_bytes_atomic(filepath, text)
252
- else:
253
- write_text_atomic(filepath, text, newline)
254
- except Exception:
255
- print(f"❌ Write to {filepath} failed. Original preserved at backup: {backup_path}")
256
- raise
257
-
258
-
259
- from .detect import should_compress
260
- from .validate import validate
261
-
262
- MAX_RETRIES = 2
263
-
264
-
265
- # ---------- Claude Calls ----------
266
-
267
-
268
- def call_claude(prompt: str) -> str:
269
- """Send a prompt to Claude.
270
-
271
- Prefers the Anthropic SDK when ANTHROPIC_API_KEY is set; otherwise falls
272
- back to the ``claude --print`` CLI (which handles desktop auth).
273
-
274
- On Windows the CLI subprocess decoding defaults to the system codepage
275
- (cp1251 / cp1252) and crashes on UTF-8 output — see issue #152. Pinning
276
- ``encoding="utf-8"`` with ``errors="replace"`` matches the CLI's actual
277
- native I/O and prevents the UnicodeDecodeError before validation can
278
- report. Windows users with non-ASCII content can also set
279
- ``ANTHROPIC_API_KEY`` to route through the SDK and skip the subprocess.
280
- """
281
- api_key = os.environ.get("ANTHROPIC_API_KEY")
282
- if api_key:
283
- try:
284
- import anthropic
285
-
286
- client = anthropic.Anthropic(api_key=api_key)
287
- msg = client.messages.create(
288
- model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
289
- max_tokens=8192,
290
- messages=[{"role": "user", "content": prompt}],
291
- )
292
- return strip_llm_wrapper(msg.content[0].text.strip())
293
- except ImportError:
294
- pass # anthropic not installed, fall back to CLI
295
- # Fallback: use claude CLI (handles desktop auth).
296
- # Resolve binary via shutil.which so Windows .cmd/.bat shims (e.g.
297
- # %APPDATA%\npm\claude.CMD) work without shell=True. On POSIX,
298
- # shutil.which returns the same absolute path as the implicit lookup,
299
- # so this is a no-op there. Falls back to bare "claude" if not found
300
- # on PATH so subprocess raises a clear FileNotFoundError.
301
- claude_bin = shutil.which("claude") or "claude"
302
- try:
303
- result = subprocess.run(
304
- [claude_bin, "--print"],
305
- input=prompt,
306
- text=True,
307
- capture_output=True,
308
- check=True,
309
- encoding="utf-8",
310
- errors="replace",
311
- )
312
- return strip_llm_wrapper(result.stdout.strip())
313
- except subprocess.CalledProcessError as e:
314
- raise RuntimeError(f"Claude call failed:\n{e.stderr}")
315
-
316
-
317
- def build_compress_prompt(original: str) -> str:
318
- return f"""
319
- Compress this markdown into caveman format.
320
-
321
- STRICT RULES:
322
- - Do NOT modify anything inside ``` code blocks
323
- - Do NOT modify anything inside a 4-space-indented code block either — those are code too, and they are validated
324
- - Do NOT modify anything inside inline backticks
325
- - Preserve ALL URLs exactly
326
- - Preserve ALL headings exactly
327
- - Preserve file paths and commands
328
- - Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
329
-
330
- Only compress natural language.
331
-
332
- TEXT:
333
- {original}
334
- """
335
-
336
-
337
- def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str:
338
- errors_str = "\n".join(f"- {e}" for e in errors)
339
- return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
340
-
341
- CRITICAL RULES:
342
- - DO NOT recompress or rephrase the file
343
- - ONLY fix the listed errors — leave everything else exactly as-is
344
- - The ORIGINAL is provided as reference only (to restore missing content)
345
- - Preserve caveman style in all untouched sections
346
-
347
- ERRORS TO FIX:
348
- {errors_str}
349
-
350
- HOW TO FIX:
351
- - Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
352
- - Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
353
- - Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
354
- - Do not touch any section not mentioned in the errors
355
-
356
- ORIGINAL (reference only):
357
- {original}
358
-
359
- COMPRESSED (fix this):
360
- {compressed}
361
-
362
- Return ONLY the fixed compressed file. No explanation.
363
- """
364
-
365
-
366
- # ---------- Core Logic ----------
367
-
368
-
369
- def compress_file(filepath: Path) -> bool:
370
- # Resolve and validate path
371
- filepath = filepath.resolve()
372
- MAX_FILE_SIZE = 500_000 # 500KB
373
- if not filepath.exists():
374
- raise FileNotFoundError(f"File not found: {filepath}")
375
- if filepath.stat().st_size > MAX_FILE_SIZE:
376
- raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
377
-
378
- # Refuse files that look like they contain secrets or PII. Compressing ships
379
- # the raw bytes to the Anthropic API — a third-party boundary — so we fail
380
- # loudly rather than silently exfiltrate credentials or keys. Override is
381
- # intentional: the user must rename the file if the heuristic is wrong.
382
- if is_sensitive_path(filepath):
383
- raise ValueError(
384
- f"Refusing to compress {filepath}: filename looks sensitive "
385
- "(credentials, keys, secrets, or known private paths). "
386
- "Compression sends file contents to the Anthropic API. "
387
- "Rename the file if this is a false positive."
388
- )
389
-
390
- print(f"Processing: {filepath}")
391
-
392
- if not should_compress(filepath):
393
- print("Skipping (not natural language)")
394
- return False
395
-
396
- original_text, newline, original_raw = read_source(filepath)
397
- # Store backup outside the source directory so skill auto-loaders don't
398
- # re-ingest the `.original.md` copy as a live file. Mirror the source's
399
- # parent-dir name + stem under a platform-aware base to reduce collisions.
400
- backup_dir = backup_dir_for(filepath)
401
- backup_path = backup_dir / (filepath.stem + ".original.md")
402
-
403
- if not original_text.strip():
404
- print("❌ Refusing to compress: file is empty or whitespace-only.")
405
- return False
406
-
407
- # Check if backup already exists to prevent accidental overwriting
408
- if backup_path.exists():
409
- print(f"⚠️ Backup file already exists: {backup_path}")
410
- print("The original backup may contain important content.")
411
- print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
412
- return False
413
-
414
- # Split YAML frontmatter off before compression. Claude tends to strip or
415
- # rewrite frontmatter despite preserve-structure rules; we keep it verbatim
416
- # by removing it from the input and re-prepending it to the output.
417
- frontmatter, body = split_frontmatter(original_text)
418
- if frontmatter:
419
- print(f"Detected YAML frontmatter ({len(frontmatter)} chars) — preserving verbatim")
420
-
421
- if not body.strip():
422
- print("❌ Refusing to compress: body is empty after frontmatter removal.")
423
- return False
424
-
425
- # Step 1: Compress (body only, frontmatter excluded)
426
- print("Compressing with Claude...")
427
- compressed_body = call_claude(build_compress_prompt(body))
428
-
429
- if compressed_body is None or not compressed_body.strip():
430
- print("❌ Compression aborted: Claude returned an empty response.")
431
- print(" Original file is untouched (no backup created).")
432
- return False
433
-
434
- # Compare the BODY (not the whole file) — frontmatter is preserved verbatim
435
- # and would never change, so identity must be judged on the compressible part.
436
- if compressed_body.strip() == body.strip():
437
- print("❌ Compression aborted: output is identical to input.")
438
- print(" Likely causes: Claude refused, returned the prompt verbatim, or the file is")
439
- print(" already in caveman form. Original file is untouched (no backup created).")
440
- return False
441
-
442
- # Reassemble: frontmatter (verbatim) + compressed body
443
- compressed = frontmatter + compressed_body
444
-
445
- # Save original as backup, then verify the backup readback before
446
- # touching the input file. If the filesystem dropped bytes (encoding,
447
- # antivirus, disk full), unlink the bad backup and abort instead of
448
- # leaving the user with a corrupt backup + compressed primary.
449
- backup_dir.mkdir(parents=True, exist_ok=True)
450
- write_bytes_atomic(backup_path, original_raw)
451
- if backup_path.read_bytes() != original_raw:
452
- print(f"❌ Backup write verification failed: {backup_path}")
453
- print(" In-memory original differs from on-disk backup. Aborting before touching the input file.")
454
- try:
455
- backup_path.unlink()
456
- except OSError:
457
- pass
458
- return False
459
- _write_target(filepath, compressed, backup_path, newline)
460
-
461
- # Step 2: Validate + Retry
462
- for attempt in range(MAX_RETRIES):
463
- print(f"\nValidation attempt {attempt + 1}")
464
-
465
- result = validate(backup_path, filepath)
466
-
467
- if result.is_valid:
468
- print("Validation passed")
469
- break
470
-
471
- print("❌ Validation failed:")
472
- for err in result.errors:
473
- print(f" - {err}")
474
-
475
- if attempt == MAX_RETRIES - 1:
476
- # Restore original on failure
477
- _write_target(filepath, original_raw, backup_path, newline)
478
- backup_path.unlink(missing_ok=True)
479
- print("❌ Failed after retries — original restored")
480
- return False
481
-
482
- print("Fixing with Claude...")
483
- compressed = call_claude(
484
- build_fix_prompt(original_text, compressed, result.errors)
485
- )
486
-
487
- if compressed is None or not compressed.strip():
488
- print("❌ Fix attempt aborted: Claude returned an empty response.")
489
- print(" Skipping this attempt.")
490
- continue
491
-
492
- # Guard against a prose preamble smuggled in ahead of the real fixed
493
- # content (issue #588). Only enforced when the original starts with a
494
- # structural anchor (frontmatter `---` or a heading) — plain-prose
495
- # first lines get legitimately rewritten by compression, and requiring
496
- # them verbatim would reject every valid fix.
497
- anchor = first_nonblank_line(original_text)
498
- if anchor.startswith(("---", "#")) and first_nonblank_line(compressed) != anchor:
499
- print("❌ Fix attempt aborted: output does not start with the original's first line.")
500
- print(" Possible preamble leak. Skipping this attempt.")
501
- continue
502
-
503
- _write_target(filepath, compressed, backup_path, newline)
504
-
505
- return True
@@ -1,139 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Detect whether a file is natural language (compressible) or code/config (skip)."""
3
-
4
- import json
5
- import re
6
- from pathlib import Path
7
-
8
- # Extensions that are natural language and compressible
9
- COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
10
-
11
- # Extensions that are code/config and should be skipped
12
- SKIP_EXTENSIONS = {
13
- ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
14
- ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
15
- ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
16
- ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
17
- ".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
18
- }
19
-
20
- # Well-known build/config files that carry no (or a misleading) extension —
21
- # `Dockerfile` has no suffix so `.dockerfile` above never matches it, and
22
- # `CMakeLists.txt` would ride the compressible `.txt` rule. Checked by
23
- # basename before any extension rule.
24
- KNOWN_CODE_FILENAMES = {
25
- "dockerfile", "makefile", "gnumakefile", "jenkinsfile", "vagrantfile",
26
- "rakefile", "gemfile", "justfile", "procfile", "brewfile",
27
- "cmakelists.txt",
28
- }
29
-
30
- # Patterns that indicate a line is code
31
- CODE_PATTERNS = [
32
- re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
33
- re.compile(r"^\s*(def |class |function |async function |export )"),
34
- re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
35
- re.compile(r"^\s*[\}\]\);]+\s*$"), # closing braces/brackets
36
- re.compile(r"^\s*@\w+"), # decorators/annotations
37
- re.compile(r'^\s*"[^"]+"\s*:\s*'), # JSON-like key-value
38
- re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"), # assignment with literal
39
- ]
40
-
41
-
42
- def _is_code_line(line: str) -> bool:
43
- """Check if a line looks like code."""
44
- return any(p.match(line) for p in CODE_PATTERNS)
45
-
46
-
47
- def _is_json_content(text: str) -> bool:
48
- """Check if content is valid JSON."""
49
- try:
50
- json.loads(text)
51
- return True
52
- except (json.JSONDecodeError, ValueError):
53
- return False
54
-
55
-
56
- def _is_yaml_content(lines: list[str]) -> bool:
57
- """Heuristic: check if content looks like YAML."""
58
- yaml_indicators = 0
59
- for line in lines[:30]:
60
- stripped = line.strip()
61
- if stripped.startswith("---"):
62
- yaml_indicators += 1
63
- elif re.match(r"^\w[\w\s]*:\s", stripped):
64
- yaml_indicators += 1
65
- elif stripped.startswith("- ") and ":" in stripped:
66
- yaml_indicators += 1
67
- # If most non-empty lines look like YAML
68
- non_empty = sum(1 for l in lines[:30] if l.strip())
69
- return non_empty > 0 and yaml_indicators / non_empty > 0.6
70
-
71
-
72
- def detect_file_type(filepath: Path) -> str:
73
- """Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
74
-
75
- Returns:
76
- One of: 'natural_language', 'code', 'config', 'unknown'
77
- """
78
- ext = filepath.suffix.lower()
79
-
80
- # Known code filenames win over any extension rule
81
- if filepath.name.lower() in KNOWN_CODE_FILENAMES:
82
- return "code"
83
-
84
- # Extension-based classification
85
- if ext in COMPRESSIBLE_EXTENSIONS:
86
- return "natural_language"
87
- if ext in SKIP_EXTENSIONS:
88
- return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
89
-
90
- # Extensionless files (like CLAUDE.md, TODO) — check content
91
- if not ext:
92
- try:
93
- text = filepath.read_text(encoding="utf-8", errors="ignore")
94
- except (OSError, PermissionError):
95
- return "unknown"
96
-
97
- lines = text.splitlines()[:50]
98
-
99
- # Shebang means executable script, never prose
100
- if text.startswith("#!"):
101
- return "code"
102
-
103
- if _is_json_content(text[:10000]):
104
- return "config"
105
- if _is_yaml_content(lines):
106
- return "config"
107
-
108
- code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
109
- non_empty = sum(1 for l in lines if l.strip())
110
- if non_empty > 0 and code_lines / non_empty > 0.4:
111
- return "code"
112
-
113
- return "natural_language"
114
-
115
- return "unknown"
116
-
117
-
118
- def should_compress(filepath: Path) -> bool:
119
- """Return True if the file is natural language and should be compressed."""
120
- if not filepath.is_file():
121
- return False
122
- # Skip backup files
123
- if filepath.name.endswith(".original.md"):
124
- return False
125
- return detect_file_type(filepath) == "natural_language"
126
-
127
-
128
- if __name__ == "__main__":
129
- import sys
130
-
131
- if len(sys.argv) < 2:
132
- print("Usage: python detect.py <file1> [file2] ...")
133
- sys.exit(1)
134
-
135
- for path_str in sys.argv[1:]:
136
- p = Path(path_str).resolve()
137
- file_type = detect_file_type(p)
138
- compress = should_compress(p)
139
- print(f" {p.name:30s} type={file_type:20s} compress={compress}")