@event4u/agent-config 2.20.0 → 2.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,172 @@
1
+ #!/usr/bin/env python3
2
+ """Input-side memory compression — Phase 2 of step-16-caveman-substance.
3
+
4
+ Rewrites memory files (AGENTS.md, CLAUDE.md, .cursorrules, ...) to caveman
5
+ grammar (drop articles / auxiliaries) while preserving carve-outs byte-for-byte
6
+ (code blocks, numbered-options, status markers, Iron-Law ALL-CAPS, backtick
7
+ spans). Writes `.original.md` backup before mutating. Gated by Phase 0
8
+ `validate_safe_paths.assert_safe`. Idempotency guard: `original_sha256:` +
9
+ `compressed_at:` frontmatter refuse re-compression on body-hash drift.
10
+
11
+ CLI: `compress_memory.py <path> [--check|--decompress]`. Stdlib-only.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import argparse
16
+ import hashlib
17
+ import re
18
+ import sys
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+
22
+ REPO_ROOT = Path(__file__).resolve().parent.parent
23
+ sys.path.insert(0, str(REPO_ROOT / "scripts"))
24
+
25
+ from validate_safe_paths import SensitivePathError, assert_safe # noqa: E402
26
+
27
+ __all__ = ["compress_text", "compress_file", "decompress_file", "CompressionRefused"]
28
+
29
+
30
+ class CompressionRefused(RuntimeError):
31
+ """Raised when the target is already compressed and body hash diverged."""
32
+
33
+
34
+ # Carve-out region patterns — mirrors caveman-speak.md § Carve-outs (1–7).
35
+ RE_FENCE = re.compile(r"^```")
36
+ RE_NUMBERED = re.compile(r"^>?\s*\d+\.\s")
37
+ RE_STATUS = re.compile(r"^\s*(?:❌|⚠️|✅)")
38
+ RE_IRONLAW = re.compile(r"^[A-Z][A-Z0-9 ,.\-_/']{3,}$")
39
+ RE_BACKTICK_SPAN = re.compile(r"`[^`\n]+`")
40
+ RE_FRONTMATTER = re.compile(r"^---\s*$")
41
+ WORD_RE = re.compile(r"\b[A-Za-z]+\b")
42
+ DROP_TOKENS = {"the", "a", "an", "is", "are", "was", "were", "be", "been",
43
+ "being", "that", "which"}
44
+
45
+
46
+ def _compress_words(text: str) -> str:
47
+ out = WORD_RE.sub(lambda m: "" if m.group(0).lower() in DROP_TOKENS else m.group(0), text)
48
+ out = re.sub(r"[ \t]{2,}", " ", out)
49
+ return re.sub(r" +([,.;:!?])", r"\1", out)
50
+
51
+
52
+ def _compress_prose_line(line: str) -> str:
53
+ """Compress a prose line; preserve backtick-spans byte-for-byte."""
54
+ parts: list[str] = []
55
+ last = 0
56
+ for span in RE_BACKTICK_SPAN.finditer(line):
57
+ parts.append(_compress_words(line[last:span.start()]))
58
+ parts.append(span.group(0))
59
+ last = span.end()
60
+ parts.append(_compress_words(line[last:]))
61
+ return "".join(parts)
62
+
63
+
64
+ def compress_text(body: str) -> str:
65
+ """Compress a memory-file body. Idempotent on already-caveman text."""
66
+ out: list[str] = []
67
+ in_fence = False
68
+ for raw in body.splitlines(keepends=True):
69
+ stripped = raw.rstrip("\r\n")
70
+ if RE_FENCE.match(stripped):
71
+ in_fence = not in_fence
72
+ out.append(raw)
73
+ continue
74
+ if in_fence or RE_NUMBERED.match(stripped) or RE_STATUS.match(stripped) \
75
+ or RE_IRONLAW.match(stripped.strip()):
76
+ out.append(raw)
77
+ continue
78
+ out.append(_compress_prose_line(raw))
79
+ return "".join(out)
80
+
81
+
82
+ def _split_frontmatter(text: str) -> tuple[str, str]:
83
+ lines = text.splitlines(keepends=True)
84
+ if not lines or not RE_FRONTMATTER.match(lines[0].rstrip()):
85
+ return "", text
86
+ for idx in range(1, len(lines)):
87
+ if RE_FRONTMATTER.match(lines[idx].rstrip()):
88
+ return "".join(lines[: idx + 1]), "".join(lines[idx + 1:])
89
+ return "", text
90
+
91
+
92
+ def _sha256(text: str) -> str:
93
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
94
+
95
+
96
+ def _has_sha_marker(fm: str) -> bool:
97
+ return bool(re.search(r"^original_sha256:\s*[0-9a-f]{64}\s*$", fm, re.MULTILINE))
98
+
99
+
100
+ def _inject_frontmatter(fm: str, sha: str, ts: str) -> str:
101
+ drop = re.compile(r"^(original_sha256|compressed_at):.*$", re.MULTILINE)
102
+ inner = drop.sub("", fm.strip().strip("-").strip()).strip() if fm else ""
103
+ body = inner + ("\n" if inner else "")
104
+ return f"---\n{body}original_sha256: {sha}\ncompressed_at: {ts}\n---\n"
105
+
106
+
107
+ def _backup_path(target: Path) -> Path:
108
+ return target.parent / (target.name + ".original.md")
109
+
110
+
111
+ def compress_file(target: Path) -> Path:
112
+ assert_safe(target)
113
+ text = target.read_text(encoding="utf-8")
114
+ fm, body = _split_frontmatter(text)
115
+ if _has_sha_marker(fm):
116
+ if _sha256(compress_text(body)) != _sha256(body):
117
+ raise CompressionRefused(
118
+ f"{target}: body hash diverged; decompress first "
119
+ f"(`scripts/compress_memory.py {target} --decompress`)."
120
+ )
121
+ return target
122
+ backup = _backup_path(target)
123
+ backup.write_text(text, encoding="utf-8")
124
+ ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
125
+ target.write_text(
126
+ _inject_frontmatter(fm, _sha256(body), ts) + compress_text(body),
127
+ encoding="utf-8",
128
+ )
129
+ return backup
130
+
131
+
132
+ def decompress_file(target: Path) -> Path:
133
+ assert_safe(target)
134
+ backup = _backup_path(target)
135
+ if not backup.is_file():
136
+ raise FileNotFoundError(f"no backup at {backup}")
137
+ target.write_text(backup.read_text(encoding="utf-8"), encoding="utf-8")
138
+ backup.unlink()
139
+ return target
140
+
141
+
142
+ def _main(argv: list[str]) -> int:
143
+ ap = argparse.ArgumentParser(description="Compress memory files to caveman grammar.")
144
+ ap.add_argument("path", type=Path)
145
+ grp = ap.add_mutually_exclusive_group()
146
+ grp.add_argument("--check", action="store_true", help="exit 0 if safe; no writes")
147
+ grp.add_argument("--decompress", action="store_true", help="restore .original.md")
148
+ args = ap.parse_args(argv)
149
+ try:
150
+ if args.check:
151
+ assert_safe(args.path)
152
+ return 0
153
+ if args.decompress:
154
+ decompress_file(args.path)
155
+ print(f"decompressed: {args.path}")
156
+ return 0
157
+ backup = compress_file(args.path)
158
+ print(f"compressed: {args.path} (backup: {backup})")
159
+ return 0
160
+ except SensitivePathError as exc:
161
+ print(f"error: refused: {exc}", file=sys.stderr)
162
+ return 2
163
+ except CompressionRefused as exc:
164
+ print(f"error: {exc}", file=sys.stderr)
165
+ return 3
166
+ except FileNotFoundError as exc:
167
+ print(f"error: {exc}", file=sys.stderr)
168
+ return 4
169
+
170
+
171
+ if __name__ == "__main__":
172
+ raise SystemExit(_main(sys.argv[1:]))
@@ -0,0 +1,78 @@
1
+ #!/usr/bin/env python3
2
+ """Group cost-tracking sessions by conversation_id (Ruflo `conversation.mjs` `5b71c7a` ref)."""
3
+ from __future__ import annotations
4
+ import argparse, json, sys
5
+ from collections import defaultdict
6
+ from pathlib import Path
7
+
8
+ REPO_ROOT = Path(__file__).resolve().parent.parent
9
+ DEFAULT_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
10
+
11
+
12
+ def _load(path: Path) -> list[dict]:
13
+ if not path.is_file():
14
+ return []
15
+ out = []
16
+ for line in path.read_text(encoding="utf-8").splitlines():
17
+ s = line.strip()
18
+ if not s or s.startswith("#"):
19
+ continue
20
+ try:
21
+ out.append(json.loads(s))
22
+ except json.JSONDecodeError:
23
+ continue
24
+ return out
25
+
26
+
27
+ def group(rows: list[dict]) -> dict:
28
+ by_conv: dict = defaultdict(lambda: {
29
+ "sessions": 0, "total_cost_usd": 0.0, "input_tokens": 0,
30
+ "output_tokens": 0, "caveman_delta_tokens": 0,
31
+ "by_model": defaultdict(lambda: {"sessions": 0, "cost_usd": 0.0}),
32
+ })
33
+ for row in rows:
34
+ cid = str(row.get("conversation_id") or "unknown")
35
+ b = by_conv[cid]
36
+ cost = float(row.get("total_cost_usd") or 0)
37
+ b["sessions"] += 1
38
+ b["total_cost_usd"] += cost
39
+ b["input_tokens"] += int(row.get("input_tokens") or 0)
40
+ b["output_tokens"] += int(row.get("output_tokens") or 0)
41
+ b["caveman_delta_tokens"] += int(row.get("caveman_delta_tokens") or 0)
42
+ m = b["by_model"][str(row.get("model") or "unknown")]
43
+ m["sessions"] += 1
44
+ m["cost_usd"] += cost
45
+ return {cid: {**b, "by_model": dict(b["by_model"])} for cid, b in by_conv.items()}
46
+
47
+
48
+ def render_text(report: dict) -> str:
49
+ if not report:
50
+ return "cost-by-conversation: no rows.\n"
51
+ lines = ["cost-by-conversation lens · grouped by conversation_id", ""]
52
+ for cid, b in sorted(report.items()):
53
+ lines.append(
54
+ f" {cid}: {b['sessions']} sessions · ${b['total_cost_usd']:.4f} · "
55
+ f"in {b['input_tokens']:,} · out {b['output_tokens']:,} · "
56
+ f"caveman_delta {b['caveman_delta_tokens']:+,}"
57
+ )
58
+ for model, m in sorted(b["by_model"].items()):
59
+ lines.append(f" {model}: {m['sessions']} sessions · ${m['cost_usd']:.4f}")
60
+ return "\n".join(lines) + "\n"
61
+
62
+
63
+ def main(argv: list[str] | None = None) -> int:
64
+ p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
65
+ p.add_argument("--input", type=Path, default=DEFAULT_JSONL)
66
+ p.add_argument("--format", choices=["text", "json"], default="text")
67
+ args = p.parse_args(argv)
68
+ report = group(_load(args.input))
69
+ if args.format == "json":
70
+ print(json.dumps({"schema_version": "cost-by-conversation/v1",
71
+ "by_conversation": report}, indent=2))
72
+ else:
73
+ print(render_text(report))
74
+ return 0
75
+
76
+
77
+ if __name__ == "__main__":
78
+ sys.exit(main())
@@ -0,0 +1,97 @@
1
+ #!/usr/bin/env python3
2
+ """Emit `cost-summary/v1` JSON per `docs/contracts/cost-summary-schema.md`.
3
+
4
+ Reads `agents/cost-tracking/sessions.jsonl` (or `--input`), aggregates by
5
+ session, conversation, and model. Honors the caveman suspended-multiplier
6
+ contract (delta = 0 while suspended; see `caveman-telemetry.md`).
7
+ """
8
+ from __future__ import annotations
9
+ import argparse, json, sys
10
+ from collections import defaultdict
11
+ from datetime import datetime, timezone
12
+ from pathlib import Path
13
+
14
+ REPO_ROOT = Path(__file__).resolve().parent.parent
15
+ DEFAULT_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
16
+ SCHEMA = "cost-summary/v1"
17
+ MULTIPLIER_VERSION = "v1"
18
+ MULTIPLIER_ACTIVE = False
19
+
20
+
21
+ def _load(path: Path) -> list[dict]:
22
+ if not path.is_file():
23
+ return []
24
+ out = []
25
+ for line in path.read_text(encoding="utf-8").splitlines():
26
+ s = line.strip()
27
+ if not s or s.startswith("#"):
28
+ continue
29
+ try:
30
+ out.append(json.loads(s))
31
+ except json.JSONDecodeError:
32
+ continue
33
+ return out
34
+
35
+
36
+ def _delta(row: dict) -> int:
37
+ if not MULTIPLIER_ACTIVE:
38
+ return 0
39
+ return int(row.get("caveman_delta_tokens") or 0)
40
+
41
+
42
+ def _zero_kv() -> dict:
43
+ return {"sessions": 0, "total_cost_usd": 0.0, "input_tokens": 0,
44
+ "output_tokens": 0, "caveman_delta_tokens": 0}
45
+
46
+
47
+ def _zero_model() -> dict:
48
+ return {"sessions": 0, "total_cost_usd": 0.0, "input_tokens": 0, "output_tokens": 0}
49
+
50
+
51
+ def aggregate(rows: list[dict]) -> dict:
52
+ by_sess: dict = defaultdict(_zero_kv)
53
+ by_conv: dict = defaultdict(_zero_kv)
54
+ by_model: dict = defaultdict(_zero_model)
55
+ totals = _zero_kv()
56
+ for row in rows:
57
+ sid = str(row.get("sessionId") or row.get("session_id") or "unknown")
58
+ cid = str(row.get("conversation_id") or "unknown")
59
+ model = str(row.get("model") or "unknown")
60
+ cost = float(row.get("total_cost_usd") or 0)
61
+ itok = int(row.get("input_tokens") or 0)
62
+ otok = int(row.get("output_tokens") or 0)
63
+ delta = _delta(row)
64
+ for bucket in (by_sess[sid], by_conv[cid], totals):
65
+ bucket["sessions"] += 1
66
+ bucket["total_cost_usd"] += cost
67
+ bucket["input_tokens"] += itok
68
+ bucket["output_tokens"] += otok
69
+ bucket["caveman_delta_tokens"] += delta
70
+ m = by_model[model]
71
+ m["sessions"] += 1
72
+ m["total_cost_usd"] += cost
73
+ m["input_tokens"] += itok
74
+ m["output_tokens"] += otok
75
+ totals["caveman_multiplier_version"] = MULTIPLIER_VERSION
76
+ totals["caveman_multiplier_active"] = MULTIPLIER_ACTIVE
77
+ return {
78
+ "schema_version": SCHEMA,
79
+ "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
80
+ "totals": totals,
81
+ "by_session": [{"key": k, **v} for k, v in sorted(by_sess.items())],
82
+ "by_conversation": [{"key": k, **v} for k, v in sorted(by_conv.items())],
83
+ "by_model": [{"model": k, **v} for k, v in sorted(by_model.items())],
84
+ }
85
+
86
+
87
+ def main(argv: list[str] | None = None) -> int:
88
+ p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
89
+ p.add_argument("--input", type=Path, default=DEFAULT_JSONL)
90
+ p.add_argument("--format", choices=["json"], default="json")
91
+ args = p.parse_args(argv)
92
+ print(json.dumps(aggregate(_load(args.input)), indent=2))
93
+ return 0
94
+
95
+
96
+ if __name__ == "__main__":
97
+ sys.exit(main())
@@ -177,8 +177,9 @@ def main() -> int:
177
177
  roadmaps = sorted(REPO_ROOT.glob(ROADMAP_GLOB))
178
178
  horizon_weeks = _read_horizon_weeks()
179
179
  if not roadmaps:
180
- print(f"❌ no roadmaps matched {ROADMAP_GLOB}", file=sys.stderr)
181
- return 1
180
+ if not QUIET:
181
+ print(f"✅ no active roadmaps under {ROADMAP_GLOB} — nothing to lint")
182
+ return 0
182
183
  failed = 0
183
184
  summary: list[tuple[str, str]] = []
184
185
  for roadmap in roadmaps:
@@ -55,11 +55,13 @@ TARGETS: list[tuple[str, list[tuple[str, str]]]] = [
55
55
  [
56
56
  (r"(Browse all )(\d+)( commands\])", "commands"),
57
57
  (r"(package \(rules \+ )(\d+)( skills)", "skills"),
58
- # Hero line: **NNN Skills** · **NNN Rules** · **NNN Commands** · **NNN Guidelines**
59
- (r"(<strong>)(\d+)( Skills</strong>)", "skills"),
60
- (r"(<strong>)(\d+)( Rules</strong>)", "rules"),
61
- (r"(<strong>)(\d+)( Guidelines</strong>)", "guidelines"),
62
- # NOTE: hero `<strong>N Commands</strong>` and tools-blurb
58
+ # Hero badges: shields.io URLs `Skills-NNN-<color>` etc.
59
+ # Format: https://img.shields.io/badge/<Label>-<N>-<hex>?style=flat-square
60
+ (r"(/badge/Skills-)(\d+)(-)", "skills"),
61
+ (r"(/badge/Rules-)(\d+)(-)", "rules"),
62
+ (r"(/badge/Guidelines-)(\d+)(-)", "guidelines"),
63
+ (r"(/badge/Personas-)(\d+)(-)", "personas"),
64
+ # NOTE: hero `Commands-N` badge and tools-blurb
63
65
  # `skills + N native commands` are owned by
64
66
  # `check_command_count_messaging.py` (Phase-1.2 of
65
67
  # road-to-pr-34-followups). Those surfaces advertise the
@@ -0,0 +1,129 @@
1
+ #!/usr/bin/env python3
2
+ """Mechanical carve-out validator for caveman-compressed replies.
3
+
4
+ Given a pre-compression reply and a post-compression reply, assert that
5
+ every carve-out region from `.agent-src.uncompressed/rules/caveman-speak.md`
6
+ § Carve-outs survived byte-for-byte:
7
+
8
+ 1. Triple-backtick code blocks (any language).
9
+ 2. Numbered-option lines (`^>?\\s*\\d+\\.\\s` plus the
10
+ `**Recommendation:**` / `**Empfehlung:**` label).
11
+ 3. Backtick spans (file paths, command names, identifiers).
12
+ 4. Status / error marker lines (prefix `❌`, `⚠️`, `✅`).
13
+ 5. Triple-backtick ALL-CAPS Iron-Law literal fences (subset of (1) —
14
+ reported separately for diagnostics).
15
+
16
+ Stdlib only. Exit 0 = all carve-outs preserved; exit 1 = drift detected.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import argparse
21
+ import difflib
22
+ import re
23
+ import sys
24
+ from pathlib import Path
25
+
26
+ # Triple-backtick fenced blocks (greedy across lines). Group 1 = body.
27
+ RE_CODE_FENCE = re.compile(r"```[^\n]*\n(.*?)\n```", re.DOTALL)
28
+ # Numbered-option line: optional `> ` quote prefix, digits, dot, space.
29
+ RE_NUMBERED = re.compile(r"^>?\s*\d+\.\s.*$", re.MULTILINE)
30
+ # Recommendation labels (both languages).
31
+ RE_RECOMMEND = re.compile(r"^\*\*(Recommendation|Empfehlung):\*\*.*$", re.MULTILINE)
32
+ # Backtick spans — single-tick, non-greedy, no newlines inside.
33
+ RE_BACKTICK_SPAN = re.compile(r"`[^`\n]+`")
34
+ # Status / error marker lines (full line containing the marker).
35
+ RE_STATUS_LINE = re.compile(r"^.*[❌⚠✅].*$", re.MULTILINE)
36
+ # Iron-Law ALL-CAPS fence body — letters + spaces + basic punctuation, ≥ 80 % uppercase.
37
+ RE_ALLCAPS_LINE = re.compile(r"^[A-Z0-9 ,\.\-—:'\"·/\(\)]+$")
38
+
39
+
40
+ def _extract_code_fences(text: str) -> list[str]:
41
+ return [m.group(0) for m in RE_CODE_FENCE.finditer(text)]
42
+
43
+
44
+ def _extract_lines(text: str, pattern: re.Pattern) -> list[str]:
45
+ return [m.group(0) for m in pattern.finditer(text)]
46
+
47
+
48
+ def _extract_backtick_spans(text: str) -> list[str]:
49
+ # Excludes triple-backtick fences (handled separately).
50
+ stripped = RE_CODE_FENCE.sub("", text)
51
+ return RE_BACKTICK_SPAN.findall(stripped)
52
+
53
+
54
+ def _is_allcaps_fence_body(body: str) -> bool:
55
+ lines = [ln.strip() for ln in body.splitlines() if ln.strip()]
56
+ if not lines:
57
+ return False
58
+ return all(RE_ALLCAPS_LINE.match(ln) for ln in lines)
59
+
60
+
61
+ def _extract_allcaps_fences(text: str) -> list[str]:
62
+ out: list[str] = []
63
+ for m in RE_CODE_FENCE.finditer(text):
64
+ if _is_allcaps_fence_body(m.group(1)):
65
+ out.append(m.group(0))
66
+ return out
67
+
68
+
69
+ CHECKS = (
70
+ ("code_fences", _extract_code_fences),
71
+ ("numbered_options", lambda t: _extract_lines(t, RE_NUMBERED)),
72
+ ("recommendation_labels", lambda t: _extract_lines(t, RE_RECOMMEND)),
73
+ ("backtick_spans", _extract_backtick_spans),
74
+ ("status_markers", lambda t: _extract_lines(t, RE_STATUS_LINE)),
75
+ ("allcaps_iron_law_fences", _extract_allcaps_fences),
76
+ )
77
+
78
+
79
+ def validate(pre: str, post: str) -> list[tuple[str, list[str]]]:
80
+ """Return list of (carve_out_name, unified_diff_lines) per drifted category."""
81
+ failures: list[tuple[str, list[str]]] = []
82
+ for name, extractor in CHECKS:
83
+ pre_list = extractor(pre)
84
+ post_list = extractor(post)
85
+ if pre_list == post_list:
86
+ continue
87
+ diff = list(difflib.unified_diff(
88
+ [s + "\n" for s in pre_list],
89
+ [s + "\n" for s in post_list],
90
+ fromfile=f"pre/{name}",
91
+ tofile=f"post/{name}",
92
+ lineterm="",
93
+ ))
94
+ failures.append((name, diff))
95
+ return failures
96
+
97
+
98
+ def _render(failures: list[tuple[str, list[str]]]) -> str:
99
+ out = ["caveman carve-out validator: DRIFT DETECTED", ""]
100
+ for name, diff in failures:
101
+ out.append(f"❌ carve-out `{name}` drifted:")
102
+ out.extend(diff)
103
+ out.append("")
104
+ return "\n".join(out)
105
+
106
+
107
+ def main(argv: list[str] | None = None) -> int:
108
+ p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
109
+ p.add_argument("pre", type=Path, help="Pre-compression reply file.")
110
+ p.add_argument("post", type=Path, help="Post-compression reply file.")
111
+ args = p.parse_args(argv)
112
+ if not args.pre.is_file():
113
+ print(f"pre file not found: {args.pre}", file=sys.stderr)
114
+ return 2
115
+ if not args.post.is_file():
116
+ print(f"post file not found: {args.post}", file=sys.stderr)
117
+ return 2
118
+ pre = args.pre.read_text(encoding="utf-8")
119
+ post = args.post.read_text(encoding="utf-8")
120
+ failures = validate(pre, post)
121
+ if failures:
122
+ print(_render(failures))
123
+ return 1
124
+ print("caveman carve-out validator: all carve-outs preserved ✅")
125
+ return 0
126
+
127
+
128
+ if __name__ == "__main__":
129
+ sys.exit(main())
@@ -0,0 +1,118 @@
1
+ #!/usr/bin/env python3
2
+ """Sensitive-path denylist — refuses files that almost certainly hold secrets or PII.
3
+
4
+ Phase 0 of step-16-caveman-substance. Gates Phase 2 (`scripts/compress_memory.py`):
5
+ any consumer-supplied path must pass `assert_safe()` before bytes are read or
6
+ shipped to a third-party API.
7
+
8
+ Ported from Caveman `plugins/caveman/skills/caveman-compress/scripts/compress.py`
9
+ (upstream `63a91ec`). Adapted to repo conventions: explicit `SensitivePathError`,
10
+ CLI entry point, no `anthropic` import.
11
+
12
+ Public API:
13
+ is_sensitive(path: pathlib.Path) -> bool
14
+ assert_safe(path: pathlib.Path) -> None # raises SensitivePathError
15
+
16
+ CLI:
17
+ python3 scripts/validate_safe_paths.py <path> # exit 0 = safe, 2 = sensitive
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import re
22
+ import sys
23
+ from pathlib import Path
24
+
25
+ __all__ = ["SensitivePathError", "is_sensitive", "assert_safe"]
26
+
27
+
28
+ class SensitivePathError(ValueError):
29
+ """Raised when a path matches the sensitive-file denylist."""
30
+
31
+
32
+ # Filenames that almost certainly hold secrets or PII. Matched against the
33
+ # basename only (case-insensitive). Compressing or shipping these to an LLM API
34
+ # is a third-party data boundary developers on sensitive codebases cannot cross.
35
+ SENSITIVE_BASENAME_REGEX = re.compile(
36
+ r"(?ix)^("
37
+ r"\.env(\..+)?"
38
+ r"|\.netrc"
39
+ r"|credentials(\..+)?"
40
+ r"|secrets?(\..+)?"
41
+ r"|passwords?(\..+)?"
42
+ r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
43
+ r"|authorized_keys"
44
+ r"|known_hosts"
45
+ r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
46
+ r")$"
47
+ )
48
+
49
+ # Path components (any segment, case-insensitive) that mark a sensitive
50
+ # directory. Catches `~/.ssh/known_hosts` even when the basename slips past the
51
+ # regex above.
52
+ SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
53
+
54
+ # Substring tokens checked against the normalised basename (separators stripped
55
+ # so `api-key`, `api_key`, `apikey` all match). Catches creative renames like
56
+ # `prod-secret-token.txt` that bypass the explicit basename regex.
57
+ SENSITIVE_NAME_TOKENS = (
58
+ "secret",
59
+ "credential",
60
+ "password",
61
+ "passwd",
62
+ "apikey",
63
+ "accesskey",
64
+ "token",
65
+ "privatekey",
66
+ )
67
+
68
+ _SEP_STRIP_RE = re.compile(r"[_\-\s.]")
69
+
70
+
71
+ def is_sensitive(path: Path) -> bool:
72
+ """Return True if `path` matches the sensitive-file denylist."""
73
+ name = path.name
74
+ if SENSITIVE_BASENAME_REGEX.match(name):
75
+ return True
76
+ lowered_parts = {p.lower() for p in path.parts}
77
+ if lowered_parts & SENSITIVE_PATH_COMPONENTS:
78
+ return True
79
+ lower = _SEP_STRIP_RE.sub("", name.lower())
80
+ return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
81
+
82
+
83
+ def assert_safe(path: Path) -> None:
84
+ """Raise `SensitivePathError` if `path` matches the denylist.
85
+
86
+ Intended as a hard guard at the top of any function that reads bytes from
87
+ a consumer-supplied path and ships them to a third-party API. Override is
88
+ intentional: the user must rename the file if the heuristic is wrong.
89
+ """
90
+ if is_sensitive(path):
91
+ raise SensitivePathError(
92
+ f"Refusing to operate on {path}: filename or path looks sensitive "
93
+ "(credentials, keys, secrets, or known private directories). "
94
+ "Rename the file if this is a false positive."
95
+ )
96
+
97
+
98
+ def _main(argv: list[str]) -> int:
99
+ if len(argv) != 2 or argv[1] in ("-h", "--help"):
100
+ print(
101
+ "usage: validate_safe_paths.py <path>\n"
102
+ " exit 0 — path is safe\n"
103
+ " exit 2 — path matches the sensitive-file denylist",
104
+ file=sys.stderr,
105
+ )
106
+ return 0 if (len(argv) == 2 and argv[1] in ("-h", "--help")) else 2
107
+ target = Path(argv[1])
108
+ try:
109
+ assert_safe(target)
110
+ except SensitivePathError as exc:
111
+ print(f"SensitivePathError: {exc}", file=sys.stderr)
112
+ return 2
113
+ print(f"safe: {target}")
114
+ return 0
115
+
116
+
117
+ if __name__ == "__main__":
118
+ sys.exit(_main(sys.argv))