@event4u/agent-config 2.20.1 → 2.21.0

package/scripts/bench_compress_memory.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Offline bench for input-side memory compression (Phase 2 / Step 11).
+
+Runs `compress_memory.py` over a fixed corpus of memory-target files, records
+pre/post char counts, approximates input-token savings (chars / 4 — the
+GPT-4 / Claude rule of thumb), and emits `bench/reports/caveman-v2.{json,md}`.
+
+Offline (no API calls). Cadence-aligned with `docs/benchmarks.md`. Citation
+in `bench/reports/caveman-v2.md` notes the chars→tokens approximation and
+points at upstream tiktoken / claude-tokenizer if a calibrated number is
+later needed.
+"""
+from __future__ import annotations
+
+import json
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+COMPRESS_SCRIPT = REPO_ROOT / "scripts" / "compress_memory.py"
+REPORT_JSON = REPO_ROOT / "bench" / "reports" / "caveman-v2.json"
+REPORT_MD = REPO_ROOT / "bench" / "reports" / "caveman-v2.md"
+
+CORPUS: list[tuple[str, str]] = [
+    ("AGENTS.md", "thin-root-package"),
+    (".agent-src.uncompressed/templates/AGENTS.md", "thin-root-consumer-template"),
+    (".agent-src/templates/AGENTS.md", "thin-root-consumer-generated"),
+    ("docs/contracts/ai-council-config.md", "prose-heavy-contract"),
+    ("docs/contracts/implement-ticket-flow.md", "prose-heavy-contract"),
+    ("docs/contracts/command-clusters.md", "prose-heavy-contract"),
+    ("docs/contracts/mental-models.md", "prose-heavy-contract"),
+    ("docs/contracts/kernel-membership.md", "prose-heavy-contract"),
+    ("docs/contracts/load-context-budget-model.md", "prose-heavy-contract"),
+    ("docs/contracts/mcp-cloud-scope.md", "prose-heavy-contract"),
+    ("docs/contracts/context-spine.md", "prose-heavy-contract"),
+    ("docs/contracts/rule-classification.md", "rule-classification"),
+]
+
+
+def chars_to_tokens(n: int) -> int:
+    """Approximate token count via chars / 4 (GPT-4/Claude English heuristic)."""
+    return round(n / 4)
+
+
+def bench_one(rel_path: str, category: str) -> dict:
+    src = REPO_ROOT / rel_path
+    if not src.is_file():
+        return {"path": rel_path, "category": category, "error": "not-found"}
+    with tempfile.TemporaryDirectory() as tmp:
+        target = Path(tmp) / src.name
+        shutil.copy(src, target)
+        before_chars = target.stat().st_size
+        result = subprocess.run(
+            [sys.executable, str(COMPRESS_SCRIPT), str(target)],
+            capture_output=True, text=True, cwd=REPO_ROOT,
+        )
+        if result.returncode != 0:
+            return {"path": rel_path, "category": category,
+                    "error": f"exit-{result.returncode}", "stderr": result.stderr[:200]}
+        after_chars = target.stat().st_size
+        before_tok = chars_to_tokens(before_chars)
+        after_tok = chars_to_tokens(after_chars)
+        return {
+            "path": rel_path,
+            "category": category,
+            "before_chars": before_chars,
+            "after_chars": after_chars,
+            "delta_chars": after_chars - before_chars,
+            "saving_pct_chars": (before_chars - after_chars) * 100 / before_chars,
+            "before_tokens_est": before_tok,
+            "after_tokens_est": after_tok,
+            "delta_tokens_est": after_tok - before_tok,
+            "saving_pct_tokens_est": (before_tok - after_tok) * 100 / before_tok if before_tok else 0.0,
+        }
+
+
+def aggregate(rows: list[dict]) -> dict:
+    rows_ok = [r for r in rows if "error" not in r]
+    savings = [r["saving_pct_chars"] for r in rows_ok]
+    by_cat: dict[str, list[float]] = {}
+    for r in rows_ok:
+        by_cat.setdefault(r["category"], []).append(r["saving_pct_chars"])
+    return {
+        "calls": len(rows),
+        "errors": len(rows) - len(rows_ok),
+        "median_saving_pct": statistics.median(savings) if savings else 0.0,
+        "p10_saving_pct": statistics.quantiles(savings, n=10)[0] if len(savings) >= 10 else min(savings, default=0.0),
+        "p90_saving_pct": statistics.quantiles(savings, n=10)[8] if len(savings) >= 10 else max(savings, default=0.0),
+        "stdev_saving_pct": statistics.pstdev(savings) if len(savings) > 1 else 0.0,
+        "total_chars_saved": sum(r["before_chars"] - r["after_chars"] for r in rows_ok),
+        "total_tokens_est_saved": sum(r["before_tokens_est"] - r["after_tokens_est"] for r in rows_ok),
+        "by_category_median_pct": {k: statistics.median(v) for k, v in by_cat.items()},
+    }
+
+
+def render_md(payload: dict) -> str:
+    agg = payload["aggregate"]
+    lines = [
+        "# caveman-v2 — input-side memory compression bench",
+        "",
+        f"**Generated:** {payload['generated_at']}",
+        f"**Schema:** `caveman-v2` (input-side; offline; chars→tokens via /4 heuristic)",
+        f"**Script:** `scripts/bench_compress_memory.py`",
+        "",
+        "## Headline",
+        "",
+        f"- Median char saving: **{agg['median_saving_pct']:+.2f}%** (p10 {agg['p10_saving_pct']:+.2f}% · p90 {agg['p90_saving_pct']:+.2f}%)",
+        f"- Total chars saved across corpus: **{agg['total_chars_saved']:+,}**",
+        f"- Total tokens (estimate) saved across corpus: **{agg['total_tokens_est_saved']:+,}**",
+        f"- Files: {agg['calls']} · errors: {agg['errors']}",
+        "",
+        "## By category (median %)",
+        "",
+        "| Category | Median saving |",
+        "|---|---:|",
+    ]
+    for cat, med in sorted(agg["by_category_median_pct"].items()):
+        lines.append(f"| {cat} | {med:+.2f}% |")
+    lines += ["", "## Per file", "",
+              "| Path | Category | Before | After | Δ chars | Saving % |",
+              "|---|---|---:|---:|---:|---:|"]
+    for r in payload["rows"]:
+        if "error" in r:
+            lines.append(f"| `{r['path']}` | {r['category']} | — | — | — | {r['error']} |")
+        else:
+            lines.append(
+                f"| `{r['path']}` | {r['category']} | {r['before_chars']:,} | {r['after_chars']:,} | "
+                f"{r['delta_chars']:+,} | {r['saving_pct_chars']:+.2f}% |"
+            )
+    lines += ["", "## Methodology",
+              "",
+              "- Offline run: `compress_memory.py` writes `.original.md` backup + frontmatter (`original_sha256`, `compressed_at`). The frontmatter pair (≈ 120 chars) is the fixed compression tax — files with little prose net negative.",
+              "- chars → tokens approximation: `tokens ≈ chars / 4` (GPT-4 / Claude English rule of thumb). Calibrated number requires `tiktoken` or `claude-tokenizer`; deferred until a consumer requests pinpoint numbers.",
+              "- The `caveman-v1` output-side verdict (`vs_terse` median −9.27%) is orthogonal — input-side savings apply to the always-loaded memory budget, not the reply stream.",
+              "",
+              "## Interpretation",
+              "",
+              "- **Thin-Root files net negative.** `AGENTS.md` and `templates/AGENTS.md` already follow `agents-md-thin-root` (≥ 40 % pointer ratio). The compressor's frontmatter pair adds more bytes than the sparse prose loses. **Do not compress Thin-Root files.**",
+              "- **Prose-heavy contract docs net 3–6 % saving.** Useful but modest. Pays off when the file is large and frequently loaded.",
+              "- **Rule of thumb:** target files with > 5 KB and visible paragraph prose; skip pointer-only files.",
+              ""]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    rows = [bench_one(p, c) for p, c in CORPUS]
+    payload = {
+        "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "schema": "caveman-v2",
+        "rows": rows,
+        "aggregate": aggregate(rows),
+    }
+    REPORT_JSON.parent.mkdir(parents=True, exist_ok=True)
+    REPORT_JSON.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    REPORT_MD.write_text(render_md(payload), encoding="utf-8")
+    print(f"wrote: {REPORT_JSON}")
+    print(f"wrote: {REPORT_MD}")
+    print(f"median saving: {payload['aggregate']['median_saving_pct']:+.2f}%")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/bench_run.py

@@ -21,6 +21,7 @@ from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
+sys.path.insert(0, str(REPO_ROOT))
 
 from _lib import script_output  # type: ignore[import-not-found]  # noqa: E402
 from _lib.bench_cost import aggregate_sessions  # noqa: E402
@@ -33,6 +34,9 @@ from _lib.bench_report import (  # noqa: E402
     write_json,
     write_markdown,
 )
+from _lib import bench_caveman  # noqa: E402
+from _lib.bench_caveman_report import build_caveman_report, render_caveman_markdown  # noqa: E402
+from _lib.bench_cost import load_pricing  # noqa: E402
 from bench_runner import run_corpus  # noqa: E402
 
 try:
@@ -41,11 +45,12 @@ except ImportError:
     script_output.error("error: PyYAML required (pip install pyyaml)")
     sys.exit(2)
 
-BENCH_RUN_VERSION = "0.1.0"
+BENCH_RUN_VERSION = "0.2.0"
 PRICING_PATH = REPO_ROOT / "bench" / "pricing.yaml"
 SESSIONS_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
 REPORTS_DIR = REPO_ROOT / "bench" / "reports"
 CORPUS_DIR = REPO_ROOT / "tests" / "eval"
+CAVEMAN_CORPUS = REPO_ROOT / "bench" / "corpora" / "caveman" / "prompts.yaml"
 BASELINE_COLLECTOR = REPO_ROOT / "scripts" / "bench_runner.py"
 
 
@@ -110,8 +115,21 @@ def main(argv: list[str] | None = None) -> int:
                     help="Override timestamp (test hook); defaults to UTC now")
     ap.add_argument("--no-write", action="store_true",
                     help="Compute the report but do not write files (dry run)")
+    ap.add_argument("--caveman", action="store_true",
+                    help="Run the caveman three-arm compression bench instead of the "
+                         "selection-accuracy bench (step-16 Phase 1).")
+    ap.add_argument("--caveman-max-prompts", type=int, default=None,
+                    help="Cap prompts in the caveman bench (smoke test).")
+    ap.add_argument("--caveman-dry-run", action="store_true",
+                    help="Caveman: skip live API calls; emit a stub report with "
+                         "zero tokens (wiring check only).")
+    ap.add_argument("--caveman-report-tag", default="caveman-v1",
+                    help="Filename tag for the caveman report (default: caveman-v1).")
     args = ap.parse_args(argv)
 
+    if args.caveman:
+        return _run_caveman(args)
+
     corpus_path = CORPUS_DIR / f"corpus-{args.corpus}.yaml"
     if not corpus_path.is_file():
         script_output.error(f"error: corpus not found: {corpus_path}")
@@ -151,5 +169,105 @@ def main(argv: list[str] | None = None) -> int:
     return 0 if verdict["overall"] in ("pass", "partial") else 1
 
 
+class _DryRunClient:
+    """Stub client for --caveman-dry-run. Returns empty CouncilResponse-shaped objects."""
+
+    def ask(self, system_prompt: str, user_prompt: str, max_tokens: int = 1024):
+        from ai_council.clients import CouncilResponse
+        return CouncilResponse(
+            provider="dry-run", model="stub", text="",
+            input_tokens=0, output_tokens=0, latency_ms=0, error=None,
+        )
+
+
+def _build_anthropic_client():
+    from ai_council.clients import AnthropicClient, load_anthropic_key
+    return AnthropicClient(api_key=load_anthropic_key())
+
+
+def _run_caveman(args: argparse.Namespace) -> int:
+    if not CAVEMAN_CORPUS.is_file():
+        script_output.error(f"error: caveman corpus not found: {CAVEMAN_CORPUS}")
+        return 2
+
+    if args.caveman_dry_run:
+        client = _DryRunClient()
+        transport = "dry-run"
+        model = "stub"
+    else:
+        try:
+            client = _build_anthropic_client()
+        except Exception as exc:  # noqa: BLE001
+            script_output.error(f"error: cannot build Anthropic client: {exc}")
+            return 2
+        transport = "api"
+        model = getattr(client, "model", "claude-sonnet-4-5")
+
+    def _progress(done: int, total: int, pid: str, arm: str, ar) -> None:
+        if args.quiet:
+            return
+        err = f" ERR={ar.error}" if ar.error else ""
+        print(f"[{done:>3}/{total}] {pid} · {arm:<14} "
+              f"in={ar.input_tokens:>4} out={ar.output_tokens:>4} "
+              f"{ar.latency_ms:>5}ms{err}", file=sys.stderr)
+
+    results = bench_caveman.run_caveman_bench(
+        client, CAVEMAN_CORPUS,
+        max_prompts=args.caveman_max_prompts,
+        on_progress=_progress,
+    )
+
+    rates, sourced_on = load_pricing(PRICING_PATH)
+    sonnet_rates = rates.get("sonnet", {"input": 0.0, "output": 0.0})
+
+    report = build_caveman_report(
+        results=results,
+        corpus_path_rel=str(CAVEMAN_CORPUS.relative_to(REPO_ROOT)),
+        generated_at=utc_now_iso(),
+        bench_run_version=BENCH_RUN_VERSION,
+        model=model,
+        transport=transport,
+        pricing_rates=sonnet_rates,
+        pricing_sourced_on=sourced_on,
+    )
+
+    stamp = args.stamp or utc_now_filename_stamp()
+    json_path, md_path = report_paths(REPORTS_DIR, args.caveman_report_tag, stamp)
+    # Override: caveman roadmap pins the filename to `caveman-v1.{json,md}` (no stamp).
+    fixed_json = REPORTS_DIR / f"{args.caveman_report_tag}.json"
+    fixed_md = REPORTS_DIR / f"{args.caveman_report_tag}.md"
+
+    if not args.no_write:
+        write_json(fixed_json, report)
+        fixed_md.parent.mkdir(parents=True, exist_ok=True)
+        fixed_md.write_text(render_caveman_markdown(report), encoding="utf-8")
+        # Also drop a timestamped copy for the cadence trail.
+        write_json(json_path, report)
+        json_path.with_suffix(".md").write_text(
+            render_caveman_markdown(report), encoding="utf-8"
+        )
+
+    cost = report["cost"]
+    headline = (
+        f"caveman · prompts {report['corpus']['prompt_count']} · "
+        f"calls {cost['totals']['calls']} · errors {cost['totals']['errors']} · "
+        f"vs_raw med {report['caveman']['aggregate']['savings_vs_raw']['median']:.2%} · "
+        f"vs_terse med {report['caveman']['aggregate']['savings_vs_terse']['median']:.2%} · "
+        f"cost ${cost['totals']['total_cost_usd']:.6f}"
+    )
+    if args.quiet:
+        print(headline)
+        if not args.no_write:
+            print(f"report: {fixed_md.relative_to(REPO_ROOT)}")
+    else:
+        print(render_caveman_markdown(report))
+        if not args.no_write:
+            print(f"\n→ json:     {fixed_json.relative_to(REPO_ROOT)}")
+            print(f"→ markdown: {fixed_md.relative_to(REPO_ROOT)}")
+            print(f"→ trail:    {json_path.relative_to(REPO_ROOT)}")
+
+    return 0 if cost["totals"]["errors"] == 0 else 1
+
+
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

package/scripts/caveman_stats.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Caveman per-session / per-conversation / lifetime token-delta lens.
+
+Reads sessions.jsonl, groups by sessionId + conversation_id, emits per-row
+caveman delta tokens. Honors the suspended-multiplier contract in
+`docs/contracts/caveman-telemetry.md` (delta = 0 while suspended).
+"""
+from __future__ import annotations
+import argparse, json, sys
+from collections import defaultdict
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
+TELEMETRY_DOC = REPO_ROOT / "docs" / "contracts" / "caveman-telemetry.md"
+
+# Mirrors `docs/contracts/caveman-telemetry.md` `v1` constants.
+MULTIPLIER_VERSION = "v1"
+MULTIPLIER_VALUE = 0.9155
+MULTIPLIER_ACTIVE = False  # suspended pending v2
+
+
+def _load(path: Path) -> list[dict]:
+    if not path.is_file():
+        return []
+    rows: list[dict] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        try:
+            rows.append(json.loads(line))
+        except json.JSONDecodeError:
+            continue
+    return rows
+
+
+def _delta(row: dict) -> int:
+    """Per-row delta with suspended-multiplier guard."""
+    if not MULTIPLIER_ACTIVE:
+        return 0
+    explicit = row.get("caveman_delta_tokens")
+    if isinstance(explicit, (int, float)):
+        return int(explicit)
+    compressed = row.get("caveman_compressed_tokens")
+    if isinstance(compressed, (int, float)) and compressed > 0:
+        return int(compressed * MULTIPLIER_VALUE - compressed)
+    return 0
+
+
+def aggregate(rows: list[dict]) -> dict:
+    _zero = lambda: {"sessions": 0, "delta_tokens": 0, "compressed_tokens": 0}
+    by_session: dict[str, dict] = defaultdict(_zero)
+    by_conv: dict[str, dict] = defaultdict(_zero)
+    lifetime = _zero()
+    for row in rows:
+        sid = str(row.get("sessionId") or row.get("session_id") or "unknown")
+        cid = str(row.get("conversation_id") or "unknown")
+        delta = _delta(row)
+        comp = int(row.get("caveman_compressed_tokens") or 0)
+        for bucket in (by_session[sid], by_conv[cid], lifetime):
+            bucket["sessions"] += 1
+            bucket["delta_tokens"] += delta
+            bucket["compressed_tokens"] += comp
+    return {
+        "schema_version": "caveman-stats/v1",
+        "multiplier_version": MULTIPLIER_VERSION,
+        "multiplier_value": MULTIPLIER_VALUE,
+        "multiplier_active": MULTIPLIER_ACTIVE,
+        "lifetime": lifetime,
+        "by_session": dict(by_session),
+        "by_conversation": dict(by_conv),
+    }
+
+
+def render_text(report: dict) -> str:
+    lines = [
+        f"caveman-stats {report['schema_version']} · multiplier {report['multiplier_version']}"
+        f" ({'ACTIVE' if report['multiplier_active'] else 'SUSPENDED'}) · "
+        f"value {report['multiplier_value']:.4f}",
+        "",
+        f"  lifetime: {report['lifetime']['sessions']} sessions · "
+        f"delta_tokens = {report['lifetime']['delta_tokens']:+,} · "
+        f"compressed_tokens = {report['lifetime']['compressed_tokens']:,}",
+        "",
+        "  by conversation:",
+    ]
+    for cid, b in sorted(report["by_conversation"].items()):
+        lines.append(
+            f"    {cid}: {b['sessions']} sessions · "
+            f"delta = {b['delta_tokens']:+,} · compressed = {b['compressed_tokens']:,}"
+        )
+    if not report["multiplier_active"]:
+        lines += [
+            "",
+            "  Note: multiplier suspended — see docs/contracts/caveman-telemetry.md",
+            "  (delta_tokens = 0 until kill-criterion satisfied in caveman-v2).",
+        ]
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--input", type=Path, default=DEFAULT_JSONL)
+    parser.add_argument("--format", choices=["text", "json"], default="text")
+    args = parser.parse_args(argv)
+
+    rows = _load(args.input)
+    report = aggregate(rows)
+
+    if args.format == "json":
+        print(json.dumps(report, indent=2))
+    else:
+        print(render_text(report))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/check_command_count_messaging.py

@@ -17,7 +17,7 @@ Canonical counts:
 Patterns checked (per file):
 
     README.md
-      hero row     "<strong>{N} Commands</strong>"            → active
+      hero badge   "/badge/Commands-{N}-…"                    → active
       browse line  "Browse all {N} active commands"           → active
       browse meta  "{N} files total"                          → total
       browse meta  "{N} are deprecation shims"                → shims
@@ -84,7 +84,7 @@ def main() -> int:
 
     checks = [
         # README.md
-        (README, r"<strong>(\d+) Commands</strong>", active, "hero row"),
+        (README, r"/badge/Commands-(\d+)-", active, "hero badge"),
         (README, r"Browse all (\d+) active commands", active, "browse line"),
         (README, r"\+ (\d+) native commands\)", active, "tools blurb"),
         # docs/getting-started.md

package/scripts/compress_memory.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Input-side memory compression — Phase 2 of step-16-caveman-substance.
+
+Rewrites memory files (AGENTS.md, CLAUDE.md, .cursorrules, ...) to caveman
+grammar (drop articles / auxiliaries) while preserving carve-outs byte-for-byte
+(code blocks, numbered-options, status markers, Iron-Law ALL-CAPS, backtick
+spans). Writes `.original.md` backup before mutating. Gated by Phase 0
+`validate_safe_paths.assert_safe`. Idempotency guard: `original_sha256:` +
+`compressed_at:` frontmatter refuse re-compression on body-hash drift.
+
+CLI: `compress_memory.py <path> [--check|--decompress]`. Stdlib-only.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import re
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+from validate_safe_paths import SensitivePathError, assert_safe  # noqa: E402
+
+__all__ = ["compress_text", "compress_file", "decompress_file", "CompressionRefused"]
+
+
+class CompressionRefused(RuntimeError):
+    """Raised when the target is already compressed and body hash diverged."""
+
+
+# Carve-out region patterns — mirrors caveman-speak.md § Carve-outs (1–7).
+RE_FENCE = re.compile(r"^```")
+RE_NUMBERED = re.compile(r"^>?\s*\d+\.\s")
+RE_STATUS = re.compile(r"^\s*(?:❌|⚠️|✅)")
+RE_IRONLAW = re.compile(r"^[A-Z][A-Z0-9 ,.\-_/']{3,}$")
+RE_BACKTICK_SPAN = re.compile(r"`[^`\n]+`")
+RE_FRONTMATTER = re.compile(r"^---\s*$")
+WORD_RE = re.compile(r"\b[A-Za-z]+\b")
+DROP_TOKENS = {"the", "a", "an", "is", "are", "was", "were", "be", "been",
+               "being", "that", "which"}
+
+
+def _compress_words(text: str) -> str:
+    out = WORD_RE.sub(lambda m: "" if m.group(0).lower() in DROP_TOKENS else m.group(0), text)
+    out = re.sub(r"[ \t]{2,}", " ", out)
+    return re.sub(r" +([,.;:!?])", r"\1", out)
+
+
+def _compress_prose_line(line: str) -> str:
+    """Compress a prose line; preserve backtick-spans byte-for-byte."""
+    parts: list[str] = []
+    last = 0
+    for span in RE_BACKTICK_SPAN.finditer(line):
+        parts.append(_compress_words(line[last:span.start()]))
+        parts.append(span.group(0))
+        last = span.end()
+    parts.append(_compress_words(line[last:]))
+    return "".join(parts)
+
+
+def compress_text(body: str) -> str:
+    """Compress a memory-file body. Idempotent on already-caveman text."""
+    out: list[str] = []
+    in_fence = False
+    for raw in body.splitlines(keepends=True):
+        stripped = raw.rstrip("\r\n")
+        if RE_FENCE.match(stripped):
+            in_fence = not in_fence
+            out.append(raw)
+            continue
+        if in_fence or RE_NUMBERED.match(stripped) or RE_STATUS.match(stripped) \
+                or RE_IRONLAW.match(stripped.strip()):
+            out.append(raw)
+            continue
+        out.append(_compress_prose_line(raw))
+    return "".join(out)
+
+
+def _split_frontmatter(text: str) -> tuple[str, str]:
+    lines = text.splitlines(keepends=True)
+    if not lines or not RE_FRONTMATTER.match(lines[0].rstrip()):
+        return "", text
+    for idx in range(1, len(lines)):
+        if RE_FRONTMATTER.match(lines[idx].rstrip()):
+            return "".join(lines[: idx + 1]), "".join(lines[idx + 1:])
+    return "", text
+
+
+def _sha256(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def _has_sha_marker(fm: str) -> bool:
+    return bool(re.search(r"^original_sha256:\s*[0-9a-f]{64}\s*$", fm, re.MULTILINE))
+
+
+def _inject_frontmatter(fm: str, sha: str, ts: str) -> str:
+    drop = re.compile(r"^(original_sha256|compressed_at):.*$", re.MULTILINE)
+    inner = drop.sub("", fm.strip().strip("-").strip()).strip() if fm else ""
+    body = inner + ("\n" if inner else "")
+    return f"---\n{body}original_sha256: {sha}\ncompressed_at: {ts}\n---\n"
+
+
+def _backup_path(target: Path) -> Path:
+    return target.parent / (target.name + ".original.md")
+
+
+def compress_file(target: Path) -> Path:
+    assert_safe(target)
+    text = target.read_text(encoding="utf-8")
+    fm, body = _split_frontmatter(text)
+    if _has_sha_marker(fm):
+        if _sha256(compress_text(body)) != _sha256(body):
+            raise CompressionRefused(
+                f"{target}: body hash diverged; decompress first "
+                f"(`scripts/compress_memory.py {target} --decompress`)."
+            )
+        return target
+    backup = _backup_path(target)
+    backup.write_text(text, encoding="utf-8")
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    target.write_text(
+        _inject_frontmatter(fm, _sha256(body), ts) + compress_text(body),
+        encoding="utf-8",
+    )
+    return backup
+
+
+def decompress_file(target: Path) -> Path:
+    assert_safe(target)
+    backup = _backup_path(target)
+    if not backup.is_file():
+        raise FileNotFoundError(f"no backup at {backup}")
+    target.write_text(backup.read_text(encoding="utf-8"), encoding="utf-8")
+    backup.unlink()
+    return target
+
+
+def _main(argv: list[str]) -> int:
+    ap = argparse.ArgumentParser(description="Compress memory files to caveman grammar.")
+    ap.add_argument("path", type=Path)
+    grp = ap.add_mutually_exclusive_group()
+    grp.add_argument("--check", action="store_true", help="exit 0 if safe; no writes")
+    grp.add_argument("--decompress", action="store_true", help="restore .original.md")
+    args = ap.parse_args(argv)
+    try:
+        if args.check:
+            assert_safe(args.path)
+            return 0
+        if args.decompress:
+            decompress_file(args.path)
+            print(f"decompressed: {args.path}")
+            return 0
+        backup = compress_file(args.path)
+        print(f"compressed: {args.path}  (backup: {backup})")
+        return 0
+    except SensitivePathError as exc:
+        print(f"error: refused: {exc}", file=sys.stderr)
+        return 2
+    except CompressionRefused as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 3
+    except FileNotFoundError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 4
+
+
+if __name__ == "__main__":
+    raise SystemExit(_main(sys.argv[1:]))