@event4u/agent-config 2.20.1 → 2.23.0

package/scripts/_lib/bench_caveman.py

@@ -0,0 +1,273 @@
+# Caveman compression bench — step-16 Phase 1 Step 4.
+#
+# Three-arm live bench against bench/corpora/caveman/prompts.yaml:
+#   compressed     — system prompt embeds caveman-speak rule (aggressive).
+#   terse_control  — system prompt = "Answer concisely. …" (carve-out-free baseline).
+#   uncompressed   — generic helpful-assistant system prompt.
+#
+# Token counts come from Anthropic API `usage` (authoritative). Carve-out
+# share is measured via regex extraction on the reply text; chars/4 yields
+# an estimated carve-out-token figure for the carve-out-tax accounting.
+#
+# Cost-touch: 10 prompts × 3 arms × claude-sonnet-4-5 (~$3/M in, ~$15/M out).
+"""Caveman compression bench runner."""
+from __future__ import annotations
+
+import re
+import statistics
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+# ── system prompts per arm ──────────────────────────────────────────────
+
+SYSTEM_PROMPT_COMPRESSED = """You are speaking in CAVEMAN-SPEAK mode (speak_scope=aggressive).
+
+Compress all body prose to caveman grammar:
+- Drop articles (the, a, an).
+- Drop linking auxiliaries (is, are, was, be) where unambiguous.
+- Drop pronouns when context is clear.
+- Keep nouns, verbs, key adjectives, negation, numbers.
+- Example: "I will now check the file and see if it exists" -> "Check file. Exists?"
+
+Carve-outs — preserve BYTE-FOR-BYTE (do NOT compress these):
+1. Triple-backtick code/literal blocks (any language, including ALL-CAPS Iron-Law fences).
+2. Numbered-options lines matching ^\\d+\\.\\s + a **Recommendation:** label.
+3. Backtick spans (file paths, command names, identifiers).
+4. Status markers: lines starting with ❌, ⚠️, or ✅.
+5. Mode markers.
+6. Markdown tables.
+7. Deliverables (PR titles, commit messages, ticket summaries, articles, the prompt
+   line of any single question asked to the user).
+
+Apply caveman compression aggressively to every other prose surface."""
+
+SYSTEM_PROMPT_TERSE = (
+    "Answer concisely. Skip preamble. Do not restate the question. "
+    "Avoid filler phrases ('Let me', 'Here is', 'I will'). Get to the answer."
+)
+
+SYSTEM_PROMPT_UNCOMPRESSED = (
+    "You are a helpful AI assistant. Answer the user's question clearly and completely."
+)
+
+ARMS: tuple[str, ...] = ("compressed", "terse_control", "uncompressed")
+ARM_SYSTEM_PROMPT: dict[str, str] = {
+    "compressed": SYSTEM_PROMPT_COMPRESSED,
+    "terse_control": SYSTEM_PROMPT_TERSE,
+    "uncompressed": SYSTEM_PROMPT_UNCOMPRESSED,
+}
+
+# ── carve-out detection ────────────────────────────────────────────────
+
+_RE_TRIPLE_BACKTICK = re.compile(r"```[\s\S]*?```")
+_RE_BACKTICK_SPAN = re.compile(r"`[^`\n]+`")
+_RE_NUMBERED_LINE = re.compile(r"^>?\s*\d+\.\s.*$", re.MULTILINE)
+_RE_STATUS_LINE = re.compile(r"^(❌|⚠️|✅).*$", re.MULTILINE)
+_RE_TABLE_LINE = re.compile(r"^\s*\|.*\|\s*$", re.MULTILINE)
+_RE_RECOMMENDATION = re.compile(r"^\*\*(Recommendation|Empfehlung):\*\*.*$", re.MULTILINE)
+
+
+def carve_out_chars(text: str) -> int:
+    """Sum byte-length of every carve-out region (union, no double-count)."""
+    if not text:
+        return 0
+    mask = bytearray(len(text))
+    for pattern in (
+        _RE_TRIPLE_BACKTICK, _RE_BACKTICK_SPAN, _RE_NUMBERED_LINE,
+        _RE_STATUS_LINE, _RE_TABLE_LINE, _RE_RECOMMENDATION,
+    ):
+        for m in pattern.finditer(text):
+            for i in range(m.start(), m.end()):
+                mask[i] = 1
+    return sum(mask)
+
+
+# ── data shapes ────────────────────────────────────────────────────────
+
+@dataclass
+class ArmResult:
+    arm: str
+    text: str
+    input_tokens: int
+    output_tokens: int
+    latency_ms: int
+    output_chars: int
+    carve_out_chars: int
+    error: str | None = None
+
+    @property
+    def realised_carve_out_pct(self) -> float:
+        return self.carve_out_chars / self.output_chars if self.output_chars else 0.0
+
+
+@dataclass
+class PromptResult:
+    id: str
+    category: str
+    expected_carve_out_pct: float
+    arms: dict[str, ArmResult] = field(default_factory=dict)
+
+    @property
+    def savings_vs_raw(self) -> float | None:
+        c = self.arms.get("compressed")
+        u = self.arms.get("uncompressed")
+        if not c or not u or u.output_tokens == 0:
+            return None
+        return 1.0 - (c.output_tokens / u.output_tokens)
+
+    @property
+    def savings_vs_terse(self) -> float | None:
+        c = self.arms.get("compressed")
+        t = self.arms.get("terse_control")
+        if not c or not t or t.output_tokens == 0:
+            return None
+        return 1.0 - (c.output_tokens / t.output_tokens)
+
+
+# ── corpus + runner ────────────────────────────────────────────────────
+
+def load_corpus(corpus_path: Path) -> list[dict[str, Any]]:
+    """Read bench/corpora/caveman/prompts.yaml → list of prompt dicts."""
+    data = yaml.safe_load(corpus_path.read_text(encoding="utf-8")) or {}
+    prompts = data.get("prompts") or []
+    if not prompts:
+        raise ValueError(f"empty corpus: {corpus_path}")
+    return prompts
+
+
+def run_arm(
+    client: Any,
+    arm: str,
+    user_prompt: str,
+    *,
+    max_tokens: int = 1024,
+) -> ArmResult:
+    """Invoke one arm against the live API. Returns ArmResult including text."""
+    t0 = time.monotonic()
+    system = ARM_SYSTEM_PROMPT[arm]
+    try:
+        resp = client.ask(system, user_prompt, max_tokens=max_tokens)
+    except Exception as exc:  # noqa: BLE001
+        latency_ms = int((time.monotonic() - t0) * 1000)
+        return ArmResult(arm=arm, text="", input_tokens=0, output_tokens=0,
+                         latency_ms=latency_ms, output_chars=0, carve_out_chars=0,
+                         error=str(exc))
+    return ArmResult(
+        arm=arm, text=resp.text or "",
+        input_tokens=int(resp.input_tokens or 0),
+        output_tokens=int(resp.output_tokens or 0),
+        latency_ms=int(resp.latency_ms or (time.monotonic() - t0) * 1000),
+        output_chars=len(resp.text or ""),
+        carve_out_chars=carve_out_chars(resp.text or ""),
+        error=resp.error,
+    )
+
+
+# ── aggregation ────────────────────────────────────────────────────────────
+
+def _stats(values: list[float]) -> dict[str, float]:
+    """Median / p10 / p90 / stdev / n on a list of floats. Empty → zeros."""
+    if not values:
+        return {"n": 0, "median": 0.0, "p10": 0.0, "p90": 0.0, "stdev": 0.0}
+    s = sorted(values)
+    n = len(s)
+    def _pct(p: float) -> float:
+        if n == 1:
+            return s[0]
+        k = (n - 1) * p
+        lo, hi = int(k), min(int(k) + 1, n - 1)
+        return s[lo] + (s[hi] - s[lo]) * (k - lo)
+    return {
+        "n": n,
+        "median": statistics.median(s),
+        "p10": _pct(0.10),
+        "p90": _pct(0.90),
+        "stdev": statistics.pstdev(s) if n > 1 else 0.0,
+    }
+
+
+def aggregate_results(results: list[PromptResult]) -> dict[str, Any]:
+    """Compute median/p10/p90 for compression metrics across the corpus."""
+    vs_raw = [r.savings_vs_raw for r in results if r.savings_vs_raw is not None]
+    vs_terse = [r.savings_vs_terse for r in results if r.savings_vs_terse is not None]
+    realised_carve_pct = [
+        r.arms["compressed"].realised_carve_out_pct
+        for r in results if "compressed" in r.arms and r.arms["compressed"].output_chars
+    ]
+    expected_carve_pct = [r.expected_carve_out_pct for r in results]
+
+    per_arm_tokens: dict[str, list[int]] = {a: [] for a in ARMS}
+    for r in results:
+        for arm in ARMS:
+            ar = r.arms.get(arm)
+            if ar:
+                per_arm_tokens[arm].append(ar.output_tokens)
+
+    return {
+        "savings_vs_raw": _stats(vs_raw),
+        "savings_vs_terse": _stats(vs_terse),
+        "realised_carve_out_pct": _stats(realised_carve_pct),
+        "expected_carve_out_pct": _stats(expected_carve_pct),
+        "output_tokens": {
+            arm: _stats([float(v) for v in per_arm_tokens[arm]]) for arm in ARMS
+        },
+    }
+
+
+def compute_cost(results: list[PromptResult], pricing: dict[str, float]) -> dict[str, Any]:
+    """Sum input/output tokens across all arms; cost from per-1M pricing dict."""
+    totals = {"input_tokens": 0, "output_tokens": 0, "calls": 0, "errors": 0}
+    per_arm: dict[str, dict[str, int]] = {a: {"input_tokens": 0, "output_tokens": 0, "calls": 0} for a in ARMS}
+    for r in results:
+        for arm, ar in r.arms.items():
+            totals["input_tokens"] += ar.input_tokens
+            totals["output_tokens"] += ar.output_tokens
+            totals["calls"] += 1
+            if ar.error:
+                totals["errors"] += 1
+            per_arm[arm]["input_tokens"] += ar.input_tokens
+            per_arm[arm]["output_tokens"] += ar.output_tokens
+            per_arm[arm]["calls"] += 1
+    cost_usd = (
+        totals["input_tokens"] / 1e6 * pricing.get("input", 0.0)
+        + totals["output_tokens"] / 1e6 * pricing.get("output", 0.0)
+    )
+    totals["total_cost_usd"] = round(cost_usd, 6)
+    return {"totals": totals, "per_arm": per_arm}
+
+
+# ── orchestrator ───────────────────────────────────────────────────────────
+
+def run_caveman_bench(
+    client: Any,
+    corpus_path: Path,
+    *,
+    max_prompts: int | None = None,
+    max_tokens: int = 1024,
+    on_progress: Any = None,
+) -> list[PromptResult]:
+    """Run all three arms over the corpus. Returns per-prompt results."""
+    prompts = load_corpus(corpus_path)
+    if max_prompts:
+        prompts = prompts[:max_prompts]
+    results: list[PromptResult] = []
+    total = len(prompts) * len(ARMS)
+    done = 0
+    for p in prompts:
+        pr = PromptResult(
+            id=str(p["id"]),
+            category=str(p.get("category", "unknown")),
+            expected_carve_out_pct=float(p.get("expected_carve_out_pct", 0.0)),
+        )
+        for arm in ARMS:
+            ar = run_arm(client, arm, str(p["prompt"]), max_tokens=max_tokens)
+            pr.arms[arm] = ar
+            done += 1
+            if on_progress:
+                on_progress(done, total, pr.id, arm, ar)
+        results.append(pr)
+    return results

package/scripts/_lib/bench_caveman_report.py

@@ -0,0 +1,152 @@
+# Caveman bench report serializer — step-16 Phase 1 Step 5.
+#
+# Emits the caveman-v1 JSON + Markdown shape. Distinct schema_version
+# ("caveman-v1") from the selection-accuracy bench (v1) because the
+# blocks are disjoint: caveman has no `selection`/`quality`, and the
+# selection bench has no three-arm compression metrics.
+"""Caveman bench report serializer."""
+from __future__ import annotations
+
+from typing import Any
+
+from _lib.bench_caveman import ARMS, PromptResult, aggregate_results, compute_cost
+
+
+def build_caveman_report(
+    *,
+    results: list[PromptResult],
+    corpus_path_rel: str,
+    generated_at: str,
+    bench_run_version: str,
+    model: str,
+    transport: str,
+    pricing_rates: dict[str, float],
+    pricing_sourced_on: str | None,
+) -> dict[str, Any]:
+    aggregate = aggregate_results(results)
+    cost = compute_cost(results, pricing_rates)
+    cost["source"] = "live-api"
+    cost["model"] = model
+    cost["pricing_sourced_on"] = pricing_sourced_on
+    errors = cost["totals"]["errors"]
+    return {
+        "schema_version": "caveman-v1",
+        "generated_at": generated_at,
+        "corpus": {
+            "id": "caveman",
+            "path": corpus_path_rel,
+            "prompt_count": len(results),
+        },
+        "runner": {
+            "bench_run_version": bench_run_version,
+            "transport": transport,
+            "model": model,
+        },
+        "caveman": {
+            "arms": list(ARMS),
+            "aggregate": aggregate,
+            "per_prompt": [_prompt_block(r) for r in results],
+        },
+        "cost": cost,
+        "verdict": {
+            "overall": "measured" if errors == 0 else "partial",
+            "errors": errors,
+        },
+    }
+
+
+def _prompt_block(r: PromptResult) -> dict[str, Any]:
+    return {
+        "id": r.id,
+        "category": r.category,
+        "expected_carve_out_pct": r.expected_carve_out_pct,
+        "realised_carve_out_pct": (
+            r.arms["compressed"].realised_carve_out_pct
+            if "compressed" in r.arms else None
+        ),
+        "savings_vs_raw": r.savings_vs_raw,
+        "savings_vs_terse": r.savings_vs_terse,
+        "arms": {
+            arm: {
+                "input_tokens": ar.input_tokens,
+                "output_tokens": ar.output_tokens,
+                "latency_ms": ar.latency_ms,
+                "output_chars": ar.output_chars,
+                "carve_out_chars": ar.carve_out_chars,
+                "error": ar.error,
+                "text": ar.text,
+            }
+            for arm, ar in r.arms.items()
+        },
+    }
+
+
+def _fmt_pct(x: float | None) -> str:
+    return f"{x:.2%}" if isinstance(x, (int, float)) else "—"
+
+
+def render_caveman_markdown(report: dict[str, Any]) -> str:
+    cv = report["caveman"]
+    agg = cv["aggregate"]
+    cost = report["cost"]
+    head = [
+        f"# Caveman Bench Report — `caveman` · {report['generated_at']}",
+        "",
+        "## Headline",
+        "",
+        f"- prompts: **{report['corpus']['prompt_count']}** · "
+        f"arms: **{', '.join(cv['arms'])}** · "
+        f"model: **{report['runner']['model']}** · "
+        f"transport: **{report['runner']['transport']}**",
+        f"- median savings vs raw: **{_fmt_pct(agg['savings_vs_raw']['median'])}** "
+        f"(p10 {_fmt_pct(agg['savings_vs_raw']['p10'])} · p90 {_fmt_pct(agg['savings_vs_raw']['p90'])})",
+        f"- median savings vs terse-control: **{_fmt_pct(agg['savings_vs_terse']['median'])}** "
+        f"(p10 {_fmt_pct(agg['savings_vs_terse']['p10'])} · p90 {_fmt_pct(agg['savings_vs_terse']['p90'])})",
+        f"- median realised carve-out share (compressed arm): **{_fmt_pct(agg['realised_carve_out_pct']['median'])}** "
+        f"(expected median {_fmt_pct(agg['expected_carve_out_pct']['median'])})",
+        f"- total cost: **${cost['totals']['total_cost_usd']:.6f}** "
+        f"(calls {cost['totals']['calls']} · errors {cost['totals']['errors']})",
+        f"- verdict: **{report['verdict']['overall']}**",
+        "",
+    ]
+    per_arm = [
+        "## Per-arm token totals",
+        "",
+        "| arm | calls | input_tokens | output_tokens | median out/prompt |",
+        "|---|---:|---:|---:|---:|",
+    ]
+    for arm in cv["arms"]:
+        a = cost["per_arm"][arm]
+        m = agg["output_tokens"][arm]["median"]
+        per_arm.append(
+            f"| `{arm}` | {a['calls']} | {a['input_tokens']} | {a['output_tokens']} | {m:.0f} |"
+        )
+    per_arm.append("")
+    per_prompt = [
+        "## Per-prompt results",
+        "",
+        "| id | category | exp.carve | real.carve | out.compressed | out.terse | out.uncompressed | vs raw | vs terse |",
+        "|---|---|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for r in cv["per_prompt"]:
+        arms = r["arms"]
+        oc = arms.get("compressed", {}).get("output_tokens", "—")
+        ot = arms.get("terse_control", {}).get("output_tokens", "—")
+        ou = arms.get("uncompressed", {}).get("output_tokens", "—")
+        per_prompt.append(
+            f"| `{r['id']}` | {r['category']} | "
+            f"{_fmt_pct(r['expected_carve_out_pct'])} | {_fmt_pct(r['realised_carve_out_pct'])} | "
+            f"{oc} | {ot} | {ou} | "
+            f"{_fmt_pct(r['savings_vs_raw'])} | {_fmt_pct(r['savings_vs_terse'])} |"
+        )
+    per_prompt.append("")
+    notes = [
+        "## Notes",
+        "",
+        f"- corpus: `{report['corpus']['path']}`",
+        f"- pricing: `bench/pricing.yaml` (sourced {cost.get('pricing_sourced_on') or '—'})",
+        f"- schema: `caveman-v1` (see `docs/contracts/benchmark-report-schema.md`)",
+        f"- bench_run version: `{report['runner']['bench_run_version']}`",
+        "",
+    ]
+    return "\n".join(head + per_arm + per_prompt + notes)

package/scripts/bench_compress_memory.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Offline bench for input-side memory compression (Phase 2 / Step 11).
+
+Runs `compress_memory.py` over a fixed corpus of memory-target files, records
+pre/post char counts, approximates input-token savings (chars / 4 — the
+GPT-4 / Claude rule of thumb), and emits `bench/reports/caveman-v2.{json,md}`.
+
+Offline (no API calls). Cadence-aligned with `docs/benchmarks.md`. Citation
+in `bench/reports/caveman-v2.md` notes the chars→tokens approximation and
+points at upstream tiktoken / claude-tokenizer if a calibrated number is
+later needed.
+"""
+from __future__ import annotations
+
+import json
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+COMPRESS_SCRIPT = REPO_ROOT / "scripts" / "compress_memory.py"
+REPORT_JSON = REPO_ROOT / "bench" / "reports" / "caveman-v2.json"
+REPORT_MD = REPO_ROOT / "bench" / "reports" / "caveman-v2.md"
+
+CORPUS: list[tuple[str, str]] = [
+    ("AGENTS.md", "thin-root-package"),
+    (".agent-src.uncompressed/templates/AGENTS.md", "thin-root-consumer-template"),
+    (".agent-src/templates/AGENTS.md", "thin-root-consumer-generated"),
+    ("docs/contracts/ai-council-config.md", "prose-heavy-contract"),
+    ("docs/contracts/implement-ticket-flow.md", "prose-heavy-contract"),
+    ("docs/contracts/command-clusters.md", "prose-heavy-contract"),
+    ("docs/contracts/mental-models.md", "prose-heavy-contract"),
+    ("docs/contracts/kernel-membership.md", "prose-heavy-contract"),
+    ("docs/contracts/load-context-budget-model.md", "prose-heavy-contract"),
+    ("docs/contracts/mcp-cloud-scope.md", "prose-heavy-contract"),
+    ("docs/contracts/context-spine.md", "prose-heavy-contract"),
+    ("docs/contracts/rule-classification.md", "rule-classification"),
+]
+
+
+def chars_to_tokens(n: int) -> int:
+    """Approximate token count via chars / 4 (GPT-4/Claude English heuristic)."""
+    return round(n / 4)
+
+
+def bench_one(rel_path: str, category: str) -> dict:
+    src = REPO_ROOT / rel_path
+    if not src.is_file():
+        return {"path": rel_path, "category": category, "error": "not-found"}
+    with tempfile.TemporaryDirectory() as tmp:
+        target = Path(tmp) / src.name
+        shutil.copy(src, target)
+        before_chars = target.stat().st_size
+        result = subprocess.run(
+            [sys.executable, str(COMPRESS_SCRIPT), str(target)],
+            capture_output=True, text=True, cwd=REPO_ROOT,
+        )
+        if result.returncode != 0:
+            return {"path": rel_path, "category": category,
+                    "error": f"exit-{result.returncode}", "stderr": result.stderr[:200]}
+        after_chars = target.stat().st_size
+        before_tok = chars_to_tokens(before_chars)
+        after_tok = chars_to_tokens(after_chars)
+        return {
+            "path": rel_path,
+            "category": category,
+            "before_chars": before_chars,
+            "after_chars": after_chars,
+            "delta_chars": after_chars - before_chars,
+            "saving_pct_chars": (before_chars - after_chars) * 100 / before_chars,
+            "before_tokens_est": before_tok,
+            "after_tokens_est": after_tok,
+            "delta_tokens_est": after_tok - before_tok,
+            "saving_pct_tokens_est": (before_tok - after_tok) * 100 / before_tok if before_tok else 0.0,
+        }
+
+
+def aggregate(rows: list[dict]) -> dict:
+    rows_ok = [r for r in rows if "error" not in r]
+    savings = [r["saving_pct_chars"] for r in rows_ok]
+    by_cat: dict[str, list[float]] = {}
+    for r in rows_ok:
+        by_cat.setdefault(r["category"], []).append(r["saving_pct_chars"])
+    return {
+        "calls": len(rows),
+        "errors": len(rows) - len(rows_ok),
+        "median_saving_pct": statistics.median(savings) if savings else 0.0,
+        "p10_saving_pct": statistics.quantiles(savings, n=10)[0] if len(savings) >= 10 else min(savings, default=0.0),
+        "p90_saving_pct": statistics.quantiles(savings, n=10)[8] if len(savings) >= 10 else max(savings, default=0.0),
+        "stdev_saving_pct": statistics.pstdev(savings) if len(savings) > 1 else 0.0,
+        "total_chars_saved": sum(r["before_chars"] - r["after_chars"] for r in rows_ok),
+        "total_tokens_est_saved": sum(r["before_tokens_est"] - r["after_tokens_est"] for r in rows_ok),
+        "by_category_median_pct": {k: statistics.median(v) for k, v in by_cat.items()},
+    }
+
+
+def render_md(payload: dict) -> str:
+    agg = payload["aggregate"]
+    lines = [
+        "# caveman-v2 — input-side memory compression bench",
+        "",
+        f"**Generated:** {payload['generated_at']}",
+        f"**Schema:** `caveman-v2` (input-side; offline; chars→tokens via /4 heuristic)",
+        f"**Script:** `scripts/bench_compress_memory.py`",
+        "",
+        "## Headline",
+        "",
+        f"- Median char saving: **{agg['median_saving_pct']:+.2f}%** (p10 {agg['p10_saving_pct']:+.2f}% · p90 {agg['p90_saving_pct']:+.2f}%)",
+        f"- Total chars saved across corpus: **{agg['total_chars_saved']:+,}**",
+        f"- Total tokens (estimate) saved across corpus: **{agg['total_tokens_est_saved']:+,}**",
+        f"- Files: {agg['calls']} · errors: {agg['errors']}",
+        "",
+        "## By category (median %)",
+        "",
+        "| Category | Median saving |",
+        "|---|---:|",
+    ]
+    for cat, med in sorted(agg["by_category_median_pct"].items()):
+        lines.append(f"| {cat} | {med:+.2f}% |")
+    lines += ["", "## Per file", "",
+              "| Path | Category | Before | After | Δ chars | Saving % |",
+              "|---|---|---:|---:|---:|---:|"]
+    for r in payload["rows"]:
+        if "error" in r:
+            lines.append(f"| `{r['path']}` | {r['category']} | — | — | — | {r['error']} |")
+        else:
+            lines.append(
+                f"| `{r['path']}` | {r['category']} | {r['before_chars']:,} | {r['after_chars']:,} | "
+                f"{r['delta_chars']:+,} | {r['saving_pct_chars']:+.2f}% |"
+            )
+    lines += ["", "## Methodology",
+              "",
+              "- Offline run: `compress_memory.py` writes `.original.md` backup + frontmatter (`original_sha256`, `compressed_at`). The frontmatter pair (≈ 120 chars) is the fixed compression tax — files with little prose net negative.",
+              "- chars → tokens approximation: `tokens ≈ chars / 4` (GPT-4 / Claude English rule of thumb). Calibrated number requires `tiktoken` or `claude-tokenizer`; deferred until a consumer requests pinpoint numbers.",
+              "- The `caveman-v1` output-side verdict (`vs_terse` median −9.27%) is orthogonal — input-side savings apply to the always-loaded memory budget, not the reply stream.",
+              "",
+              "## Interpretation",
+              "",
+              "- **Thin-Root files net negative.** `AGENTS.md` and `templates/AGENTS.md` already follow `agents-md-thin-root` (≥ 40 % pointer ratio). The compressor's frontmatter pair adds more bytes than the sparse prose loses. **Do not compress Thin-Root files.**",
+              "- **Prose-heavy contract docs net 3–6 % saving.** Useful but modest. Pays off when the file is large and frequently loaded.",
+              "- **Rule of thumb:** target files with > 5 KB and visible paragraph prose; skip pointer-only files.",
+              ""]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    rows = [bench_one(p, c) for p, c in CORPUS]
+    payload = {
+        "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "schema": "caveman-v2",
+        "rows": rows,
+        "aggregate": aggregate(rows),
+    }
+    REPORT_JSON.parent.mkdir(parents=True, exist_ok=True)
+    REPORT_JSON.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    REPORT_MD.write_text(render_md(payload), encoding="utf-8")
+    print(f"wrote: {REPORT_JSON}")
+    print(f"wrote: {REPORT_MD}")
+    print(f"median saving: {payload['aggregate']['median_saving_pct']:+.2f}%")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())