@event4u/agent-config 2.20.0 → 2.21.0

package/scripts/_lib/bench_caveman_report.py

@@ -0,0 +1,152 @@
+# Caveman bench report serializer — step-16 Phase 1 Step 5.
+#
+# Emits the caveman-v1 JSON + Markdown shape. Distinct schema_version
+# ("caveman-v1") from the selection-accuracy bench (v1) because the
+# blocks are disjoint: caveman has no `selection`/`quality`, and the
+# selection bench has no three-arm compression metrics.
+"""Caveman bench report serializer."""
+from __future__ import annotations
+
+from typing import Any
+
+from _lib.bench_caveman import ARMS, PromptResult, aggregate_results, compute_cost
+
+
+def build_caveman_report(
+    *,
+    results: list[PromptResult],
+    corpus_path_rel: str,
+    generated_at: str,
+    bench_run_version: str,
+    model: str,
+    transport: str,
+    pricing_rates: dict[str, float],
+    pricing_sourced_on: str | None,
+) -> dict[str, Any]:
+    aggregate = aggregate_results(results)
+    cost = compute_cost(results, pricing_rates)
+    cost["source"] = "live-api"
+    cost["model"] = model
+    cost["pricing_sourced_on"] = pricing_sourced_on
+    errors = cost["totals"]["errors"]
+    return {
+        "schema_version": "caveman-v1",
+        "generated_at": generated_at,
+        "corpus": {
+            "id": "caveman",
+            "path": corpus_path_rel,
+            "prompt_count": len(results),
+        },
+        "runner": {
+            "bench_run_version": bench_run_version,
+            "transport": transport,
+            "model": model,
+        },
+        "caveman": {
+            "arms": list(ARMS),
+            "aggregate": aggregate,
+            "per_prompt": [_prompt_block(r) for r in results],
+        },
+        "cost": cost,
+        "verdict": {
+            "overall": "measured" if errors == 0 else "partial",
+            "errors": errors,
+        },
+    }
+
+
+def _prompt_block(r: PromptResult) -> dict[str, Any]:
+    return {
+        "id": r.id,
+        "category": r.category,
+        "expected_carve_out_pct": r.expected_carve_out_pct,
+        "realised_carve_out_pct": (
+            r.arms["compressed"].realised_carve_out_pct
+            if "compressed" in r.arms else None
+        ),
+        "savings_vs_raw": r.savings_vs_raw,
+        "savings_vs_terse": r.savings_vs_terse,
+        "arms": {
+            arm: {
+                "input_tokens": ar.input_tokens,
+                "output_tokens": ar.output_tokens,
+                "latency_ms": ar.latency_ms,
+                "output_chars": ar.output_chars,
+                "carve_out_chars": ar.carve_out_chars,
+                "error": ar.error,
+                "text": ar.text,
+            }
+            for arm, ar in r.arms.items()
+        },
+    }
+
+
+def _fmt_pct(x: float | None) -> str:
+    return f"{x:.2%}" if isinstance(x, (int, float)) else "—"
+
+
+def render_caveman_markdown(report: dict[str, Any]) -> str:
+    cv = report["caveman"]
+    agg = cv["aggregate"]
+    cost = report["cost"]
+    head = [
+        f"# Caveman Bench Report — `caveman` · {report['generated_at']}",
+        "",
+        "## Headline",
+        "",
+        f"- prompts: **{report['corpus']['prompt_count']}** · "
+        f"arms: **{', '.join(cv['arms'])}** · "
+        f"model: **{report['runner']['model']}** · "
+        f"transport: **{report['runner']['transport']}**",
+        f"- median savings vs raw: **{_fmt_pct(agg['savings_vs_raw']['median'])}** "
+        f"(p10 {_fmt_pct(agg['savings_vs_raw']['p10'])} · p90 {_fmt_pct(agg['savings_vs_raw']['p90'])})",
+        f"- median savings vs terse-control: **{_fmt_pct(agg['savings_vs_terse']['median'])}** "
+        f"(p10 {_fmt_pct(agg['savings_vs_terse']['p10'])} · p90 {_fmt_pct(agg['savings_vs_terse']['p90'])})",
+        f"- median realised carve-out share (compressed arm): **{_fmt_pct(agg['realised_carve_out_pct']['median'])}** "
+        f"(expected median {_fmt_pct(agg['expected_carve_out_pct']['median'])})",
+        f"- total cost: **${cost['totals']['total_cost_usd']:.6f}** "
+        f"(calls {cost['totals']['calls']} · errors {cost['totals']['errors']})",
+        f"- verdict: **{report['verdict']['overall']}**",
+        "",
+    ]
+    per_arm = [
+        "## Per-arm token totals",
+        "",
+        "| arm | calls | input_tokens | output_tokens | median out/prompt |",
+        "|---|---:|---:|---:|---:|",
+    ]
+    for arm in cv["arms"]:
+        a = cost["per_arm"][arm]
+        m = agg["output_tokens"][arm]["median"]
+        per_arm.append(
+            f"| `{arm}` | {a['calls']} | {a['input_tokens']} | {a['output_tokens']} | {m:.0f} |"
+        )
+    per_arm.append("")
+    per_prompt = [
+        "## Per-prompt results",
+        "",
+        "| id | category | exp.carve | real.carve | out.compressed | out.terse | out.uncompressed | vs raw | vs terse |",
+        "|---|---|---:|---:|---:|---:|---:|---:|---:|",
+    ]
+    for r in cv["per_prompt"]:
+        arms = r["arms"]
+        oc = arms.get("compressed", {}).get("output_tokens", "—")
+        ot = arms.get("terse_control", {}).get("output_tokens", "—")
+        ou = arms.get("uncompressed", {}).get("output_tokens", "—")
+        per_prompt.append(
+            f"| `{r['id']}` | {r['category']} | "
+            f"{_fmt_pct(r['expected_carve_out_pct'])} | {_fmt_pct(r['realised_carve_out_pct'])} | "
+            f"{oc} | {ot} | {ou} | "
+            f"{_fmt_pct(r['savings_vs_raw'])} | {_fmt_pct(r['savings_vs_terse'])} |"
+        )
+    per_prompt.append("")
+    notes = [
+        "## Notes",
+        "",
+        f"- corpus: `{report['corpus']['path']}`",
+        f"- pricing: `bench/pricing.yaml` (sourced {cost.get('pricing_sourced_on') or '—'})",
+        f"- schema: `caveman-v1` (see `docs/contracts/benchmark-report-schema.md`)",
+        f"- bench_run version: `{report['runner']['bench_run_version']}`",
+        "",
+    ]
+    return "\n".join(head + per_arm + per_prompt + notes)

package/scripts/bench_compress_memory.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Offline bench for input-side memory compression (Phase 2 / Step 11).
+
+Runs `compress_memory.py` over a fixed corpus of memory-target files, records
+pre/post char counts, approximates input-token savings (chars / 4 — the
+GPT-4 / Claude rule of thumb), and emits `bench/reports/caveman-v2.{json,md}`.
+
+Offline (no API calls). Cadence-aligned with `docs/benchmarks.md`. Citation
+in `bench/reports/caveman-v2.md` notes the chars→tokens approximation and
+points at upstream tiktoken / claude-tokenizer if a calibrated number is
+later needed.
+"""
+from __future__ import annotations
+
+import json
+import shutil
+import statistics
+import subprocess
+import sys
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+COMPRESS_SCRIPT = REPO_ROOT / "scripts" / "compress_memory.py"
+REPORT_JSON = REPO_ROOT / "bench" / "reports" / "caveman-v2.json"
+REPORT_MD = REPO_ROOT / "bench" / "reports" / "caveman-v2.md"
+
+CORPUS: list[tuple[str, str]] = [
+    ("AGENTS.md", "thin-root-package"),
+    (".agent-src.uncompressed/templates/AGENTS.md", "thin-root-consumer-template"),
+    (".agent-src/templates/AGENTS.md", "thin-root-consumer-generated"),
+    ("docs/contracts/ai-council-config.md", "prose-heavy-contract"),
+    ("docs/contracts/implement-ticket-flow.md", "prose-heavy-contract"),
+    ("docs/contracts/command-clusters.md", "prose-heavy-contract"),
+    ("docs/contracts/mental-models.md", "prose-heavy-contract"),
+    ("docs/contracts/kernel-membership.md", "prose-heavy-contract"),
+    ("docs/contracts/load-context-budget-model.md", "prose-heavy-contract"),
+    ("docs/contracts/mcp-cloud-scope.md", "prose-heavy-contract"),
+    ("docs/contracts/context-spine.md", "prose-heavy-contract"),
+    ("docs/contracts/rule-classification.md", "rule-classification"),
+]
+
+
+def chars_to_tokens(n: int) -> int:
+    """Approximate token count via chars / 4 (GPT-4/Claude English heuristic)."""
+    return round(n / 4)
+
+
+def bench_one(rel_path: str, category: str) -> dict:
+    src = REPO_ROOT / rel_path
+    if not src.is_file():
+        return {"path": rel_path, "category": category, "error": "not-found"}
+    with tempfile.TemporaryDirectory() as tmp:
+        target = Path(tmp) / src.name
+        shutil.copy(src, target)
+        before_chars = target.stat().st_size
+        result = subprocess.run(
+            [sys.executable, str(COMPRESS_SCRIPT), str(target)],
+            capture_output=True, text=True, cwd=REPO_ROOT,
+        )
+        if result.returncode != 0:
+            return {"path": rel_path, "category": category,
+                    "error": f"exit-{result.returncode}", "stderr": result.stderr[:200]}
+        after_chars = target.stat().st_size
+        before_tok = chars_to_tokens(before_chars)
+        after_tok = chars_to_tokens(after_chars)
+        return {
+            "path": rel_path,
+            "category": category,
+            "before_chars": before_chars,
+            "after_chars": after_chars,
+            "delta_chars": after_chars - before_chars,
+            "saving_pct_chars": (before_chars - after_chars) * 100 / before_chars,
+            "before_tokens_est": before_tok,
+            "after_tokens_est": after_tok,
+            "delta_tokens_est": after_tok - before_tok,
+            "saving_pct_tokens_est": (before_tok - after_tok) * 100 / before_tok if before_tok else 0.0,
+        }
+
+
+def aggregate(rows: list[dict]) -> dict:
+    rows_ok = [r for r in rows if "error" not in r]
+    savings = [r["saving_pct_chars"] for r in rows_ok]
+    by_cat: dict[str, list[float]] = {}
+    for r in rows_ok:
+        by_cat.setdefault(r["category"], []).append(r["saving_pct_chars"])
+    return {
+        "calls": len(rows),
+        "errors": len(rows) - len(rows_ok),
+        "median_saving_pct": statistics.median(savings) if savings else 0.0,
+        "p10_saving_pct": statistics.quantiles(savings, n=10)[0] if len(savings) >= 10 else min(savings, default=0.0),
+        "p90_saving_pct": statistics.quantiles(savings, n=10)[8] if len(savings) >= 10 else max(savings, default=0.0),
+        "stdev_saving_pct": statistics.pstdev(savings) if len(savings) > 1 else 0.0,
+        "total_chars_saved": sum(r["before_chars"] - r["after_chars"] for r in rows_ok),
+        "total_tokens_est_saved": sum(r["before_tokens_est"] - r["after_tokens_est"] for r in rows_ok),
+        "by_category_median_pct": {k: statistics.median(v) for k, v in by_cat.items()},
+    }
+
+
+def render_md(payload: dict) -> str:
+    agg = payload["aggregate"]
+    lines = [
+        "# caveman-v2 — input-side memory compression bench",
+        "",
+        f"**Generated:** {payload['generated_at']}",
+        f"**Schema:** `caveman-v2` (input-side; offline; chars→tokens via /4 heuristic)",
+        f"**Script:** `scripts/bench_compress_memory.py`",
+        "",
+        "## Headline",
+        "",
+        f"- Median char saving: **{agg['median_saving_pct']:+.2f}%** (p10 {agg['p10_saving_pct']:+.2f}% · p90 {agg['p90_saving_pct']:+.2f}%)",
+        f"- Total chars saved across corpus: **{agg['total_chars_saved']:+,}**",
+        f"- Total tokens (estimate) saved across corpus: **{agg['total_tokens_est_saved']:+,}**",
+        f"- Files: {agg['calls']} · errors: {agg['errors']}",
+        "",
+        "## By category (median %)",
+        "",
+        "| Category | Median saving |",
+        "|---|---:|",
+    ]
+    for cat, med in sorted(agg["by_category_median_pct"].items()):
+        lines.append(f"| {cat} | {med:+.2f}% |")
+    lines += ["", "## Per file", "",
+              "| Path | Category | Before | After | Δ chars | Saving % |",
+              "|---|---|---:|---:|---:|---:|"]
+    for r in payload["rows"]:
+        if "error" in r:
+            lines.append(f"| `{r['path']}` | {r['category']} | — | — | — | {r['error']} |")
+        else:
+            lines.append(
+                f"| `{r['path']}` | {r['category']} | {r['before_chars']:,} | {r['after_chars']:,} | "
+                f"{r['delta_chars']:+,} | {r['saving_pct_chars']:+.2f}% |"
+            )
+    lines += ["", "## Methodology",
+              "",
+              "- Offline run: `compress_memory.py` writes `.original.md` backup + frontmatter (`original_sha256`, `compressed_at`). The frontmatter pair (≈ 120 chars) is the fixed compression tax — files with little prose net negative.",
+              "- chars → tokens approximation: `tokens ≈ chars / 4` (GPT-4 / Claude English rule of thumb). Calibrated number requires `tiktoken` or `claude-tokenizer`; deferred until a consumer requests pinpoint numbers.",
+              "- The `caveman-v1` output-side verdict (`vs_terse` median −9.27%) is orthogonal — input-side savings apply to the always-loaded memory budget, not the reply stream.",
+              "",
+              "## Interpretation",
+              "",
+              "- **Thin-Root files net negative.** `AGENTS.md` and `templates/AGENTS.md` already follow `agents-md-thin-root` (≥ 40 % pointer ratio). The compressor's frontmatter pair adds more bytes than the sparse prose loses. **Do not compress Thin-Root files.**",
+              "- **Prose-heavy contract docs net 3–6 % saving.** Useful but modest. Pays off when the file is large and frequently loaded.",
+              "- **Rule of thumb:** target files with > 5 KB and visible paragraph prose; skip pointer-only files.",
+              ""]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    rows = [bench_one(p, c) for p, c in CORPUS]
+    payload = {
+        "generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "schema": "caveman-v2",
+        "rows": rows,
+        "aggregate": aggregate(rows),
+    }
+    REPORT_JSON.parent.mkdir(parents=True, exist_ok=True)
+    REPORT_JSON.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
+    REPORT_MD.write_text(render_md(payload), encoding="utf-8")
+    print(f"wrote: {REPORT_JSON}")
+    print(f"wrote: {REPORT_MD}")
+    print(f"median saving: {payload['aggregate']['median_saving_pct']:+.2f}%")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/bench_run.py

@@ -21,6 +21,7 @@ from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
+sys.path.insert(0, str(REPO_ROOT))
 
 from _lib import script_output  # type: ignore[import-not-found]  # noqa: E402
 from _lib.bench_cost import aggregate_sessions  # noqa: E402
@@ -33,6 +34,9 @@ from _lib.bench_report import (  # noqa: E402
     write_json,
     write_markdown,
 )
+from _lib import bench_caveman  # noqa: E402
+from _lib.bench_caveman_report import build_caveman_report, render_caveman_markdown  # noqa: E402
+from _lib.bench_cost import load_pricing  # noqa: E402
 from bench_runner import run_corpus  # noqa: E402
 
 try:
@@ -41,11 +45,12 @@ except ImportError:
     script_output.error("error: PyYAML required (pip install pyyaml)")
     sys.exit(2)
 
-BENCH_RUN_VERSION = "0.1.0"
+BENCH_RUN_VERSION = "0.2.0"
 PRICING_PATH = REPO_ROOT / "bench" / "pricing.yaml"
 SESSIONS_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
 REPORTS_DIR = REPO_ROOT / "bench" / "reports"
 CORPUS_DIR = REPO_ROOT / "tests" / "eval"
+CAVEMAN_CORPUS = REPO_ROOT / "bench" / "corpora" / "caveman" / "prompts.yaml"
 BASELINE_COLLECTOR = REPO_ROOT / "scripts" / "bench_runner.py"
 
 
@@ -110,8 +115,21 @@ def main(argv: list[str] | None = None) -> int:
                     help="Override timestamp (test hook); defaults to UTC now")
     ap.add_argument("--no-write", action="store_true",
                     help="Compute the report but do not write files (dry run)")
+    ap.add_argument("--caveman", action="store_true",
+                    help="Run the caveman three-arm compression bench instead of the "
+                         "selection-accuracy bench (step-16 Phase 1).")
+    ap.add_argument("--caveman-max-prompts", type=int, default=None,
+                    help="Cap prompts in the caveman bench (smoke test).")
+    ap.add_argument("--caveman-dry-run", action="store_true",
+                    help="Caveman: skip live API calls; emit a stub report with "
+                         "zero tokens (wiring check only).")
+    ap.add_argument("--caveman-report-tag", default="caveman-v1",
+                    help="Filename tag for the caveman report (default: caveman-v1).")
     args = ap.parse_args(argv)
 
+    if args.caveman:
+        return _run_caveman(args)
+
     corpus_path = CORPUS_DIR / f"corpus-{args.corpus}.yaml"
     if not corpus_path.is_file():
         script_output.error(f"error: corpus not found: {corpus_path}")
@@ -151,5 +169,105 @@ def main(argv: list[str] | None = None) -> int:
     return 0 if verdict["overall"] in ("pass", "partial") else 1
 
 
+class _DryRunClient:
+    """Stub client for --caveman-dry-run. Returns empty CouncilResponse-shaped objects."""
+
+    def ask(self, system_prompt: str, user_prompt: str, max_tokens: int = 1024):
+        from ai_council.clients import CouncilResponse
+        return CouncilResponse(
+            provider="dry-run", model="stub", text="",
+            input_tokens=0, output_tokens=0, latency_ms=0, error=None,
+        )
+
+
+def _build_anthropic_client():
+    from ai_council.clients import AnthropicClient, load_anthropic_key
+    return AnthropicClient(api_key=load_anthropic_key())
+
+
+def _run_caveman(args: argparse.Namespace) -> int:
+    if not CAVEMAN_CORPUS.is_file():
+        script_output.error(f"error: caveman corpus not found: {CAVEMAN_CORPUS}")
+        return 2
+
+    if args.caveman_dry_run:
+        client = _DryRunClient()
+        transport = "dry-run"
+        model = "stub"
+    else:
+        try:
+            client = _build_anthropic_client()
+        except Exception as exc:  # noqa: BLE001
+            script_output.error(f"error: cannot build Anthropic client: {exc}")
+            return 2
+        transport = "api"
+        model = getattr(client, "model", "claude-sonnet-4-5")
+
+    def _progress(done: int, total: int, pid: str, arm: str, ar) -> None:
+        if args.quiet:
+            return
+        err = f" ERR={ar.error}" if ar.error else ""
+        print(f"[{done:>3}/{total}] {pid} · {arm:<14} "
+              f"in={ar.input_tokens:>4} out={ar.output_tokens:>4} "
+              f"{ar.latency_ms:>5}ms{err}", file=sys.stderr)
+
+    results = bench_caveman.run_caveman_bench(
+        client, CAVEMAN_CORPUS,
+        max_prompts=args.caveman_max_prompts,
+        on_progress=_progress,
+    )
+
+    rates, sourced_on = load_pricing(PRICING_PATH)
+    sonnet_rates = rates.get("sonnet", {"input": 0.0, "output": 0.0})
+
+    report = build_caveman_report(
+        results=results,
+        corpus_path_rel=str(CAVEMAN_CORPUS.relative_to(REPO_ROOT)),
+        generated_at=utc_now_iso(),
+        bench_run_version=BENCH_RUN_VERSION,
+        model=model,
+        transport=transport,
+        pricing_rates=sonnet_rates,
+        pricing_sourced_on=sourced_on,
+    )
+
+    stamp = args.stamp or utc_now_filename_stamp()
+    json_path, md_path = report_paths(REPORTS_DIR, args.caveman_report_tag, stamp)
+    # Override: caveman roadmap pins the filename to `caveman-v1.{json,md}` (no stamp).
+    fixed_json = REPORTS_DIR / f"{args.caveman_report_tag}.json"
+    fixed_md = REPORTS_DIR / f"{args.caveman_report_tag}.md"
+
+    if not args.no_write:
+        write_json(fixed_json, report)
+        fixed_md.parent.mkdir(parents=True, exist_ok=True)
+        fixed_md.write_text(render_caveman_markdown(report), encoding="utf-8")
+        # Also drop a timestamped copy for the cadence trail.
+        write_json(json_path, report)
+        json_path.with_suffix(".md").write_text(
+            render_caveman_markdown(report), encoding="utf-8"
+        )
+
+    cost = report["cost"]
+    headline = (
+        f"caveman · prompts {report['corpus']['prompt_count']} · "
+        f"calls {cost['totals']['calls']} · errors {cost['totals']['errors']} · "
+        f"vs_raw med {report['caveman']['aggregate']['savings_vs_raw']['median']:.2%} · "
+        f"vs_terse med {report['caveman']['aggregate']['savings_vs_terse']['median']:.2%} · "
+        f"cost ${cost['totals']['total_cost_usd']:.6f}"
+    )
+    if args.quiet:
+        print(headline)
+        if not args.no_write:
+            print(f"report: {fixed_md.relative_to(REPO_ROOT)}")
+    else:
+        print(render_caveman_markdown(report))
+        if not args.no_write:
+            print(f"\n→ json:     {fixed_json.relative_to(REPO_ROOT)}")
+            print(f"→ markdown: {fixed_md.relative_to(REPO_ROOT)}")
+            print(f"→ trail:    {json_path.relative_to(REPO_ROOT)}")
+
+    return 0 if cost["totals"]["errors"] == 0 else 1
+
+
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

package/scripts/caveman_stats.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Caveman per-session / per-conversation / lifetime token-delta lens.
+
+Reads sessions.jsonl, groups by sessionId + conversation_id, emits per-row
+caveman delta tokens. Honors the suspended-multiplier contract in
+`docs/contracts/caveman-telemetry.md` (delta = 0 while suspended).
+"""
+from __future__ import annotations
+import argparse, json, sys
+from collections import defaultdict
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_JSONL = REPO_ROOT / "agents" / "cost-tracking" / "sessions.jsonl"
+TELEMETRY_DOC = REPO_ROOT / "docs" / "contracts" / "caveman-telemetry.md"
+
+# Mirrors `docs/contracts/caveman-telemetry.md` `v1` constants.
+MULTIPLIER_VERSION = "v1"
+MULTIPLIER_VALUE = 0.9155
+MULTIPLIER_ACTIVE = False  # suspended pending v2
+
+
+def _load(path: Path) -> list[dict]:
+    if not path.is_file():
+        return []
+    rows: list[dict] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        try:
+            rows.append(json.loads(line))
+        except json.JSONDecodeError:
+            continue
+    return rows
+
+
+def _delta(row: dict) -> int:
+    """Per-row delta with suspended-multiplier guard."""
+    if not MULTIPLIER_ACTIVE:
+        return 0
+    explicit = row.get("caveman_delta_tokens")
+    if isinstance(explicit, (int, float)):
+        return int(explicit)
+    compressed = row.get("caveman_compressed_tokens")
+    if isinstance(compressed, (int, float)) and compressed > 0:
+        return int(compressed * MULTIPLIER_VALUE - compressed)
+    return 0
+
+
+def aggregate(rows: list[dict]) -> dict:
+    _zero = lambda: {"sessions": 0, "delta_tokens": 0, "compressed_tokens": 0}
+    by_session: dict[str, dict] = defaultdict(_zero)
+    by_conv: dict[str, dict] = defaultdict(_zero)
+    lifetime = _zero()
+    for row in rows:
+        sid = str(row.get("sessionId") or row.get("session_id") or "unknown")
+        cid = str(row.get("conversation_id") or "unknown")
+        delta = _delta(row)
+        comp = int(row.get("caveman_compressed_tokens") or 0)
+        for bucket in (by_session[sid], by_conv[cid], lifetime):
+            bucket["sessions"] += 1
+            bucket["delta_tokens"] += delta
+            bucket["compressed_tokens"] += comp
+    return {
+        "schema_version": "caveman-stats/v1",
+        "multiplier_version": MULTIPLIER_VERSION,
+        "multiplier_value": MULTIPLIER_VALUE,
+        "multiplier_active": MULTIPLIER_ACTIVE,
+        "lifetime": lifetime,
+        "by_session": dict(by_session),
+        "by_conversation": dict(by_conv),
+    }
+
+
+def render_text(report: dict) -> str:
+    lines = [
+        f"caveman-stats {report['schema_version']} · multiplier {report['multiplier_version']}"
+        f" ({'ACTIVE' if report['multiplier_active'] else 'SUSPENDED'}) · "
+        f"value {report['multiplier_value']:.4f}",
+        "",
+        f"  lifetime: {report['lifetime']['sessions']} sessions · "
+        f"delta_tokens = {report['lifetime']['delta_tokens']:+,} · "
+        f"compressed_tokens = {report['lifetime']['compressed_tokens']:,}",
+        "",
+        "  by conversation:",
+    ]
+    for cid, b in sorted(report["by_conversation"].items()):
+        lines.append(
+            f"    {cid}: {b['sessions']} sessions · "
+            f"delta = {b['delta_tokens']:+,} · compressed = {b['compressed_tokens']:,}"
+        )
+    if not report["multiplier_active"]:
+        lines += [
+            "",
+            "  Note: multiplier suspended — see docs/contracts/caveman-telemetry.md",
+            "  (delta_tokens = 0 until kill-criterion satisfied in caveman-v2).",
+        ]
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--input", type=Path, default=DEFAULT_JSONL)
+    parser.add_argument("--format", choices=["text", "json"], default="text")
+    args = parser.parse_args(argv)
+
+    rows = _load(args.input)
+    report = aggregate(rows)
+
+    if args.format == "json":
+        print(json.dumps(report, indent=2))
+    else:
+        print(render_text(report))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/check_command_count_messaging.py

@@ -17,7 +17,7 @@ Canonical counts:
 Patterns checked (per file):
 
     README.md
-      hero row     "<strong>{N} Commands</strong>"            → active
+      hero badge   "/badge/Commands-{N}-…"                    → active
       browse line  "Browse all {N} active commands"           → active
       browse meta  "{N} files total"                          → total
       browse meta  "{N} are deprecation shims"                → shims
@@ -84,7 +84,7 @@ def main() -> int:
 
     checks = [
         # README.md
-        (README, r"<strong>(\d+) Commands</strong>", active, "hero row"),
+        (README, r"/badge/Commands-(\d+)-", active, "hero badge"),
         (README, r"Browse all (\d+) active commands", active, "browse line"),
         (README, r"\+ (\d+) native commands\)", active, "tools blurb"),
         # docs/getting-started.md