@jairodri/huckleberry-cli 1.0.1 → 1.1.0

package/skills/claw-compactor/benchmark/results/benchmark_results.json

@@ -0,0 +1,405 @@
+{
+  "results": [
+    {
+      "sample_id": "sample-01-devops",
+      "compressor": "NoCompression",
+      "original_tokens": 4404,
+      "compressed_tokens": 4404,
+      "compression_ratio": 1.0,
+      "space_saving_pct": 0.0,
+      "rouge_l_f1": 1.0,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 1.0,
+      "info_retention_f1": 1.0,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.0,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-01T09:01:00Z] USER: Hey, our CI pipeline is failing on the docker build step. Getting 'no space left on device'.\n\n[2026-03-01T09:01:15Z] ASSISTANT: That's a classic Docker disk space issue. F",
+      "sample_description": "DevOps troubleshooting session \u2014 Docker / CI pipeline",
+      "channel": "discord-tech"
+    },
+    {
+      "sample_id": "sample-01-devops",
+      "compressor": "RandomDrop",
+      "original_tokens": 4404,
+      "compressed_tokens": 3728,
+      "compression_ratio": 0.8465,
+      "space_saving_pct": 15.3,
+      "rouge_l_f1": 0.9072,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 0.8522,
+      "info_retention_f1": 0.9474,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.4,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-01T09:01:00Z] USER: Hey, our CI pipeline is failing on the docker build step. Getting 'no space left on device'.\n\n[2026-03-01T09:01:15Z] ASSISTANT: That's a classic Docker disk space issue. F",
+      "sample_description": "DevOps troubleshooting session \u2014 Docker / CI pipeline",
+      "channel": "discord-tech"
+    },
+    {
+      "sample_id": "sample-01-devops",
+      "compressor": "RuleCompressor",
+      "original_tokens": 4404,
+      "compressed_tokens": 3883,
+      "compression_ratio": 0.8817,
+      "space_saving_pct": 11.8,
+      "rouge_l_f1": 0.9041,
+      "rouge_l_precision": 0.9731,
+      "rouge_l_recall": 0.8616,
+      "info_retention_f1": 0.9655,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 8.5,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-01T09:01:00Z] USER: Hey, our CI pipeline is failing on the docker build step. Getting 'no space left on device'.\n\n[2026-03-01T09:01:15Z] ASSISTANT: That's a classic Docker disk space issue. F",
+      "sample_description": "DevOps troubleshooting session \u2014 Docker / CI pipeline",
+      "channel": "discord-tech"
+    },
+    {
+      "sample_id": "sample-01-devops",
+      "compressor": "Engram",
+      "original_tokens": 4404,
+      "compressed_tokens": 536,
+      "compression_ratio": 0.1217,
+      "space_saving_pct": 87.8,
+      "rouge_l_f1": 0.0288,
+      "rouge_l_precision": 0.172,
+      "rouge_l_recall": 0.0183,
+      "info_retention_f1": 0.4516,
+      "info_retention_recall": 0.7,
+      "keywords_original": 30,
+      "keywords_found": 21,
+      "latency_ms": 43493.9,
+      "llm_calls": 2,
+      "compressed_preview": "## Key Context\n\n- Self-hosted GHA runner (Docker Compose stack) suffered cascading disk/memory failures resolved 2026-03-01; full K8s+AWS migration now planned\n- Node.js service had WebSocket listener",
+      "sample_description": "DevOps troubleshooting session \u2014 Docker / CI pipeline",
+      "channel": "discord-tech"
+    },
+    {
+      "sample_id": "sample-02-trading",
+      "compressor": "NoCompression",
+      "original_tokens": 3460,
+      "compressed_tokens": 3460,
+      "compression_ratio": 1.0,
+      "space_saving_pct": 0.0,
+      "rouge_l_f1": 1.0,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 1.0,
+      "info_retention_f1": 1.0,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.0,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-02T08:00:00Z] USER: Let's discuss backtesting methodology. I'm seeing significant overfitting in my momentum strategy.\n\n[2026-03-02T08:00:20Z] ASSISTANT: Overfitting in momentum strategies is",
+      "sample_description": "Quantitative trading strategy discussion session",
+      "channel": "discord-quant"
+    },
+    {
+      "sample_id": "sample-02-trading",
+      "compressor": "RandomDrop",
+      "original_tokens": 3460,
+      "compressed_tokens": 2640,
+      "compression_ratio": 0.763,
+      "space_saving_pct": 23.7,
+      "rouge_l_f1": 0.8466,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 0.7651,
+      "info_retention_f1": 0.8462,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.3,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-02T08:00:00Z] USER: Let's discuss backtesting methodology. I'm seeing significant overfitting in my momentum strategy.\n\n[2026-03-02T08:00:20Z] ASSISTANT: Overfitting in momentum strategies is",
+      "sample_description": "Quantitative trading strategy discussion session",
+      "channel": "discord-quant"
+    },
+    {
+      "sample_id": "sample-02-trading",
+      "compressor": "RuleCompressor",
+      "original_tokens": 3460,
+      "compressed_tokens": 2983,
+      "compression_ratio": 0.8621,
+      "space_saving_pct": 13.8,
+      "rouge_l_f1": 0.8869,
+      "rouge_l_precision": 0.9787,
+      "rouge_l_recall": 0.8326,
+      "info_retention_f1": 0.9474,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 5.4,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-02T08:00:00Z] USER: Let's discuss backtesting methodology. I'm seeing significant overfitting in my momentum strategy.\n\n[2026-03-02T08:00:20Z] ASSISTANT: Overfitting in momentum strategies is",
+      "sample_description": "Quantitative trading strategy discussion session",
+      "channel": "discord-quant"
+    },
+    {
+      "sample_id": "sample-02-trading",
+      "compressor": "Engram",
+      "original_tokens": 3460,
+      "compressed_tokens": 456,
+      "compression_ratio": 0.1318,
+      "space_saving_pct": 86.8,
+      "rouge_l_f1": 0.0459,
+      "rouge_l_precision": 0.216,
+      "rouge_l_recall": 0.0296,
+      "info_retention_f1": 0.4071,
+      "info_retention_recall": 0.6333,
+      "keywords_original": 30,
+      "keywords_found": 19,
+      "latency_ms": 38831.7,
+      "llm_calls": 2,
+      "compressed_preview": "## Key Context\n- **Strategy**: Crypto perps (Binance + OKX), 4\u201324hr holds, 5-min OHLCV; signals: RSI + volume momentum + funding rate; severe overfitting confirmed (IS Sharpe 2.3 vs OOS 0.4; 12 params",
+      "sample_description": "Quantitative trading strategy discussion session",
+      "channel": "discord-quant"
+    },
+    {
+      "sample_id": "sample-03-ml-short",
+      "compressor": "NoCompression",
+      "original_tokens": 1856,
+      "compressed_tokens": 1856,
+      "compression_ratio": 1.0,
+      "space_saving_pct": 0.0,
+      "rouge_l_f1": 1.0,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 1.0,
+      "info_retention_f1": 1.0,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.0,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-03T14:00:00Z] USER: I'm training a text classification model. Getting 68% accuracy, want to improve.\n\n[2026-03-03T14:00:20Z] ASSISTANT: 68% is a reasonable starting point. What's your dataset",
+      "sample_description": "Short ML model training session",
+      "channel": "discord-ml"
+    },
+    {
+      "sample_id": "sample-03-ml-short",
+      "compressor": "RandomDrop",
+      "original_tokens": 1856,
+      "compressed_tokens": 1456,
+      "compression_ratio": 0.7845,
+      "space_saving_pct": 21.6,
+      "rouge_l_f1": 0.8364,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 0.751,
+      "info_retention_f1": 0.8889,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.2,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-03T14:00:00Z] USER: I'm training a text classification model. Getting 68% accuracy, want to improve.\n\n[2026-03-03T14:00:20Z] ASSISTANT: 68% is a reasonable starting point. What's your dataset",
+      "sample_description": "Short ML model training session",
+      "channel": "discord-ml"
+    },
+    {
+      "sample_id": "sample-03-ml-short",
+      "compressor": "RuleCompressor",
+      "original_tokens": 1856,
+      "compressed_tokens": 1744,
+      "compression_ratio": 0.9397,
+      "space_saving_pct": 6.0,
+      "rouge_l_f1": 0.9407,
+      "rouge_l_precision": 0.9782,
+      "rouge_l_recall": 0.9163,
+      "info_retention_f1": 0.9474,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 3.1,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-03T14:00:00Z] USER: I'm training a text classification model. Getting 68% accuracy, want to improve.\n\n[2026-03-03T14:00:20Z] ASSISTANT: 68% is a reasonable starting point. What's your dataset",
+      "sample_description": "Short ML model training session",
+      "channel": "discord-ml"
+    },
+    {
+      "sample_id": "sample-03-ml-short",
+      "compressor": "Engram",
+      "original_tokens": 1856,
+      "compressed_tokens": 288,
+      "compression_ratio": 0.1552,
+      "space_saving_pct": 84.5,
+      "rouge_l_f1": 0.0547,
+      "rouge_l_precision": 0.2349,
+      "rouge_l_recall": 0.0357,
+      "info_retention_f1": 0.384,
+      "info_retention_recall": 0.5333,
+      "keywords_original": 30,
+      "keywords_found": 16,
+      "latency_ms": 18680.8,
+      "llm_calls": 2,
+      "compressed_preview": "## Key Context\n- **Project:** 3-class text classification (balanced, 10k samples); baseline BERT-base-uncased accuracy 68% \u2192 final hold-out **78.1% accuracy / 77.6% macro F1**; generalization gap 1.2%",
+      "sample_description": "Short ML model training session",
+      "channel": "discord-ml"
+    },
+    {
+      "sample_id": "sample-04-mixed-long",
+      "compressor": "NoCompression",
+      "original_tokens": 4597,
+      "compressed_tokens": 4597,
+      "compression_ratio": 1.0,
+      "space_saving_pct": 0.0,
+      "rouge_l_f1": 1.0,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 1.0,
+      "info_retention_f1": 1.0,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.0,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-04T09:00:00Z] USER: Good morning! Let's plan the microservices migration. Current monolith is 300k LOC Django.\n\n[2026-03-04T09:00:20Z] ASSISTANT: Morning! 300k LOC Django monolith is a signif",
+      "sample_description": "Mixed topics long session \u2014 system architecture, DB tuning, security",
+      "channel": "discord-general"
+    },
+    {
+      "sample_id": "sample-04-mixed-long",
+      "compressor": "RandomDrop",
+      "original_tokens": 4597,
+      "compressed_tokens": 3401,
+      "compression_ratio": 0.7398,
+      "space_saving_pct": 26.0,
+      "rouge_l_f1": 0.8261,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 0.7371,
+      "info_retention_f1": 0.8889,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.4,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-04T09:00:00Z] USER: Good morning! Let's plan the microservices migration. Current monolith is 300k LOC Django.\n\n[2026-03-04T09:00:20Z] ASSISTANT: Morning! 300k LOC Django monolith is a signif",
+      "sample_description": "Mixed topics long session \u2014 system architecture, DB tuning, security",
+      "channel": "discord-general"
+    },
+    {
+      "sample_id": "sample-04-mixed-long",
+      "compressor": "RuleCompressor",
+      "original_tokens": 4597,
+      "compressed_tokens": 4291,
+      "compression_ratio": 0.9334,
+      "space_saving_pct": 6.7,
+      "rouge_l_f1": 0.9392,
+      "rouge_l_precision": 0.9732,
+      "rouge_l_recall": 0.917,
+      "info_retention_f1": 0.9655,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 7.6,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-04T09:00:00Z] USER: Good morning! Let's plan the \u00b5svc migration. Current monolith is 300k LOC Django.\n\n[2026-03-04T09:00:20Z] ASSISTANT: Morning! 300k LOC Django monolith is a significant mig",
+      "sample_description": "Mixed topics long session \u2014 system architecture, DB tuning, security",
+      "channel": "discord-general"
+    },
+    {
+      "sample_id": "sample-04-mixed-long",
+      "compressor": "Engram",
+      "original_tokens": 4597,
+      "compressed_tokens": 449,
+      "compression_ratio": 0.0977,
+      "space_saving_pct": 90.2,
+      "rouge_l_f1": 0.0257,
+      "rouge_l_precision": 0.1869,
+      "rouge_l_recall": 0.0161,
+      "info_retention_f1": 0.4071,
+      "info_retention_recall": 0.6333,
+      "keywords_original": 30,
+      "keywords_found": 19,
+      "latency_ms": 38376.6,
+      "llm_calls": 2,
+      "compressed_preview": "## Key Context\n\n- **Project:** 300k LOC Django monolith \u2192 microservices via Strangler Fig; drivers: 50k users, 10 teams, 40-min deploys; timeline: 18\u201324 months\n- **Extraction order:** Notifications\u2192Au",
+      "sample_description": "Mixed topics long session \u2014 system architecture, DB tuning, security",
+      "channel": "discord-general"
+    },
+    {
+      "sample_id": "sample-05-sysadmin",
+      "compressor": "NoCompression",
+      "original_tokens": 3248,
+      "compressed_tokens": 3248,
+      "compression_ratio": 1.0,
+      "space_saving_pct": 0.0,
+      "rouge_l_f1": 1.0,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 1.0,
+      "info_retention_f1": 1.0,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.0,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-05T10:00:00Z] USER: Need help setting up a VPN between our office and AWS VPC. Currently using WireGuard.\n\n[2026-03-05T10:00:20Z] ASSISTANT: WireGuard for site-to-site VPN is a great choice \u2014",
+      "sample_description": "System administration and network configuration session",
+      "channel": "discord-sysops"
+    },
+    {
+      "sample_id": "sample-05-sysadmin",
+      "compressor": "RandomDrop",
+      "original_tokens": 3248,
+      "compressed_tokens": 2576,
+      "compression_ratio": 0.7931,
+      "space_saving_pct": 20.7,
+      "rouge_l_f1": 0.8431,
+      "rouge_l_precision": 1.0,
+      "rouge_l_recall": 0.7603,
+      "info_retention_f1": 0.9831,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 0.3,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-05T10:00:00Z] USER: Need help setting up a VPN between our office and AWS VPC. Currently using WireGuard.\n\n[2026-03-05T10:00:20Z] ASSISTANT: WireGuard for site-to-site VPN is a great choice \u2014",
+      "sample_description": "System administration and network configuration session",
+      "channel": "discord-sysops"
+    },
+    {
+      "sample_id": "sample-05-sysadmin",
+      "compressor": "RuleCompressor",
+      "original_tokens": 3248,
+      "compressed_tokens": 3027,
+      "compression_ratio": 0.932,
+      "space_saving_pct": 6.8,
+      "rouge_l_f1": 0.9438,
+      "rouge_l_precision": 0.9829,
+      "rouge_l_recall": 0.9185,
+      "info_retention_f1": 0.9655,
+      "info_retention_recall": 1.0,
+      "keywords_original": 30,
+      "keywords_found": 30,
+      "latency_ms": 5.6,
+      "llm_calls": 0,
+      "compressed_preview": "[2026-03-05T10:00:00Z] USER: Need help setting up a VPN between our office and AWS VPC. Currently using WireGuard.\n\n[2026-03-05T10:00:20Z] ASSISTANT: WireGuard for site-to-site VPN is a great choice -",
+      "sample_description": "System administration and network configuration session",
+      "channel": "discord-sysops"
+    },
+    {
+      "sample_id": "sample-05-sysadmin",
+      "compressor": "Engram",
+      "original_tokens": 3248,
+      "compressed_tokens": 382,
+      "compression_ratio": 0.1176,
+      "space_saving_pct": 88.2,
+      "rouge_l_f1": 0.0349,
+      "rouge_l_precision": 0.2235,
+      "rouge_l_recall": 0.022,
+      "info_retention_f1": 0.42,
+      "info_retention_recall": 0.7,
+      "keywords_original": 30,
+      "keywords_found": 21,
+      "latency_ms": 33280.2,
+      "llm_calls": 2,
+      "compressed_preview": "## Key Context\n\n- **Environment:** EC2 t3.micro, us-west-2, Elastic IP; serves office (10.0.0.0/24) over WireGuard VPN (10.200.2.0/24, port 51820); VPC 172.31.0.0/16; bare-metal\u2192AWS migration planned ",
+      "sample_description": "System administration and network configuration session",
+      "channel": "discord-sysops"
+    }
+  ],
+  "run_timestamp": "2026-03-06T00:50:38Z"
+}

package/skills/claw-compactor/benchmark/run_benchmark.py

@@ -0,0 +1,238 @@
+"""run_benchmark.py — Execute the Engram compression benchmark.
+
+Runs all compressors against all samples, collects metrics, saves results.
+
+Usage:
+    cd /path/to/claw-compactor
+    python3 benchmark/run_benchmark.py [--skip-engram] [--samples data/*.json]
+
+Results saved to benchmark/results/benchmark_results.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+# Ensure project lib is importable
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJECT_ROOT))
+sys.path.insert(0, str(PROJECT_ROOT / "scripts"))
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+from benchmark.compressors import (
+    NoCompressor,
+    RandomDropCompressor,
+    RuleCompressor,
+    EngramCompressor,
+    BaseCompressor,
+)
+from benchmark.evaluate import evaluate, messages_to_text, estimate_tokens
+
+
+# ---------------------------------------------------------------------------
+# Load benchmark samples
+# ---------------------------------------------------------------------------
+
+def load_samples(data_dir: Path) -> list[dict]:
+    """Load all JSON sample files from data_dir."""
+    samples = []
+    for f in sorted(data_dir.glob("*.json")):
+        try:
+            data = json.loads(f.read_text())
+            samples.append(data)
+            logger.info(f"Loaded sample: {f.name} ({len(data.get('messages', []))} msgs)")
+        except Exception as e:
+            logger.warning(f"Failed to load {f.name}: {e}")
+    return samples
+
+
+# ---------------------------------------------------------------------------
+# Run benchmark
+# ---------------------------------------------------------------------------
+
+def run_benchmark(
+    samples: list[dict],
+    compressors: list[BaseCompressor],
+    results_dir: Path,
+) -> list[dict]:
+    """Run all compressors against all samples and collect results."""
+
+    all_results = []
+    total_runs = len(samples) * len(compressors)
+    run_n = 0
+
+    for sample in samples:
+        sample_id = sample.get("session_id", sample.get("session_id", "unknown"))
+        messages = sample.get("messages", [])
+        original_text = messages_to_text(messages)
+        orig_tokens = estimate_tokens(original_text)
+
+        logger.info(
+            f"\n{'='*60}\nSample: {sample_id}\n"
+            f"  Messages: {len(messages)}\n"
+            f"  Original tokens: {orig_tokens}\n"
+            f"  Description: {sample.get('description', '-')}"
+        )
+
+        for compressor in compressors:
+            run_n += 1
+            logger.info(f"\n[{run_n}/{total_runs}] Running {compressor.name} on {sample_id}...")
+
+            start_ms = time.perf_counter() * 1000
+
+            try:
+                compressed_text, llm_calls = compressor.compress(messages)
+                latency_ms = time.perf_counter() * 1000 - start_ms
+
+                result = evaluate(
+                    sample_id=sample_id,
+                    compressor_name=compressor.name,
+                    original_text=original_text,
+                    compressed_text=compressed_text,
+                    latency_ms=latency_ms,
+                    llm_calls=llm_calls,
+                )
+
+                logger.info(
+                    f"  ✓ {compressor.name}: "
+                    f"ratio={result.compression_ratio:.3f} "
+                    f"({result.space_saving_pct:.1f}% saved), "
+                    f"ROUGE-L={result.rouge_l.get('f1', 0):.3f}, "
+                    f"IR-F1={result.info_retention.get('f1', 0):.3f}, "
+                    f"latency={result.latency_ms:.0f}ms, "
+                    f"LLM calls={result.llm_calls}"
+                )
+
+                result_dict = result.to_dict()
+                result_dict["sample_description"] = sample.get("description", "")
+                result_dict["channel"] = sample.get("channel", "")
+                all_results.append(result_dict)
+
+            except Exception as e:
+                latency_ms = time.perf_counter() * 1000 - start_ms
+                logger.error(f"  ✗ {compressor.name} FAILED: {e}")
+                all_results.append({
+                    "sample_id": sample_id,
+                    "compressor": compressor.name,
+                    "error": str(e),
+                    "latency_ms": round(latency_ms, 1),
+                    "original_tokens": orig_tokens,
+                    "compressed_tokens": 0,
+                    "compression_ratio": 1.0,
+                    "space_saving_pct": 0.0,
+                    "rouge_l_f1": 0.0,
+                    "info_retention_f1": 0.0,
+                    "llm_calls": 0,
+                })
+
+    # Save results
+    results_dir.mkdir(parents=True, exist_ok=True)
+    out_path = results_dir / "benchmark_results.json"
+    out_path.write_text(json.dumps({"results": all_results, "run_timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, indent=2))
+    logger.info(f"\nResults saved to {out_path}")
+
+    return all_results
+
+
+# ---------------------------------------------------------------------------
+# Summary table
+# ---------------------------------------------------------------------------
+
+def print_summary(results: list[dict]) -> None:
+    """Print a quick summary table to stdout."""
+    from collections import defaultdict
+
+    # Group by compressor
+    by_compressor: dict[str, list[dict]] = defaultdict(list)
+    for r in results:
+        if "error" not in r:
+            by_compressor[r["compressor"]].append(r)
+
+    print("\n" + "=" * 80)
+    print("BENCHMARK SUMMARY")
+    print("=" * 80)
+    header = f"{'Compressor':<22} {'Ratio':>8} {'Saved%':>8} {'ROUGE-L':>9} {'IR-F1':>8} {'Latency(ms)':>12} {'LLM calls':>10}"
+    print(header)
+    print("-" * 80)
+
+    for name, runs in by_compressor.items():
+        avg_ratio = sum(r["compression_ratio"] for r in runs) / len(runs)
+        avg_saved = sum(r["space_saving_pct"] for r in runs) / len(runs)
+        avg_rouge = sum(r["rouge_l_f1"] for r in runs) / len(runs)
+        avg_ir = sum(r["info_retention_f1"] for r in runs) / len(runs)
+        avg_lat = sum(r["latency_ms"] for r in runs) / len(runs)
+        avg_llm = sum(r["llm_calls"] for r in runs) / len(runs)
+        print(
+            f"{name:<22} {avg_ratio:>8.3f} {avg_saved:>8.1f} "
+            f"{avg_rouge:>9.3f} {avg_ir:>8.3f} "
+            f"{avg_lat:>12.0f} {avg_llm:>10.1f}"
+        )
+
+    print("=" * 80)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Engram compression benchmark")
+    parser.add_argument(
+        "--skip-engram", action="store_true",
+        help="Skip EngramCompressor (requires LLM proxy at localhost:8403)"
+    )
+    parser.add_argument(
+        "--data-dir", type=Path,
+        default=Path(__file__).parent / "data",
+        help="Directory containing sample JSON files"
+    )
+    parser.add_argument(
+        "--results-dir", type=Path,
+        default=Path(__file__).parent / "results",
+        help="Directory to save results JSON"
+    )
+    args = parser.parse_args()
+
+    # Build compressor list
+    compressors: list[BaseCompressor] = [
+        NoCompressor(),
+        RandomDropCompressor(target_ratio=0.4, seed=42),
+        RuleCompressor(),
+    ]
+    if not args.skip_engram:
+        compressors.append(EngramCompressor(use_reflector=True))
+    else:
+        logger.info("Skipping EngramCompressor (--skip-engram)")
+
+    # Load samples
+    samples = load_samples(args.data_dir)
+    if not samples:
+        logger.error(f"No sample files found in {args.data_dir}")
+        sys.exit(1)
+
+    logger.info(
+        f"\nStarting benchmark:\n"
+        f"  Samples: {len(samples)}\n"
+        f"  Compressors: {[c.name for c in compressors]}\n"
+        f"  Total runs: {len(samples) * len(compressors)}\n"
+    )
+
+    results = run_benchmark(samples, compressors, args.results_dir)
+    print_summary(results)
+
+    logger.info(f"\n✓ Benchmark complete. {len(results)} results collected.")
+
+
+if __name__ == "__main__":
+    main()