@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/tests/test_roundtrip_comprehensive.py

@@ -0,0 +1,192 @@
+"""Comprehensive roundtrip tests — dictionary, RLE, and combined must be perfectly reversible."""
+import sys
+import pytest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+
+from lib.dictionary import build_codebook, compress_text, decompress_text
+from lib.rle import compress, decompress, compress_paths, decompress_paths, compress_ip_families, decompress_ip_families
+
+
+class TestDictionaryRoundtrip:
+    """Dictionary compression must be perfectly reversible."""
+
+    @pytest.fixture
+    def codebook(self):
+        texts = [
+            "The server at 192.168.1.100 runs Python 3.11. "
+            "The server at 192.168.1.100 handles requests. "
+            "Python 3.11 is the main runtime. "
+            "The workspace at /Users/duke/workspace contains files. "
+            "/Users/duke/workspace has memory files.",
+        ]
+        return build_codebook(texts)
+
+    def test_basic_roundtrip(self, codebook):
+        original = "The server at 192.168.1.100 runs Python 3.11."
+        compressed = compress_text(original, codebook)
+        decompressed = decompress_text(compressed, codebook)
+        assert decompressed == original
+
+    def test_empty_roundtrip(self, codebook):
+        assert decompress_text(compress_text("", codebook), codebook) == ""
+
+    def test_no_matches_roundtrip(self, codebook):
+        original = "zzzz xxxx yyyy qqqq"
+        assert decompress_text(compress_text(original, codebook), codebook) == original
+
+    def test_unicode_roundtrip(self, codebook):
+        original = "中文 192.168.1.100 日本語 Python 3.11"
+        compressed = compress_text(original, codebook)
+        assert decompress_text(compressed, codebook) == original
+
+    def test_multiline_roundtrip(self, codebook):
+        original = "Line 1: server at 192.168.1.100\nLine 2: Python 3.11\nLine 3: done"
+        compressed = compress_text(original, codebook)
+        assert decompress_text(compressed, codebook) == original
+
+    def test_special_chars_roundtrip(self, codebook):
+        original = "regex: [a-z]+ and $HOME and \\n and 192.168.1.100"
+        compressed = compress_text(original, codebook)
+        assert decompress_text(compressed, codebook) == original
+
+    def test_large_text_roundtrip(self, codebook):
+        original = ("The server at 192.168.1.100 runs Python 3.11. " * 100).strip()
+        compressed = compress_text(original, codebook)
+        assert decompress_text(compressed, codebook) == original
+
+    def test_codebook_codes_dont_collide(self, codebook):
+        """Codes in output shouldn't accidentally match other entries."""
+        original = "simple text with $AA literal dollar signs"
+        compressed = compress_text(original, codebook)
+        decompressed = decompress_text(compressed, codebook)
+        assert decompressed == original
+
+    def test_all_entries_roundtrip(self, codebook):
+        """Each codebook entry individually roundtrips."""
+        for phrase, code in codebook.items():
+            compressed = compress_text(phrase, codebook)
+            decompressed = decompress_text(compressed, codebook)
+            assert decompressed == phrase, f"Failed for: {phrase}"
+
+    def test_repeated_compression_stable(self, codebook):
+        """Compressing already-compressed text then decompressing should work."""
+        original = "The server at 192.168.1.100 runs Python 3.11."
+        c1 = compress_text(original, codebook)
+        # Decompress should always get back to original
+        assert decompress_text(c1, codebook) == original
+
+
+class TestRLERoundtrip:
+    """RLE/path compression must be reversible."""
+
+    def test_path_roundtrip(self):
+        ws = "/Users/duke/workspace"
+        original = f"File at {ws}/memory/test.md and {ws}/TOOLS.md"
+        compressed = compress_paths(original, [ws])
+        decompressed = decompress_paths(compressed, ws)
+        assert decompressed == original
+
+    def test_path_no_match(self):
+        ws = "/Users/duke/workspace"
+        original = "No paths here at all"
+        compressed = compress_paths(original, [ws])
+        decompressed = decompress_paths(compressed, ws)
+        assert decompressed == original
+
+    def test_path_empty(self):
+        ws = "/Users/duke/workspace"
+        assert decompress_paths(compress_paths("", [ws]), ws) == ""
+
+    def test_ip_roundtrip(self):
+        original = "Server 192.168.1.100 and 192.168.1.200 and 192.168.1.50"
+        compressed, prefix_map = compress_ip_families(original)
+        decompressed = decompress_ip_families(compressed, prefix_map)
+        assert decompressed == original
+
+    def test_ip_no_families(self):
+        original = "No IPs here"
+        compressed, prefix_map = compress_ip_families(original)
+        decompressed = decompress_ip_families(compressed, prefix_map)
+        assert decompressed == original
+
+    def test_ip_single_occurrence(self):
+        original = "One IP: 10.0.0.1"
+        compressed, prefix_map = compress_ip_families(original)
+        decompressed = decompress_ip_families(compressed, prefix_map)
+        assert decompressed == original
+
+    def test_ip_empty(self):
+        compressed, prefix_map = compress_ip_families("")
+        assert decompress_ip_families(compressed, prefix_map) == ""
+
+    def test_mixed_content_roundtrip(self):
+        ws = "/Users/duke/workspace"
+        original = (
+            f"Server at 192.168.1.100 runs on {ws}/scripts/main.py.\n"
+            f"Backup at 192.168.1.200 uses {ws}/memory/backup.md.\n"
+            f"Another host 192.168.1.50 with {ws}/TOOLS.md"
+        )
+        # Path compression
+        p_compressed = compress_paths(original, [ws])
+        p_back = decompress_paths(p_compressed, ws)
+        assert p_back == original
+
+        # IP compression
+        i_compressed, ip_map = compress_ip_families(original)
+        i_back = decompress_ip_families(i_compressed, ip_map)
+        assert i_back == original
+
+
+class TestCombinedRoundtrip:
+    """Combined dictionary + RLE roundtrip."""
+
+    def test_dict_then_rle_roundtrip(self):
+        texts = [
+            "Python 3.11 is great. Python 3.11 runs fast. Python 3.11 everywhere.",
+        ]
+        codebook = build_codebook(texts)
+        original = "Python 3.11 is used on all servers."
+
+        # Dict compress → RLE compress → RLE decompress → Dict decompress
+        dict_compressed = compress_text(original, codebook)
+        rle_compressed = compress_paths(dict_compressed, ["/Users/duke"])
+        rle_back = decompress_paths(rle_compressed, "/Users/duke")
+        dict_back = decompress_text(rle_back, codebook)
+        assert dict_back == original
+
+    def test_rle_then_dict_roundtrip(self):
+        ws = "/Users/duke/workspace"
+        texts = [f"{ws}/memory/test.md appears often. {ws}/memory/test.md again."]
+        codebook = build_codebook(texts)
+        original = f"Check {ws}/memory/test.md for details"
+
+        rle_compressed = compress_paths(original, [ws])
+        dict_compressed = compress_text(rle_compressed, codebook)
+        dict_back = decompress_text(dict_compressed, codebook)
+        rle_back = decompress_paths(dict_back, ws)
+        assert rle_back == original
+
+
+class TestEdgeCaseRoundtrips:
+    def test_only_whitespace(self):
+        codebook = build_codebook(["a b c d e f"])
+        original = "   \n\n\t  "
+        assert decompress_text(compress_text(original, codebook), codebook) == original
+
+    def test_only_newlines(self):
+        codebook = build_codebook(["test test test"])
+        original = "\n\n\n"
+        assert decompress_text(compress_text(original, codebook), codebook) == original
+
+    def test_very_long_text(self):
+        base = "word " * 50
+        codebook = build_codebook([base] * 3)
+        original = base * 10
+        assert decompress_text(compress_text(original, codebook), codebook) == original
+
+    def test_binary_like_content(self):
+        codebook = build_codebook(["test test"])
+        original = "hex: 0xDEADBEEF 0xCAFEBABE"
+        assert decompress_text(compress_text(original, codebook), codebook) == original

package/skills/claw-compactor/tests/test_tiers_comprehensive.py

@@ -0,0 +1,171 @@
+"""Comprehensive tests for generate_summary_tiers.py."""
+import sys
+import pytest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from generate_summary_tiers import (
+    _classify_section, _find_memory_files, generate_tiers, format_human,
+    extract_key_facts, generate_auto_summary, format_tier_template, TIERS,
+)
+
+
+class TestClassifySection:
+    def test_decision(self):
+        assert _classify_section("Important Decisions") >= 8
+
+    def test_action(self):
+        assert _classify_section("Action Items") >= 7
+
+    def test_config(self):
+        assert _classify_section("Configuration Setup") >= 6
+
+    def test_notes(self):
+        assert _classify_section("Random Notes") >= 3
+
+    def test_history(self):
+        assert _classify_section("History Log") >= 1
+
+    def test_unknown(self):
+        result = _classify_section("zzzzzzzzzzz")
+        assert isinstance(result, int)
+
+    def test_empty(self):
+        result = _classify_section("")
+        assert isinstance(result, int)
+
+    def test_case_insensitive(self):
+        a = _classify_section("DECISION")
+        b = _classify_section("decision")
+        assert a == b
+
+
+class TestFindMemoryFiles:
+    def test_finds_files(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("memory")
+        (tmp_path / "TOOLS.md").write_text("tools")
+        mem = tmp_path / "memory"
+        mem.mkdir()
+        (mem / "daily.md").write_text("daily")
+        files = _find_memory_files(str(tmp_path))
+        assert len(files) >= 2
+
+    def test_empty_workspace(self, tmp_path):
+        files = _find_memory_files(str(tmp_path))
+        assert isinstance(files, list)
+
+    def test_nonexistent(self):
+        from lib.exceptions import FileNotFoundError_
+        with pytest.raises(FileNotFoundError_):
+            _find_memory_files("/nonexistent/xyz")
+
+
+class TestGenerateTiers:
+    def test_basic(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("# Memory\n## Decisions\n- Use Python\n## Notes\n- Hello\n")
+        files = [tmp_path / "MEMORY.md"]
+        result = generate_tiers(files)
+        assert isinstance(result, dict)
+
+    def test_returns_tier_info(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("# M\n## A\nfoo\n## B\nbar\n")
+        files = [tmp_path / "MEMORY.md"]
+        result = generate_tiers(files)
+        assert "tiers" in result or "sections" in result
+
+    def test_empty_files(self, tmp_path):
+        (tmp_path / "empty.md").write_text("")
+        result = generate_tiers([tmp_path / "empty.md"])
+        assert isinstance(result, dict)
+
+    def test_multiple_files(self, tmp_path):
+        (tmp_path / "a.md").write_text("# A\nContent A\n")
+        (tmp_path / "b.md").write_text("# B\nContent B\n")
+        result = generate_tiers([tmp_path / "a.md", tmp_path / "b.md"])
+        assert isinstance(result, dict)
+
+
+class TestFormatHuman:
+    def test_basic(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("# Memory\n## Decisions\n- X\n")
+        files = [tmp_path / "MEMORY.md"]
+        result = generate_tiers(files)
+        output = format_human(result)
+        assert isinstance(output, str)
+        assert len(output) > 0
+
+    def test_mentions_levels(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("# M\n## A\nfoo\n")
+        result = generate_tiers([tmp_path / "MEMORY.md"])
+        output = format_human(result)
+        assert "Level" in output or "level" in output
+
+
+class TestExtractKeyFacts:
+    def test_basic(self):
+        text = "Server IP: 192.168.1.100\nPort: 8080\nSome filler text\nDecision: Use Python"
+        facts = extract_key_facts(text)
+        assert isinstance(facts, list)
+
+    def test_empty(self):
+        facts = extract_key_facts("")
+        assert facts == []
+
+    def test_bullets(self):
+        text = "- Important: use SSH keys\n- Note: backup at 3am\n- Random stuff"
+        facts = extract_key_facts(text)
+        assert isinstance(facts, list)
+
+    def test_preserves_key_info(self):
+        text = "Decision: migrate to Python 3.11\nAction: update CI pipeline"
+        facts = extract_key_facts(text)
+        assert len(facts) > 0
+
+
+class TestGenerateAutoSummary:
+    def test_basic(self, tmp_path):
+        (tmp_path / "test.md").write_text("# Config\nServer: 192.168.1.100\nPort: 8080\nUser: admin\n")
+        summary = generate_auto_summary([tmp_path / "test.md"])
+        assert isinstance(summary, str)
+
+    def test_custom_budget(self, tmp_path):
+        (tmp_path / "test.md").write_text("# Test\nContent\n")
+        summary = generate_auto_summary([tmp_path / "test.md"], budget=50)
+        assert isinstance(summary, str)
+
+    def test_empty_file(self, tmp_path):
+        (tmp_path / "empty.md").write_text("")
+        summary = generate_auto_summary([tmp_path / "empty.md"])
+        assert isinstance(summary, str)
+
+
+class TestTiersConfig:
+    def test_tier_0_exists(self):
+        assert 0 in TIERS
+        assert TIERS[0]["budget"] <= 300
+
+    def test_tier_1_exists(self):
+        assert 1 in TIERS
+        assert TIERS[1]["budget"] <= 1500
+
+    def test_tier_2_exists(self):
+        assert 2 in TIERS
+        assert TIERS[2]["budget"] <= 5000
+
+    def test_budgets_ascending(self):
+        assert TIERS[0]["budget"] < TIERS[1]["budget"] < TIERS[2]["budget"]
+
+
+class TestFormatTierTemplate:
+    def test_basic(self, tmp_path):
+        (tmp_path / "test.md").write_text("# Test\n## Section\nContent\n")
+        result = generate_tiers([tmp_path / "test.md"])
+        template = format_tier_template(result, 0)
+        assert isinstance(template, str)
+
+    def test_all_tiers(self, tmp_path):
+        (tmp_path / "test.md").write_text("# Test\n## A\nfoo\n## B\nbar\n")
+        result = generate_tiers([tmp_path / "test.md"])
+        for level in [0, 1, 2]:
+            template = format_tier_template(result, level)
+            assert isinstance(template, str)

package/skills/claw-compactor/tests/test_token_economics.py

@@ -0,0 +1,237 @@
+"""Token Economics tests — precise verification of compression savings.
+
+Inspired by claude-mem's rigorous token accounting approach.
+Every compression technique must prove its token savings are real and measurable.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from lib.tokens import estimate_tokens, using_tiktoken
+from lib.dictionary import (
+    build_codebook, compress_text, decompress_text, compression_stats,
+)
+from lib.tokenizer_optimizer import (
+    optimize_tokens, estimate_savings, normalize_punctuation,
+    compress_table_to_kv, strip_bold_italic, compact_bullets,
+)
+from lib.rle import compress as rle_compress, compress_paths, compress_ip_families
+from compressed_context import compress_with_stats
+
+
+# --- Token Counting Precision ---
+
+class TestTokenCountingPrecision:
+    """Verify estimate_tokens returns correct values for edge cases."""
+
+    def test_empty_returns_zero(self):
+        assert estimate_tokens("") == 0
+
+    def test_none_like_empty(self):
+        """Empty string should be 0, not 1 or error."""
+        assert estimate_tokens("") == 0
+
+    def test_single_char(self):
+        assert estimate_tokens("a") >= 1
+
+    def test_single_chinese_char(self):
+        assert estimate_tokens("你") >= 1
+
+    def test_whitespace_only(self):
+        result = estimate_tokens("   \n\n\t  ")
+        assert result >= 1  # whitespace still has tokens
+
+    def test_large_content_positive(self):
+        """Large content should give proportionally large token count."""
+        small = estimate_tokens("hello world")
+        large = estimate_tokens("hello world " * 1000)
+        assert large > small * 500  # at least 500x more
+
+    def test_known_sentence(self):
+        """A known English sentence should give a reasonable count."""
+        text = "The quick brown fox jumps over the lazy dog."
+        tokens = estimate_tokens(text)
+        # tiktoken: ~10 tokens; heuristic: ~11
+        assert 8 <= tokens <= 15
+
+    def test_chinese_text_density(self):
+        """Chinese text should have ~1 token per 1-2 chars with tiktoken."""
+        text = "这是一个测试句子用来验证中文分词"
+        tokens = estimate_tokens(text)
+        if using_tiktoken():
+            # tiktoken: roughly 1 token per Chinese char
+            assert tokens >= len(text) * 0.3
+        assert tokens >= 1
+
+    def test_consistency(self):
+        """Same text should always give same count."""
+        text = "consistency test 一致性测试"
+        a = estimate_tokens(text)
+        b = estimate_tokens(text)
+        assert a == b
+
+
+# --- Savings Calculation ---
+
+class TestSavingsCalculation:
+    """Verify savings arithmetic is correct."""
+
+    def test_savings_is_before_minus_after(self):
+        original = "**Bold** text with ，Chinese，punctuation。here。"
+        optimized = optimize_tokens(original, aggressive=True)
+        stats = estimate_savings(original, optimized)
+        expected_saved = stats["original_tokens"] - stats["optimized_tokens"]
+        char_diff = stats["original_chars"] - stats["optimized_chars"]
+        assert char_diff >= 0 or stats["original_chars"] >= stats["optimized_chars"]
+        assert stats["token_reduction_pct"] == pytest.approx(
+            (stats["original_tokens"] - stats["optimized_tokens"]) / stats["original_tokens"] * 100,
+            abs=0.15,
+        )
+
+    def test_savings_zero_for_unchanged(self):
+        text = "plain text no changes"
+        optimized = optimize_tokens(text)
+        stats = estimate_savings(text, optimized)
+        assert stats["token_reduction_pct"] == 0.0 or stats["optimized_tokens"] <= stats["original_tokens"]
+
+    def test_negative_savings_possible(self):
+        """If compression adds overhead (e.g., codebook), net can be negative."""
+        # Tiny text with a big codebook
+        codebook = {f"${chr(65+i)}{chr(65+j)}": f"word{i}{j}" for i in range(5) for j in range(5)}
+        text = "unique text"
+        stats = compression_stats(text, text, codebook)
+        # Net reduction can be negative when codebook overhead > savings
+        assert "net_reduction_pct" in stats
+
+    def test_empty_original_savings(self):
+        stats = estimate_savings("", "")
+        assert stats["original_tokens"] == 0
+        assert stats["optimized_tokens"] == 0
+        assert stats["token_reduction_pct"] == 0.0
+
+    def test_rounding(self):
+        """Percentage should be rounded to 1 decimal place."""
+        stats = estimate_savings("a" * 100, "a" * 97)
+        pct = stats["token_reduction_pct"]
+        assert pct == round(pct, 1)
+
+
+# --- Per-Technique Token Economics ---
+
+class TestTableCompressionEconomics:
+    """Tables should save 40-70% tokens when converted to key:value."""
+
+    def test_2col_table_savings(self):
+        table = (
+            "| Key | Value |\n|-----|-------|\n"
+            "| Server | gateway-prod |\n"
+            "| IP | 192.168.1.1 |\n"
+            "| Port | 8080 |\n"
+            "| Status | running |\n"
+            "| Uptime | 42 days |"
+        )
+        before = estimate_tokens(table)
+        after = estimate_tokens(compress_table_to_kv(table))
+        savings_pct = (before - after) / before * 100
+        assert savings_pct > 30, f"Only {savings_pct:.1f}% savings on 2-col table"
+
+    def test_3col_table_savings(self):
+        table = (
+            "| Host | IP | Role |\n|------|-----|------|\n"
+            "| gw | 192.168.1.1 | gateway |\n"
+            "| n1 | 192.168.1.2 | worker |\n"
+            "| n2 | 192.168.1.3 | worker |"
+        )
+        before = estimate_tokens(table)
+        after = estimate_tokens(compress_table_to_kv(table))
+        assert after < before
+
+    def test_empty_table_no_crash(self):
+        table = "| A | B |\n|---|---|\n"
+        compress_table_to_kv(table)  # Should not crash
+
+
+class TestPunctuationEconomics:
+    """Each fullwidth → halfwidth saves ~1 token."""
+
+    def test_measurable_savings(self):
+        # Fullwidth punctuation in English-heavy context saves more
+        text = "IP：192.168.1.1，Port：8080，Status：online，Mode：active，Region：US"
+        before = estimate_tokens(text)
+        after = estimate_tokens(normalize_punctuation(text))
+        # Savings depend on tokenizer; at minimum should not increase
+        assert after <= before, f"Punctuation normalization increased tokens: {before} → {after}"
+
+
+class TestDictionaryEconomics:
+    """Dictionary compression should show net savings on realistic data."""
+
+    def test_net_savings_positive(self):
+        """On text with enough repetition, net savings should be positive."""
+        texts = [
+            "The gateway at 10.0.1.1 serves example_user workspace.\n" * 3,
+            "Connect to 10.0.1.1 as example_user for deployment.\n" * 3,
+            "Node remote-node at 10.0.1.2 reports to 10.0.1.1 gateway.\n" * 3,
+        ]
+        cb = build_codebook(texts, min_freq=2)
+        if cb:  # Only test if codebook was built
+            combined = '\n'.join(texts)
+            compressed = compress_text(combined, cb)
+            stats = compression_stats(combined, compressed, cb)
+            assert stats["gross_reduction_pct"] > 0
+
+    def test_tiny_text_overhead(self):
+        """On tiny text, codebook overhead may exceed savings."""
+        texts = ["short text here", "other short text"]
+        cb = build_codebook(texts, min_freq=1)
+        # Either empty codebook or overhead is acknowledged
+        if cb:
+            stats = compression_stats("short text", compress_text("short text", cb), cb)
+            # Net could be negative — that's expected and correct
+            assert "net_reduction_pct" in stats
+
+
+class TestCompressedContextEconomics:
+    """CCP should reduce tokens on prose-heavy content."""
+
+    def test_ultra_reduces_prose(self):
+        prose = (
+            "Alex is the CTO of ExampleCorp and the founder of DataPlatform. "
+            "He has over 13 years of experience in cryptocurrency quantitative trading. "
+            "He is currently based in Los Gatos, California, with additional offices "
+            "in Shanghai and Taipei. The organization focuses on building infrastructure "
+            "for automated trading applications and monitoring systems."
+        )
+        stats = compress_with_stats(prose, "ultra")
+        assert stats["compressed_tokens"] < stats["original_tokens"]
+        text_savings_pct = (stats["original_tokens"] - stats["compressed_tokens"]) / stats["original_tokens"] * 100
+        assert text_savings_pct > 5, f"Only {text_savings_pct:.1f}% text reduction on prose"
+
+    def test_light_minimal_overhead(self):
+        text = "Simple clean text with no issues at all."
+        stats = compress_with_stats(text, "light")
+        # Light mode should barely change clean text
+        assert stats["compressed_tokens"] <= stats["original_tokens"] + 1
+
+
+class TestRLEEconomics:
+    """RLE should save tokens on path-heavy and IP-heavy text."""
+
+    def test_path_compression_saves_tokens(self):
+        ws = "/home/user/workspace"
+        text = f"File at {ws}/memory/a.md\nAlso {ws}/TOOLS.md\nAnd {ws}/MEMORY.md\n"
+        before = estimate_tokens(text)
+        after = estimate_tokens(compress_paths(text, [ws]))
+        assert after < before
+        savings = before - after
+        assert savings >= 3, f"Only saved {savings} tokens on 3 path occurrences"
+
+    def test_ip_compression_saves_tokens(self):
+        text = "Servers: 10.0.1.1, 10.0.1.2, 10.0.1.3, 10.0.1.4"
+        before = estimate_tokens(text)
+        compressed, _ = compress_ip_families(text)
+        after = estimate_tokens(compressed)
+        assert after <= before