@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/tests/test_compressed_context.py

@@ -0,0 +1,145 @@
+"""Tests for Compressed Context Protocol."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from compressed_context import (
+    compress_ultra, compress_medium, compress_light,
+    compress, compress_with_stats,
+    DECOMPRESS_INSTRUCTIONS,
+)
+
+
+SAMPLE_BIO = (
+    "Alex is the CTO of ExampleCorp and founder of DataPlatform. "
+    "He has 13+ years of crypto quantitative trading experience. "
+    "He is based in Los Gatos, CA with offices in Shanghai and Taipei."
+)
+
+SAMPLE_TECH = (
+    "The infrastructure uses Kubernetes for container orchestration "
+    "with continuous integration and deployment pipelines. "
+    "The application architecture includes distributed database "
+    "systems and monitoring dashboards for production environments."
+)
+
+
+class TestCompressUltra:
+    def test_reduces_length(self):
+        result = compress_ultra(SAMPLE_BIO)
+        assert len(result) < len(SAMPLE_BIO)
+
+    def test_preserves_names(self):
+        result = compress_ultra(SAMPLE_BIO)
+        assert "Alex" in result
+        assert "Alex" in result
+        assert "ExampleCorp" in result
+
+    def test_abbreviates(self):
+        result = compress_ultra(SAMPLE_TECH)
+        # Should use abbreviations
+        assert "k8s" in result or "infra" in result or len(result) < len(SAMPLE_TECH) * 0.8
+
+    def test_empty(self):
+        assert compress_ultra("") == ""
+
+    def test_removes_fillers(self):
+        text = "He has extensive experience. In addition, he is skilled."
+        result = compress_ultra(text)
+        assert "In addition" not in result
+
+
+class TestCompressMedium:
+    def test_less_aggressive(self):
+        ultra = compress_ultra(SAMPLE_BIO)
+        medium = compress_medium(SAMPLE_BIO)
+        # Medium should be longer than ultra (less aggressive)
+        assert len(medium) >= len(ultra)
+
+    def test_reduces_length(self):
+        result = compress_medium(SAMPLE_TECH)
+        assert len(result) < len(SAMPLE_TECH)
+
+
+class TestCompressLight:
+    def test_minimal_changes(self):
+        text = "Simple text with no issues"
+        assert compress_light(text) == text
+
+    def test_cleanup(self):
+        text = "Extra  spaces  here\n\n\n\nToo many lines"
+        result = compress_light(text)
+        assert "  " not in result.replace("  ", " ")  # might still have some
+        assert "\n\n\n" not in result
+
+
+class TestCompress:
+    def test_returns_dict(self):
+        result = compress(SAMPLE_BIO, "ultra")
+        assert "compressed" in result
+        assert "instructions" in result
+        assert result["level"] == "ultra"
+
+    def test_all_levels(self):
+        for level in ["ultra", "medium", "light"]:
+            result = compress(SAMPLE_BIO, level)
+            assert result["level"] == level
+            assert len(result["compressed"]) > 0
+            assert len(result["instructions"]) > 0
+
+    def test_invalid_level(self):
+        with pytest.raises(ValueError):
+            compress(SAMPLE_BIO, "extreme")
+
+    def test_instructions_exist(self):
+        for level in ["ultra", "medium", "light"]:
+            assert level in DECOMPRESS_INSTRUCTIONS
+
+
+class TestCompressWithStats:
+    def test_basic(self):
+        stats = compress_with_stats(SAMPLE_BIO, "ultra")
+        assert stats["original_tokens"] > 0
+        assert stats["compressed_tokens"] > 0
+        assert stats["compressed_tokens"] < stats["original_tokens"]
+        # Note: reduction_pct includes instruction overhead so may be negative for short texts
+        assert "instructions" in stats
+
+    def test_all_levels(self):
+        for level in ["ultra", "medium", "light"]:
+            stats = compress_with_stats(SAMPLE_BIO, level)
+            assert stats["original_tokens"] > 0
+            assert isinstance(stats["reduction_pct"], float)
+
+    def test_empty(self):
+        stats = compress_with_stats("", "ultra")
+        assert stats["original_tokens"] == 0
+
+    def test_unicode(self):
+        text = "中文内容测试：这是一个很长的中文段落，包含了很多信息。"
+        stats = compress_with_stats(text, "ultra")
+        assert stats["compressed_tokens"] > 0
+        assert "中文" in stats["compressed"]
+
+    def test_real_bio_compression(self):
+        """The sample bio should compress the text itself significantly."""
+        stats = compress_with_stats(SAMPLE_BIO, "ultra")
+        # Compressed text alone should be smaller than original
+        assert stats["compressed_tokens"] < stats["original_tokens"]
+        # For longer texts, net reduction (including instructions) should be positive
+        # For short texts like this, just verify text compression works
+        text_reduction = (stats["original_tokens"] - stats["compressed_tokens"]) / stats["original_tokens"] * 100
+        assert text_reduction > 5, f"Only {text_reduction:.1f}% text reduction"
+
+
+class TestCLI:
+    def test_file_compress(self, tmp_path):
+        """Test that compress works on a file's content."""
+        f = tmp_path / "test.md"
+        f.write_text(SAMPLE_BIO)
+        text = f.read_text()
+        stats = compress_with_stats(text, "ultra")
+        assert stats["compressed_tokens"] < stats["original_tokens"]

package/skills/claw-compactor/tests/test_config.py

@@ -0,0 +1,54 @@
+"""Tests for lib.config module."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from lib.config import MemCompressConfig, load_config, DEFAULT_CONFIG
+
+
+class TestMemCompressConfig:
+    def test_defaults(self) -> None:
+        cfg = MemCompressConfig()
+        assert cfg.chars_per_token == 4
+        assert cfg.level0_max_tokens == 200
+
+    def test_custom_values(self) -> None:
+        cfg = MemCompressConfig(chars_per_token=3)
+        assert cfg.chars_per_token == 3
+
+
+class TestLoadConfig:
+    def test_no_config_file(self, tmp_path: Path) -> None:
+        cfg = load_config(tmp_path)
+        assert cfg.chars_per_token == DEFAULT_CONFIG["chars_per_token"]
+
+    def test_valid_config(self, tmp_path: Path) -> None:
+        (tmp_path / "claw-compactor-config.json").write_text(
+            json.dumps({"chars_per_token": 3}), encoding="utf-8"
+        )
+        cfg = load_config(tmp_path)
+        assert cfg.chars_per_token == 3
+
+    def test_invalid_json(self, tmp_path: Path) -> None:
+        (tmp_path / "claw-compactor-config.json").write_text("{{bad", encoding="utf-8")
+        cfg = load_config(tmp_path)
+        assert cfg.chars_per_token == 4
+
+    def test_non_dict(self, tmp_path: Path) -> None:
+        (tmp_path / "claw-compactor-config.json").write_text('["arr"]', encoding="utf-8")
+        cfg = load_config(tmp_path)
+        assert cfg.chars_per_token == 4
+
+    def test_unknown_keys_ignored(self, tmp_path: Path) -> None:
+        (tmp_path / "claw-compactor-config.json").write_text(
+            json.dumps({"chars_per_token": 5, "bogus": 99}), encoding="utf-8"
+        )
+        cfg = load_config(tmp_path)
+        assert cfg.chars_per_token == 5
+
+    def test_empty_file(self, tmp_path: Path) -> None:
+        (tmp_path / "claw-compactor-config.json").write_text("", encoding="utf-8")
+        cfg = load_config(tmp_path)
+        assert cfg.chars_per_token == 4

package/skills/claw-compactor/tests/test_dedup_memory.py

@@ -0,0 +1,124 @@
+"""Tests for dedup_memory.py."""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from dedup_memory import run_dedup, format_human, _collect_entries
+from lib.dedup import find_duplicates, merge_duplicates, jaccard, _shingles
+
+
+class TestShingles:
+    def test_basic(self):
+        assert len(_shingles("the quick brown fox jumps over")) > 0
+
+    def test_short(self):
+        assert len(_shingles("hi")) == 1
+
+    def test_empty(self):
+        assert len(_shingles("")) == 1
+
+
+class TestJaccard:
+    def test_identical(self):
+        assert jaccard({1, 2, 3}, {1, 2, 3}) == 1.0
+
+    def test_disjoint(self):
+        assert jaccard({1, 2}, {3, 4}) == 0.0
+
+    def test_partial(self):
+        assert 0 < jaccard({1, 2, 3}, {2, 3, 4}) < 1
+
+    def test_empty(self):
+        assert jaccard(set(), set()) == 1.0
+
+    def test_one_empty(self):
+        assert jaccard({1}, set()) == 0.0
+
+
+class TestFindDuplicates:
+    def test_exact(self):
+        entries = ["Install Python 3.10 and configure environment", "Install Python 3.10 and configure environment", "Something different about database"]
+        groups = find_duplicates(entries, threshold=0.5)
+        assert len(groups) >= 1
+
+    def test_near(self):
+        entries = ["Install Python 3.10 and set up virtual env for dev", "Install Python 3.10 and set up virtual env for prod", "Configure nginx reverse proxy"]
+        assert len(find_duplicates(entries, threshold=0.5)) >= 1
+
+    def test_no_duplicates(self):
+        assert len(find_duplicates(["Alpha beta gamma", "One two three four", "日本語テスト"], threshold=0.8)) == 0
+
+    def test_single(self):
+        assert find_duplicates(["one"]) == []
+
+    def test_empty(self):
+        assert find_duplicates([]) == []
+
+    def test_large(self):
+        entries = ["Entry {} with filler content for testing".format(i) for i in range(500)]
+        entries += [entries[0], entries[1]]
+        assert len(find_duplicates(entries, threshold=0.8)) >= 2
+
+    def test_unicode(self):
+        entries = ["配置数据库连接字符串和环境变量", "配置数据库连接字符串和环境变量", "不同的内容关于部署"]
+        assert len(find_duplicates(entries, threshold=0.5)) >= 1
+
+
+class TestMergeDuplicates:
+    def test_no_groups(self):
+        assert merge_duplicates(["a", "b"], []) == ["a", "b"]
+
+
+class TestCollectEntries:
+    def test_workspace(self, tmp_workspace):
+        entries = _collect_entries(str(tmp_workspace))
+        assert len(entries) > 0
+
+    def test_nonexistent(self):
+        with pytest.raises(Exception):
+            _collect_entries("/nonexistent/xyz")
+
+    def test_empty(self, empty_file):
+        assert len(_collect_entries(str(empty_file))) == 0
+
+    def test_unicode(self, unicode_file):
+        assert len(_collect_entries(str(unicode_file))) > 0
+
+
+class TestRunDedup:
+    def test_basic(self, tmp_workspace):
+        result = run_dedup(str(tmp_workspace))
+        assert "total_entries" in result
+        assert "duplicate_groups" in result
+
+    def test_with_dupes(self, duplicate_content):
+        result = run_dedup(str(duplicate_content), threshold=0.5)
+        assert result["total_entries"] > 0
+
+    def test_auto_merge(self, duplicate_content):
+        result = run_dedup(str(duplicate_content), threshold=0.5, auto_merge=True)
+        assert "tokens_before" in result
+
+    def test_empty(self, tmp_path):
+        (tmp_path / "e.md").write_text("", encoding="utf-8")
+        assert run_dedup(str(tmp_path))["total_entries"] == 0
+
+    def test_nonexistent(self):
+        with pytest.raises(Exception):
+            run_dedup("/nonexistent/xyz")
+
+    def test_json(self, tmp_workspace):
+        assert json.loads(json.dumps(run_dedup(str(tmp_workspace))))
+
+
+class TestFormatHuman:
+    def test_no_dupes(self):
+        assert "No duplicates" in format_human({"total_entries": 5, "duplicate_groups": [], "entries_removed": 0})
+
+    def test_with_dupes(self, duplicate_content):
+        output = format_human(run_dedup(str(duplicate_content), threshold=0.3))
+        assert "Deduplication Report" in output

package/skills/claw-compactor/tests/test_dictionary.py

@@ -0,0 +1,327 @@
+"""Tests for dictionary-based compression."""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from lib.dictionary import (
+    build_codebook, compress_text, decompress_text,
+    save_codebook, load_codebook, compression_stats,
+    _generate_codes, _tokenize_ngrams, _extract_ip_prefixes,
+    _extract_path_prefixes,
+)
+
+
+# --- Fixtures ---
+
+@pytest.fixture
+def sample_texts():
+    """Realistic memory-style texts with repetition."""
+    return [
+        "# Server Config\n\n- Gateway IP: 10.0.1.1\n- Node: remote-node at 10.0.1.2\n"
+        "- User: example_user\n- Path: /home/user/workspace\n"
+        "- NetworkID: abc123network456\n- Token: my-secret-token-2024\n",
+
+        "# Daily Notes\n\n- Connected to 10.0.1.1 gateway\n"
+        "- Deployed on remote-node (10.0.1.2)\n"
+        "- User example_user ran backup\n"
+        "- Workspace: /home/user/workspace\n"
+        "- Using token my-secret-token-2024\n",
+
+        "# More Notes\n\n- Gateway 10.0.1.1 is stable\n"
+        "- remote-node running fine at 10.0.1.2\n"
+        "- example_user home: /home/user/workspace\n"
+        "- Auth: my-secret-token-2024\n",
+    ]
+
+
+@pytest.fixture
+def codebook():
+    return {
+        "$GW": "10.0.1.1",
+        "$JP": "10.0.1.2",
+        "$U": "example_user",
+        "$WS": "/home/user/workspace",
+        "$TK": "my-secret-token-2024",
+    }
+
+
+# --- Code generation ---
+
+class TestGenerateCodes:
+    def test_count(self):
+        codes = _generate_codes(10)
+        assert len(codes) == 10
+
+    def test_uniqueness(self):
+        codes = _generate_codes(676)
+        assert len(set(codes)) == 676
+
+    def test_format(self):
+        codes = _generate_codes(5)
+        for c in codes:
+            assert c.startswith('$')
+            assert len(c) == 3
+            assert c[1:].isalpha() and c[1:].isupper()
+
+    def test_overflow_to_3char(self):
+        codes = _generate_codes(677)
+        assert len(codes) == 677
+        assert len(codes[-1]) == 4  # $AAA
+
+
+# --- N-gram extraction ---
+
+class TestTokenizeNgrams:
+    def test_basic(self):
+        ngrams = _tokenize_ngrams("the quick brown fox jumps over the lazy dog")
+        assert any("quick brown" in k for k in ngrams)
+
+    def test_min_length_filter(self):
+        ngrams = _tokenize_ngrams("a b c d e f")
+        # All ngrams shorter than MIN_PHRASE_LEN should be filtered
+        for gram in ngrams:
+            assert len(gram) >= 6
+
+    def test_empty(self):
+        assert len(_tokenize_ngrams("")) == 0
+
+
+# --- IP prefix extraction ---
+
+class TestExtractIpPrefixes:
+    def test_finds_common_prefix(self):
+        texts = [
+            "Server at 192.168.1.10 and 192.168.1.20",
+            "Also 192.168.1.30 and 10.0.0.1",
+            "More at 192.168.1.40",
+        ]
+        prefixes = _extract_ip_prefixes(texts)
+        assert any("192.168.1." in p for p in prefixes)
+
+    def test_no_ips(self):
+        assert _extract_ip_prefixes(["no ips here"]) == {}
+
+
+# --- Path prefix extraction ---
+
+class TestExtractPathPrefixes:
+    def test_finds_common_path(self):
+        texts = [
+            "File at /Users/duke/.openclaw/workspace/MEMORY.md",
+            "Also /Users/duke/.openclaw/workspace/TOOLS.md",
+            "And /Users/duke/.openclaw/workspace/memory/2026-01.md",
+        ]
+        prefixes = _extract_path_prefixes(texts)
+        assert any("/Users/duke/.openclaw" in p for p in prefixes)
+
+    def test_no_paths(self):
+        assert _extract_path_prefixes(["no paths"]) == {}
+
+
+# --- Codebook building ---
+
+class TestBuildCodebook:
+    def test_builds_from_texts(self, sample_texts):
+        cb = build_codebook(sample_texts, min_freq=2)
+        assert len(cb) > 0
+        # All values should be real phrases
+        for code, phrase in cb.items():
+            assert code.startswith('$')
+            assert len(phrase) >= 6
+
+    def test_empty_texts(self):
+        cb = build_codebook([], min_freq=2)
+        assert cb == {}
+
+    def test_no_frequent_phrases(self):
+        cb = build_codebook(["unique text one", "different text two"], min_freq=5)
+        assert cb == {}
+
+    def test_max_entries(self, sample_texts):
+        cb = build_codebook(sample_texts, max_entries=3, min_freq=2)
+        assert len(cb) <= 3
+
+    def test_no_overlapping_codes(self, sample_texts):
+        cb = build_codebook(sample_texts, min_freq=2)
+        codes = list(cb.keys())
+        for i, c1 in enumerate(codes):
+            for c2 in codes[i+1:]:
+                assert c1 != c2
+
+
+# --- Compression / Decompression ---
+
+class TestCompressDecompress:
+    def test_basic_compress(self, codebook):
+        text = "Gateway at 10.0.1.1 running"
+        r = compress_text(text, codebook)
+        assert "$GW" in r
+        assert "10.0.1.1" not in r
+
+    def test_basic_decompress(self, codebook):
+        compressed = "Gateway at $GW running"
+        r = decompress_text(compressed, codebook)
+        assert "10.0.1.1" in r
+        assert "$GW" not in r
+
+    def test_roundtrip(self, codebook):
+        """Compression then decompression must be lossless."""
+        text = (
+            "Connect to 10.0.1.1 as example_user\n"
+            "Node 10.0.1.2 is remote-node\n"
+            "Workspace: /home/user/workspace\n"
+            "Token: my-secret-token-2024\n"
+        )
+        compressed = compress_text(text, codebook)
+        decompressed = decompress_text(compressed, codebook)
+        assert decompressed == text
+
+    def test_roundtrip_no_matches(self, codebook):
+        text = "Nothing to compress here."
+        assert decompress_text(compress_text(text, codebook), codebook) == text
+
+    def test_roundtrip_all_matches(self, codebook):
+        text = "10.0.1.1 10.0.1.2 example_user"
+        assert decompress_text(compress_text(text, codebook), codebook) == text
+
+    def test_empty_codebook(self):
+        assert compress_text("hello", {}) == "hello"
+        assert decompress_text("hello", {}) == "hello"
+
+    def test_empty_text(self, codebook):
+        assert compress_text("", codebook) == ""
+        assert decompress_text("", codebook) == ""
+
+    def test_multiple_occurrences(self, codebook):
+        text = "IP 10.0.1.1 and again 10.0.1.1"
+        compressed = compress_text(text, codebook)
+        assert compressed.count("$GW") == 2
+        assert decompress_text(compressed, codebook) == text
+
+    def test_adjacent_codes(self, codebook):
+        text = "10.0.1.1example_user"
+        compressed = compress_text(text, codebook)
+        decompressed = decompress_text(compressed, codebook)
+        assert decompressed == text
+
+
+class TestRoundtripWithBuiltCodebook:
+    """Test that build → compress → decompress is lossless."""
+
+    def test_full_pipeline(self, sample_texts):
+        cb = build_codebook(sample_texts, min_freq=2)
+        for text in sample_texts:
+            compressed = compress_text(text, cb)
+            decompressed = decompress_text(compressed, cb)
+            assert decompressed == text, f"Roundtrip failed for text starting with: {text[:50]}"
+
+    def test_large_corpus(self):
+        """Test with a larger synthetic corpus."""
+        base = "Server 192.168.1.{i} is running service-{i} on /opt/app/service-{i}/config"
+        texts = [base.format(i=i) for i in range(50)]
+        cb = build_codebook(texts, min_freq=3)
+        combined = '\n'.join(texts)
+        compressed = compress_text(combined, cb)
+        decompressed = decompress_text(compressed, cb)
+        assert decompressed == combined
+
+
+# --- Save / Load ---
+
+class TestSaveLoadCodebook:
+    def test_save_load_roundtrip(self, tmp_path, codebook):
+        path = tmp_path / "cb.json"
+        save_codebook(codebook, path)
+        loaded = load_codebook(path)
+        assert loaded == codebook
+
+    def test_load_nonexistent(self, tmp_path):
+        with pytest.raises(FileNotFoundError):
+            load_codebook(tmp_path / "nope.json")
+
+    def test_load_invalid(self, tmp_path):
+        path = tmp_path / "bad.json"
+        path.write_text('{"not_entries": true}')
+        with pytest.raises(ValueError):
+            load_codebook(path)
+
+    def test_save_creates_dirs(self, tmp_path, codebook):
+        path = tmp_path / "deep" / "nested" / "cb.json"
+        save_codebook(codebook, path)
+        assert path.exists()
+
+
+# --- Stats ---
+
+class TestCompressionStats:
+    def test_basic(self, codebook):
+        original = "10.0.1.1 is the gateway for example_user"
+        compressed = compress_text(original, codebook)
+        stats = compression_stats(original, compressed, codebook)
+        assert stats["gross_reduction_pct"] > 0
+        assert stats["codebook_entries"] == len(codebook)
+        assert stats["codes_used"] > 0
+
+    def test_empty(self):
+        stats = compression_stats("", "", {})
+        assert stats["gross_reduction_pct"] == 0.0
+
+
+# --- Integration with dictionary_compress.py CLI ---
+
+class TestCLICommands:
+    def test_build_and_stats(self, tmp_path):
+        from dictionary_compress import cmd_build, cmd_stats
+        # Create workspace
+        (tmp_path / "MEMORY.md").write_text(
+            "# Mem\n\n- Server: 192.168.1.100\n- Server: 192.168.1.100\n"
+            "- Path: /long/repeated/path/here\n- Path: /long/repeated/path/here\n"
+            "- Token: my-long-token-value-2024\n- Token: my-long-token-value-2024\n"
+            "- Again server 192.168.1.100 and /long/repeated/path/here\n"
+        )
+        mem = tmp_path / "memory"
+        mem.mkdir()
+        (mem / "day1.md").write_text(
+            "# Day\n- Used 192.168.1.100\n- At /long/repeated/path/here\n"
+            "- Auth: my-long-token-value-2024\n"
+        )
+
+        cb_path = tmp_path / "memory" / ".codebook.json"
+        result = cmd_build(tmp_path, cb_path, min_freq=2)
+        assert result["codebook_entries"] > 0
+        assert cb_path.exists()
+
+        stats = cmd_stats(tmp_path, cb_path)
+        assert stats["codebook_entries"] > 0
+
+    def test_compress_decompress_roundtrip(self, tmp_path):
+        from dictionary_compress import cmd_build, cmd_compress, cmd_decompress
+
+        original_text = (
+            "# Notes\n- IP: 10.20.30.40 is primary\n- IP: 10.20.30.40 backup too\n"
+            "- Workspace /home/user/.openclaw/workspace is main\n"
+            "- Also /home/user/.openclaw/workspace for backup\n"
+            "- Token super-secret-token-2024 used\n"
+            "- Again super-secret-token-2024 here\n"
+            "- And 10.20.30.40 again\n"
+        )
+        (tmp_path / "MEMORY.md").write_text(original_text)
+        mem = tmp_path / "memory"
+        mem.mkdir()
+
+        cb_path = mem / ".codebook.json"
+        cmd_build(tmp_path, cb_path, min_freq=2)
+
+        # Compress
+        cmd_compress(tmp_path, cb_path, dry_run=False)
+        compressed = (tmp_path / "MEMORY.md").read_text()
+        assert len(compressed) <= len(original_text)
+
+        # Decompress
+        cmd_decompress(tmp_path, cb_path, dry_run=False)
+        restored = (tmp_path / "MEMORY.md").read_text()
+        assert restored == original_text