@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/tests/test_error_handling.py

@@ -0,0 +1,194 @@
+"""Error handling tests — file not found, corrupt data, edge cases."""
+import json
+import sys
+import pytest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+
+from lib.exceptions import FileNotFoundError_, MemCompressError, ParseError, TokenEstimationError
+from lib.dictionary import load_codebook, save_codebook, compress_text, decompress_text, build_codebook
+from lib.tokens import estimate_tokens
+
+
+class TestExceptions:
+    def test_file_not_found_is_mem_compress_error(self):
+        assert issubclass(FileNotFoundError_, MemCompressError)
+
+    def test_parse_error_is_mem_compress_error(self):
+        assert issubclass(ParseError, MemCompressError)
+
+    def test_token_estimation_error(self):
+        assert issubclass(TokenEstimationError, MemCompressError)
+
+    def test_raise_file_not_found(self):
+        with pytest.raises(FileNotFoundError_):
+            raise FileNotFoundError_("test.md")
+
+    def test_raise_parse_error(self):
+        with pytest.raises(ParseError):
+            raise ParseError("bad markdown")
+
+    def test_exception_message(self):
+        e = FileNotFoundError_("missing.md")
+        assert "missing.md" in str(e)
+
+
+class TestDictionaryErrors:
+    def test_load_nonexistent_codebook(self, tmp_path):
+        with pytest.raises(Exception):
+            load_codebook(tmp_path / "nonexistent.json")
+
+    def test_load_corrupt_codebook(self, tmp_path):
+        bad = tmp_path / "bad.json"
+        bad.write_text("this is not json{{{")
+        with pytest.raises(Exception):
+            load_codebook(bad)
+
+    def test_load_empty_file(self, tmp_path):
+        empty = tmp_path / "empty.json"
+        empty.write_text("")
+        with pytest.raises(Exception):
+            load_codebook(empty)
+
+    def test_compress_empty_codebook(self):
+        result = compress_text("hello world", {})
+        assert result == "hello world"
+
+    def test_decompress_empty_codebook(self):
+        result = decompress_text("hello world", {})
+        assert result == "hello world"
+
+    def test_compress_empty_text(self):
+        result = compress_text("", {"hello": "$AA"})
+        assert result == ""
+
+    def test_build_codebook_empty_input(self):
+        result = build_codebook([])
+        assert isinstance(result, dict)
+
+    def test_build_codebook_whitespace_only(self):
+        result = build_codebook(["   \n\n  "])
+        assert isinstance(result, dict)
+
+    def test_save_load_roundtrip(self, tmp_path):
+        codebook = {"hello": "$AA", "world": "$AB"}
+        path = tmp_path / "cb.json"
+        save_codebook(codebook, path)
+        loaded = load_codebook(path)
+        assert loaded == codebook
+
+
+class TestEstimateTokenErrors:
+    def test_none_input(self):
+        """Estimate tokens with None should raise or handle gracefully."""
+        with pytest.raises((TypeError, AttributeError)):
+            estimate_tokens(None)
+
+    def test_numeric_input(self):
+        with pytest.raises((TypeError, AttributeError)):
+            estimate_tokens(42)
+
+    def test_bytes_input(self):
+        with pytest.raises((TypeError, AttributeError)):
+            estimate_tokens(b"hello")
+
+
+class TestCompressMemoryErrors:
+    def test_nonexistent_path(self):
+        from compress_memory import _collect_files
+        with pytest.raises(FileNotFoundError_):
+            _collect_files("/nonexistent/path/xyz123")
+
+    def test_empty_file(self, tmp_path):
+        from compress_memory import compress_file
+        f = tmp_path / "empty.md"
+        f.write_text("")
+        result = compress_file(f, dry_run=True, no_llm=True)
+        assert isinstance(result, dict)
+
+    def test_binary_content(self, tmp_path):
+        """Binary-ish content shouldn't crash."""
+        from compress_memory import rule_compress
+        text = "Normal text\x00\x01\x02 more text"
+        result = rule_compress(text)
+        assert isinstance(result, str)
+
+
+class TestDedupErrors:
+    def test_nonexistent_path(self):
+        from dedup_memory import _collect_entries
+        with pytest.raises(FileNotFoundError_):
+            _collect_entries("/nonexistent/xyz")
+
+    def test_empty_directory(self, tmp_path):
+        from dedup_memory import _collect_entries
+        result = _collect_entries(str(tmp_path))
+        assert result == []
+
+
+class TestEstimateTokensScript:
+    def test_nonexistent_path(self):
+        from estimate_tokens import scan_path
+        with pytest.raises(FileNotFoundError_):
+            scan_path("/nonexistent/xyz")
+
+
+class TestAuditErrors:
+    def test_nonexistent_workspace(self):
+        from audit_memory import audit_workspace
+        with pytest.raises(FileNotFoundError_):
+            audit_workspace("/nonexistent/xyz")
+
+
+class TestTiersErrors:
+    def test_nonexistent_path(self):
+        from generate_summary_tiers import _find_memory_files
+        with pytest.raises(FileNotFoundError_):
+            _find_memory_files("/nonexistent/xyz")
+
+
+class TestObservationErrors:
+    def test_nonexistent_file(self):
+        from observation_compressor import parse_session_jsonl
+        with pytest.raises(Exception):
+            parse_session_jsonl(Path("/nonexistent/session.jsonl"))
+
+    def test_corrupt_jsonl(self, tmp_path):
+        from observation_compressor import parse_session_jsonl
+        f = tmp_path / "bad.jsonl"
+        f.write_text("not json\n{bad json}\n")
+        # Should handle gracefully (skip bad lines or raise)
+        try:
+            result = parse_session_jsonl(f)
+            assert isinstance(result, list)
+        except Exception:
+            pass  # Either handling is acceptable
+
+    def test_empty_jsonl(self, tmp_path):
+        from observation_compressor import parse_session_jsonl
+        f = tmp_path / "empty.jsonl"
+        f.write_text("")
+        result = parse_session_jsonl(f)
+        assert result == []
+
+    def test_compress_empty_session(self, tmp_path):
+        from observation_compressor import compress_session
+        f = tmp_path / "empty.jsonl"
+        f.write_text("")
+        result = compress_session(f)
+        assert isinstance(result, dict)
+
+
+class TestMainEntryErrors:
+    def test_nonexistent_workspace(self):
+        from mem_compress import _workspace_path
+        with pytest.raises(SystemExit):
+            _workspace_path("/nonexistent/xyz")
+
+    def test_file_as_workspace(self, tmp_path):
+        f = tmp_path / "file.txt"
+        f.write_text("hello")
+        from mem_compress import _workspace_path
+        with pytest.raises(SystemExit):
+            _workspace_path(str(f))

package/skills/claw-compactor/tests/test_estimate_tokens.py

@@ -0,0 +1,99 @@
+"""Tests for estimate_tokens.py."""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from estimate_tokens import scan_path, format_human, _score_potential
+from lib.tokens import estimate_tokens, using_tiktoken
+
+
+class TestEstimateTokens:
+    def test_empty_string(self):
+        assert estimate_tokens("") == 0
+
+    def test_simple_english(self):
+        tokens = estimate_tokens("Hello world, this is a test.")
+        assert 0 < tokens < 50
+
+    def test_unicode_chinese(self):
+        assert estimate_tokens("你好世界") > 0
+
+    def test_large_input(self):
+        assert estimate_tokens("word " * 25000) > 1000
+
+
+class TestScorePotential:
+    def test_high_tokens(self):
+        assert _score_potential(3000, 2500) == "high"
+
+    def test_high_reduction(self):
+        assert _score_potential(1000, 800) == "high"
+
+    def test_medium(self):
+        assert _score_potential(600, 560) == "medium"
+
+    def test_low(self):
+        assert _score_potential(100, 98) == "low"
+
+    def test_zero(self):
+        assert _score_potential(0, 0) == "low"
+
+
+class TestScanPath:
+    def test_workspace(self, tmp_workspace):
+        results = scan_path(str(tmp_workspace))
+        assert len(results) > 0
+        assert all("tokens" in r for r in results)
+
+    def test_single_file(self, tmp_workspace):
+        assert len(scan_path(str(tmp_workspace / "MEMORY.md"))) == 1
+
+    def test_empty_file(self, empty_file):
+        results = scan_path(str(empty_file))
+        assert len(results) == 1
+        assert results[0]["tokens"] == 0
+
+    def test_nonexistent(self):
+        with pytest.raises(Exception):
+            scan_path("/nonexistent/xyz")
+
+    def test_large_file(self, large_file):
+        results = scan_path(str(large_file))
+        assert results[0]["tokens"] > 1000
+
+    def test_unicode(self, unicode_file):
+        assert scan_path(str(unicode_file))[0]["tokens"] > 0
+
+    def test_broken_markdown(self, broken_markdown):
+        assert len(scan_path(str(broken_markdown))) == 1
+
+    def test_threshold(self, tmp_workspace):
+        assert len(scan_path(str(tmp_workspace), threshold=999999)) == 0
+
+    def test_headers_only(self, headers_only):
+        assert len(scan_path(str(headers_only))) == 1
+
+    def test_single_line(self, single_line):
+        assert scan_path(str(single_line))[0]["tokens"] > 0
+
+    def test_sorted_desc(self, tmp_workspace):
+        tokens = [r["tokens"] for r in scan_path(str(tmp_workspace))]
+        assert tokens == sorted(tokens, reverse=True)
+
+
+class TestFormatHuman:
+    def test_empty(self):
+        assert "No files" in format_human([])
+
+    def test_normal(self, tmp_workspace):
+        output = format_human(scan_path(str(tmp_workspace)))
+        assert "Token Estimation Report" in output
+
+    def test_json_roundtrip(self, tmp_workspace):
+        results = scan_path(str(tmp_workspace))
+        j = json.dumps({"files": results})
+        assert "files" in json.loads(j)

package/skills/claw-compactor/tests/test_generate_summary_tiers.py

@@ -0,0 +1,105 @@
+"""Tests for generate_summary_tiers.py."""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from generate_summary_tiers import generate_tiers, format_human, format_tier_template, _find_memory_files, _classify_section
+
+
+class TestClassifySection:
+    def test_decision(self):
+        assert _classify_section("Important Decisions") >= 9
+
+    def test_archive(self):
+        assert _classify_section("Archive") <= 2
+
+    def test_action(self):
+        assert _classify_section("Action Items") >= 8
+
+    def test_unknown(self):
+        assert _classify_section("Random Header") == 5
+
+    def test_case_insensitive(self):
+        assert _classify_section("CRITICAL BUGS") >= 9
+
+
+class TestFindMemoryFiles:
+    def test_workspace(self, tmp_workspace):
+        files = _find_memory_files(str(tmp_workspace))
+        assert len(files) > 0
+        assert "MEMORY.md" in str(files[0])
+
+    def test_single_file(self, tmp_workspace):
+        assert len(_find_memory_files(str(tmp_workspace / "MEMORY.md"))) == 1
+
+    def test_nonexistent(self):
+        with pytest.raises(Exception):
+            _find_memory_files("/nonexistent/xyz")
+
+    def test_empty_dir(self, tmp_path):
+        assert len(_find_memory_files(str(tmp_path))) == 0
+
+
+class TestGenerateTiers:
+    def test_basic(self, tmp_workspace):
+        result = generate_tiers(_find_memory_files(str(tmp_workspace)))
+        assert "total_tokens" in result
+        assert len(result["tiers"]) == 3
+
+    def test_budgets_respected(self, tmp_workspace):
+        result = generate_tiers(_find_memory_files(str(tmp_workspace)))
+        for tier in result["tiers"].values():
+            assert tier["tokens_used"] <= tier["budget"]
+
+    def test_tier_ordering(self, tmp_workspace):
+        result = generate_tiers(_find_memory_files(str(tmp_workspace)))
+        counts = [result["tiers"][i]["sections_included"] for i in range(3)]
+        assert counts[0] <= counts[1] <= counts[2]
+
+    def test_empty_file(self, empty_file):
+        assert generate_tiers([empty_file])["total_tokens"] == 0
+
+    def test_unicode(self, unicode_file):
+        assert generate_tiers([unicode_file])["total_tokens"] > 0
+
+    def test_large_file(self, large_file):
+        assert generate_tiers([large_file])["total_sections"] > 100
+
+    def test_headers_only(self, headers_only):
+        assert isinstance(generate_tiers([headers_only]), dict)
+
+    def test_single_line(self, single_line):
+        assert generate_tiers([single_line])["total_sections"] >= 1
+
+    def test_broken_markdown(self, broken_markdown):
+        assert isinstance(generate_tiers([broken_markdown]), dict)
+
+
+class TestFormatHuman:
+    def test_output(self, tmp_workspace):
+        output = format_human(generate_tiers(_find_memory_files(str(tmp_workspace))))
+        assert "Summary Tier Analysis" in output
+        for level in range(3):
+            assert "Level {}".format(level) in output
+
+
+class TestFormatTierTemplate:
+    def test_markdown(self, tmp_workspace):
+        result = generate_tiers(_find_memory_files(str(tmp_workspace)))
+        for level in range(3):
+            t = format_tier_template(result, level)
+            assert "Level {}".format(level) in t
+            assert "Budget:" in t
+
+    def test_empty(self, empty_file):
+        assert isinstance(format_tier_template(generate_tiers([empty_file]), 0), str)
+
+
+class TestJsonOutput:
+    def test_serializable(self, tmp_workspace):
+        result = generate_tiers(_find_memory_files(str(tmp_workspace)))
+        assert "tiers" in json.loads(json.dumps(result))

package/skills/claw-compactor/tests/test_integration.py

@@ -0,0 +1,249 @@
+"""Integration tests: full pipeline and cross-module scenarios."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from estimate_tokens import scan_path, format_human as est_format
+from compress_memory import rule_compress, compress_file, generate_llm_prompt
+from generate_summary_tiers import generate_tiers
+from dedup_memory import run_dedup
+from lib.tokens import estimate_tokens
+from lib.markdown import parse_sections, strip_markdown_redundancy
+from lib.dedup import find_duplicates
+
+
+class TestFullPipeline:
+    """End-to-end: estimate → compress → tiers → dedup."""
+
+    def test_estimate_compress_tiers_dedup(self, tmp_workspace: Path) -> None:
+        """Full pipeline on a populated workspace."""
+        ws = str(tmp_workspace)
+
+        # 1. Estimate
+        results = scan_path(ws)
+        assert len(results) > 0
+        total_before = sum(r["tokens"] for r in results)
+        assert total_before > 0
+
+        # 2. Compress each daily file
+        memory_dir = tmp_workspace / "memory"
+        for f in sorted(memory_dir.glob("*.md")):
+            stats = compress_file(f, dry_run=True, no_llm=True)
+            assert stats["original_tokens"] >= stats["rule_compressed_tokens"]
+            assert stats["rule_reduction_pct"] >= 0
+
+        # 3. Generate tiers
+        files = [tmp_workspace / "MEMORY.md"] + sorted(memory_dir.glob("*.md"))
+        tier_result = generate_tiers(files)
+        assert tier_result["total_tokens"] > 0
+        assert 0 in tier_result["tiers"]
+        assert 1 in tier_result["tiers"]
+        assert 2 in tier_result["tiers"]
+        # Level 0 should use fewer tokens than Level 2
+        assert tier_result["tiers"][0]["tokens_used"] <= tier_result["tiers"][2]["tokens_used"]
+
+        # 4. Dedup
+        dedup_result = run_dedup(ws)
+        assert "duplicate_groups" in dedup_result
+        assert dedup_result["total_entries"] >= 0
+
+    def test_unicode_pipeline(self, unicode_file: Path) -> None:
+        """Pipeline with Chinese/mixed content."""
+        # Estimate
+        results = scan_path(str(unicode_file))
+        assert len(results) == 1
+        assert results[0]["tokens"] > 0
+
+        # Compress
+        stats = compress_file(unicode_file, dry_run=True, no_llm=True)
+        assert stats["original_tokens"] > 0
+
+        # Parse sections
+        text = unicode_file.read_text(encoding="utf-8")
+        sections = parse_sections(text)
+        assert len(sections) >= 1
+
+    def test_large_file_pipeline(self, large_file: Path) -> None:
+        """Pipeline with ~100k+ char file."""
+        results = scan_path(str(large_file))
+        assert results[0]["tokens"] > 10000
+
+        stats = compress_file(large_file, dry_run=True, no_llm=True)
+        assert stats["original_tokens"] > 10000
+        # Rule compression should remove at least something
+        assert stats["rule_compressed_tokens"] <= stats["original_tokens"]
+
+    def test_empty_file_resilience(self, empty_file: Path) -> None:
+        """All tools handle empty file gracefully."""
+        results = scan_path(str(empty_file))
+        assert len(results) >= 1  # the file exists, just empty
+
+        stats = compress_file(empty_file, dry_run=True, no_llm=True)
+        assert stats["original_tokens"] == 0
+
+    def test_broken_markdown_resilience(self, broken_markdown: Path) -> None:
+        """Broken markdown doesn't crash anything."""
+        results = scan_path(str(broken_markdown))
+        assert len(results) == 1
+
+        stats = compress_file(broken_markdown, dry_run=True, no_llm=True)
+        assert stats["original_tokens"] > 0
+
+        sections = parse_sections(broken_markdown.read_text(encoding="utf-8"))
+        assert len(sections) >= 1
+
+    def test_dedup_finds_known_duplicates(self, duplicate_content: Path) -> None:
+        """Dedup correctly identifies known duplicate content."""
+        result = run_dedup(str(duplicate_content), threshold=0.5)
+        assert result["total_entries"] >= 2
+        # The two files have nearly identical "Setup" sections
+        assert len(result["duplicate_groups"]) >= 1
+
+    def test_dedup_auto_merge(self, duplicate_content: Path) -> None:
+        """Auto-merge removes duplicates and preserves unique content."""
+        result = run_dedup(str(duplicate_content), threshold=0.5, auto_merge=True)
+        if result["duplicate_groups"]:
+            assert result.get("tokens_saved", 0) >= 0
+            if "files_modified" in result:
+                # Verify unique content survived
+                b_text = (duplicate_content / "b.md").read_text(encoding="utf-8")
+                assert "Unique content" in b_text
+
+
+class TestCompressionQuality:
+    """Verify compression preserves information."""
+
+    def test_rule_compress_preserves_facts(self) -> None:
+        """Rule compression keeps key facts."""
+        text = (
+            "# 2025-01-15 Session Notes\n\n"
+            "- Fixed critical bug in API endpoint /v2/orders\n"
+            "- Server IP: 10.0.2.1\n"
+            "- Deployed version 2.1.3 to production\n"
+            "- Met with Alice about Q2 roadmap\n"
+            "- Decision: migrate to PostgreSQL 16\n"
+        )
+        compressed = rule_compress(text)
+        # All critical facts should survive rule compression
+        assert "10.0.2.1" in compressed
+        assert "2.1.3" in compressed
+        assert "PostgreSQL" in compressed
+
+    def test_rule_compress_removes_dupes(self) -> None:
+        """Rule compression removes exact duplicate lines."""
+        text = (
+            "# Notes\n"
+            "- Deploy on AWS\n"
+            "- Deploy on AWS\n"
+            "- Different thing\n"
+            "- Deploy on AWS\n"
+        )
+        compressed = rule_compress(text)
+        assert compressed.count("Deploy on AWS") == 1
+        assert "Different thing" in compressed
+
+    def test_llm_prompt_contains_content(self) -> None:
+        """LLM prompt includes the content to compress."""
+        prompt = generate_llm_prompt("My test content here", target_pct=40)
+        assert "My test content here" in prompt
+        assert "40%" in prompt
+
+    def test_compression_ratio_reasonable(self, tmp_workspace: Path) -> None:
+        """Rule compression achieves reasonable ratio on real-ish content."""
+        mem_file = tmp_workspace / "MEMORY.md"
+        text = mem_file.read_text(encoding="utf-8")
+        original_tokens = estimate_tokens(text)
+        compressed = rule_compress(text)
+        compressed_tokens = estimate_tokens(compressed)
+        # Rule compression alone shouldn't inflate
+        assert compressed_tokens <= original_tokens
+
+
+class TestTierQuality:
+    """Verify tier generation correctness."""
+
+    def test_tier_budget_respected(self, tmp_workspace: Path) -> None:
+        """Each tier stays within its token budget."""
+        files = [tmp_workspace / "MEMORY.md"]
+        result = generate_tiers(files)
+        for tier_level, tier in result["tiers"].items():
+            assert tier["tokens_used"] <= tier["budget"], (
+                f"Tier {tier_level} exceeded budget: "
+                f"{tier['tokens_used']} > {tier['budget']}"
+            )
+
+    def test_tier_ordering(self, tmp_workspace: Path) -> None:
+        """Higher tiers include more content."""
+        files = [tmp_workspace / "MEMORY.md"]
+        result = generate_tiers(files)
+        tiers = result["tiers"]
+        assert tiers[0]["tokens_used"] <= tiers[1]["tokens_used"]
+        assert tiers[1]["tokens_used"] <= tiers[2]["tokens_used"]
+
+    def test_tier_priority_ordering(self, tmp_workspace: Path) -> None:
+        """Higher-priority sections are included first."""
+        files = [tmp_workspace / "MEMORY.md"]
+        result = generate_tiers(files)
+        for tier in result["tiers"].values():
+            if len(tier["sections"]) >= 2:
+                priorities = [s["priority"] for s in tier["sections"]]
+                # Should be roughly descending (it IS sorted descending)
+                assert priorities == sorted(priorities, reverse=True)
+
+
+class TestCLIScripts:
+    """Test scripts work as standalone CLI tools."""
+
+    def _run_script(self, script: str, args: list, cwd: str = None) -> subprocess.CompletedProcess:
+        script_path = Path(__file__).parent.parent / "scripts" / script
+        return subprocess.run(
+            [sys.executable, str(script_path)] + args,
+            capture_output=True, text=True, timeout=30,
+            cwd=cwd,
+        )
+
+    def test_estimate_tokens_cli(self, tmp_workspace: Path) -> None:
+        r = self._run_script("estimate_tokens.py", [str(tmp_workspace)])
+        assert r.returncode == 0
+        assert "tokens" in r.stdout.lower()
+
+    def test_estimate_tokens_json(self, tmp_workspace: Path) -> None:
+        r = self._run_script("estimate_tokens.py", [str(tmp_workspace), "--json"])
+        assert r.returncode == 0
+        data = json.loads(r.stdout)
+        assert "files" in data
+
+    def test_compress_dry_run(self, tmp_workspace: Path) -> None:
+        r = self._run_script("compress_memory.py", [
+            str(tmp_workspace / "MEMORY.md"), "--dry-run", "--json"
+        ])
+        assert r.returncode == 0
+
+    def test_tiers_cli(self, tmp_workspace: Path) -> None:
+        r = self._run_script("generate_summary_tiers.py", [str(tmp_workspace)])
+        assert r.returncode == 0
+        assert "Level" in r.stdout
+
+    def test_tiers_json(self, tmp_workspace: Path) -> None:
+        r = self._run_script("generate_summary_tiers.py", [str(tmp_workspace), "--json"])
+        assert r.returncode == 0
+        data = json.loads(r.stdout)
+        assert "tiers" in data
+
+    def test_dedup_cli(self, tmp_workspace: Path) -> None:
+        r = self._run_script("dedup_memory.py", [str(tmp_workspace)])
+        assert r.returncode == 0
+
+    def test_dedup_json(self, tmp_workspace: Path) -> None:
+        r = self._run_script("dedup_memory.py", [str(tmp_workspace), "--json"])
+        assert r.returncode == 0
+        data = json.loads(r.stdout)
+        assert "duplicate_groups" in data
+
+    def test_nonexistent_path(self) -> None:
+        r = self._run_script("estimate_tokens.py", ["/nonexistent/xyz123"])
+        assert r.returncode != 0