@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/tests/test_main_entry.py

@@ -0,0 +1,176 @@
+"""Tests for the unified entry point (mem_compress.py)."""
+import json
+import sys
+import pytest
+from pathlib import Path
+from unittest.mock import patch
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from mem_compress import build_parser, COMMAND_MAP, _workspace_path, _count_tokens_in_workspace
+
+
+class TestBuildParser:
+    """Test argument parser construction."""
+
+    def test_parser_accepts_all_commands(self):
+        parser = build_parser()
+        for cmd in ["compress", "estimate", "dedup", "tiers", "audit",
+                     "observe", "dict", "optimize", "full", "benchmark"]:
+            args = parser.parse_args(["/tmp/ws", cmd])
+            assert args.command == cmd
+            assert args.workspace == "/tmp/ws"
+
+    def test_parser_json_flag(self):
+        parser = build_parser()
+        args = parser.parse_args(["/tmp/ws", "estimate", "--json"])
+        assert args.json is True
+
+    def test_parser_dry_run(self):
+        parser = build_parser()
+        args = parser.parse_args(["/tmp/ws", "compress", "--dry-run"])
+        assert args.dry_run is True
+
+    def test_parser_since(self):
+        parser = build_parser()
+        args = parser.parse_args(["/tmp/ws", "observe", "--since", "2026-01-01"])
+        assert args.since == "2026-01-01"
+
+    def test_parser_threshold_val(self):
+        parser = build_parser()
+        args = parser.parse_args(["/tmp/ws", "dedup", "--threshold-val", "0.8"])
+        assert args.threshold_val == 0.8
+
+    def test_parser_rejects_unknown_command(self):
+        parser = build_parser()
+        with pytest.raises(SystemExit):
+            parser.parse_args(["/tmp/ws", "unknown"])
+
+    def test_parser_requires_workspace(self):
+        parser = build_parser()
+        with pytest.raises(SystemExit):
+            parser.parse_args(["estimate"])
+
+
+class TestCommandMap:
+    """Test command routing."""
+
+    def test_all_commands_have_handlers(self):
+        expected = {"compress", "estimate", "dedup", "tiers", "audit",
+                    "observe", "dict", "optimize", "full", "benchmark", "install", "auto", "engram"}
+        assert set(COMMAND_MAP.keys()) == expected
+
+    def test_handlers_are_callable(self):
+        for name, handler in COMMAND_MAP.items():
+            assert callable(handler), f"{name} handler not callable"
+
+
+class TestWorkspacePath:
+    """Test workspace validation."""
+
+    def test_valid_workspace(self, tmp_path):
+        result = _workspace_path(str(tmp_path))
+        assert result == tmp_path
+
+    def test_invalid_workspace(self):
+        with pytest.raises(SystemExit):
+            _workspace_path("/nonexistent/path/xyz")
+
+
+class TestCountTokens:
+    """Test workspace token counting."""
+
+    def test_empty_workspace(self, tmp_path):
+        tokens = _count_tokens_in_workspace(tmp_path)
+        assert tokens == 0
+
+    def test_workspace_with_files(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("Hello world this is a test of token counting")
+        tokens = _count_tokens_in_workspace(tmp_path)
+        assert tokens > 0
+
+    def test_workspace_with_memory_dir(self, tmp_path):
+        mem = tmp_path / "memory"
+        mem.mkdir()
+        (mem / "2026-01-01.md").write_text("Some daily notes here")
+        tokens = _count_tokens_in_workspace(tmp_path)
+        assert tokens > 0
+
+
+class TestEstimateCommand:
+    """Test estimate command."""
+
+    def test_estimate_json(self, tmp_path):
+        (tmp_path / "MEMORY.md").write_text("# Test\nSome content here")
+        from mem_compress import cmd_estimate
+        import argparse
+        args = argparse.Namespace(json=True, threshold=0)
+        result = cmd_estimate(tmp_path, args)
+        assert result == 0
+
+    def test_estimate_human(self, tmp_path, capsys):
+        (tmp_path / "MEMORY.md").write_text("# Test\nSome content")
+        from mem_compress import cmd_estimate
+        import argparse
+        args = argparse.Namespace(json=False, threshold=0)
+        result = cmd_estimate(tmp_path, args)
+        assert result == 0
+        out = capsys.readouterr().out
+        assert "token" in out.lower() or "Token" in out or len(out) > 0
+
+
+class TestCompressCommand:
+    """Test compress command."""
+
+    def test_compress_dry_run(self, tmp_path):
+        mem = tmp_path / "memory"
+        mem.mkdir()
+        (mem / "test.md").write_text("# Test\n\nHello world\n\nHello world\n")
+        from mem_compress import cmd_compress
+        import argparse
+        args = argparse.Namespace(json=False, dry_run=True, older_than=None)
+        result = cmd_compress(tmp_path, args)
+        assert result == 0
+
+
+class TestDedupCommand:
+    """Test dedup command."""
+
+    def test_dedup_json(self, tmp_path):
+        mem = tmp_path / "memory"
+        mem.mkdir()
+        (mem / "a.md").write_text("# Section\nSome content about X and Y\n")
+        (mem / "b.md").write_text("# Section\nSome content about X and Y\n")
+        from mem_compress import cmd_dedup
+        import argparse
+        args = argparse.Namespace(json=True, auto_merge=False, threshold_val=0.6)
+        result = cmd_dedup(tmp_path, args)
+        assert result == 0
+
+
+class TestBenchmarkCommand:
+    """Test benchmark command."""
+
+    def test_benchmark_human(self, tmp_path, capsys):
+        (tmp_path / "MEMORY.md").write_text("# Memory\nSome content\n## Section\nMore content\n")
+        (tmp_path / "TOOLS.md").write_text("# Tools\n| Key | Value |\n|-----|-------|\n| a | b |\n")
+        from mem_compress import cmd_benchmark
+        import argparse
+        args = argparse.Namespace(json=False)
+        result = cmd_benchmark(tmp_path, args)
+        assert result == 0
+        out = capsys.readouterr().out
+        assert "Performance Report" in out
+        assert "Rule Engine" in out
+        assert "TOTAL" in out
+
+    def test_benchmark_json(self, tmp_path, capsys):
+        (tmp_path / "MEMORY.md").write_text("# Memory\nContent\n")
+        from mem_compress import cmd_benchmark
+        import argparse
+        args = argparse.Namespace(json=True)
+        result = cmd_benchmark(tmp_path, args)
+        assert result == 0
+        data = json.loads(capsys.readouterr().out)
+        assert "total_before" in data
+        assert "total_after" in data
+        assert "steps" in data

package/skills/claw-compactor/tests/test_markdown_advanced.py

@@ -0,0 +1,284 @@
+"""Tests for advanced markdown compression functions."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from lib.markdown import (
+    normalize_chinese_punctuation,
+    strip_emoji,
+    remove_empty_sections,
+    compress_markdown_table,
+    merge_similar_bullets,
+    merge_short_bullets,
+)
+
+
+class TestNormalizeChinesePunctuation:
+    def test_basic(self):
+        assert normalize_chinese_punctuation("你好，世界！") == "你好,世界!"
+
+    def test_all_punctuation(self):
+        r = normalize_chinese_punctuation('说："你好"；（这里）')
+        assert '；' not in r
+        assert '\u201c' not in r and '\u201d' not in r
+        assert '（' not in r and '）' not in r
+
+    def test_no_change_ascii(self):
+        text = "Hello, world!"
+        assert normalize_chinese_punctuation(text) == text
+
+    def test_mixed(self):
+        r = normalize_chinese_punctuation("IP：192.168.1.1，端口：8080")
+        assert r == "IP:192.168.1.1,端口:8080"
+
+    def test_empty(self):
+        assert normalize_chinese_punctuation("") == ""
+
+    def test_brackets(self):
+        assert normalize_chinese_punctuation("【重要】") == "[重要]"
+
+
+class TestStripEmoji:
+    def test_removes_emoji(self):
+        assert "Title" in strip_emoji("⚠️ Title")
+        assert "⚠" not in strip_emoji("⚠️ Title")
+
+    def test_preserves_text(self):
+        assert strip_emoji("no emoji here") == "no emoji here"
+
+    def test_multiple_emoji(self):
+        r = strip_emoji("🏆 Winner 🎉 Party 🚀 Launch")
+        assert "Winner" in r and "Party" in r and "Launch" in r
+        assert "🏆" not in r
+
+    def test_empty(self):
+        assert strip_emoji("") == ""
+
+    def test_only_emoji(self):
+        r = strip_emoji("🎉🎊🎈")
+        assert r.strip() == ""
+
+    def test_chinese_with_emoji(self):
+        r = strip_emoji("📊 项目状态")
+        assert "项目状态" in r
+
+    def test_no_double_spaces(self):
+        r = strip_emoji("A 🎉 B")
+        assert "  " not in r
+
+
+class TestRemoveEmptySections:
+    def test_removes_empty(self):
+        text = "# Title\n\nContent\n\n## Empty\n\n## HasContent\n\nStuff"
+        r = remove_empty_sections(text)
+        assert "Empty" not in r
+        assert "HasContent" in r
+
+    def test_preserves_parent_with_children(self):
+        text = "# Parent\n\n## Child\n\nContent"
+        r = remove_empty_sections(text)
+        assert "# Parent" in r
+        assert "## Child" in r
+
+    def test_no_sections(self):
+        text = "Just plain text"
+        assert remove_empty_sections(text) == text
+
+    def test_all_empty(self):
+        text = "## A\n\n## B\n\n## C\n"
+        r = remove_empty_sections(text)
+        assert "A" not in r and "B" not in r and "C" not in r
+
+    def test_empty_input(self):
+        assert remove_empty_sections("") == ""
+
+    def test_nested_empty(self):
+        text = "# Top\n\n## Mid\n\n### Deep\n\nContent"
+        r = remove_empty_sections(text)
+        assert "# Top" in r  # has child
+        assert "## Mid" in r  # has child
+        assert "### Deep" in r  # has content
+
+
+class TestCompressMarkdownTable:
+    def test_two_column(self):
+        table = "| Key | Value |\n|-----|-------|\n| A | 1 |\n| B | 2 |"
+        r = compress_markdown_table(table)
+        assert "- A: 1" in r
+        assert "- B: 2" in r
+        assert "|" not in r
+
+    def test_three_column(self):
+        table = "| Name | Type | IP |\n|------|------|----|\n| srv1 | VPS | 1.2.3.4 |"
+        r = compress_markdown_table(table)
+        assert "srv1" in r
+        assert "Type=VPS" in r
+        assert "IP=1.2.3.4" in r
+
+    def test_wide_table_preserved(self):
+        header = "| A | B | C | D | E |"
+        sep = "|---|---|---|---|---|"
+        row = "| 1 | 2 | 3 | 4 | 5 |"
+        table = f"{header}\n{sep}\n{row}"
+        r = compress_markdown_table(table)
+        assert "|" in r  # preserved
+
+    def test_no_table(self):
+        text = "# Title\n\nJust text"
+        assert compress_markdown_table(text) == text
+
+    def test_mixed_content(self):
+        text = "Before\n\n| K | V |\n|---|---|\n| x | y |\n\nAfter"
+        r = compress_markdown_table(text)
+        assert "Before" in r
+        assert "After" in r
+        assert "- x: y" in r
+
+    def test_empty_table(self):
+        table = "| K | V |\n|---|---|"
+        r = compress_markdown_table(table)
+        # No rows, should be kept as-is or be harmless
+        assert isinstance(r, str)
+
+    def test_real_world_table(self):
+        table = (
+            "| 节点 | 类型 | IP | 状态 |\n"
+            "|------|------|-----|------|\n"
+            "| my-server | macOS | localhost | ✅ |\n"
+            "| remote-node | VPS | 10.0.1.2 | ✅ |"
+        )
+        r = compress_markdown_table(table)
+        assert "my-server" in r
+        assert "|" not in r  # 4 cols = compressed
+
+
+class TestMergeSimilarBullets:
+    def test_merges_near_duplicates(self):
+        text = "- Deploy v2.1 to staging\n- Deploy v2.1 to staging server"
+        r = merge_similar_bullets(text)
+        assert r.count("Deploy") == 1
+        assert "server" in r  # kept longer
+
+    def test_preserves_different_items(self):
+        text = "- Apple\n- Banana\n- Cherry"
+        r = merge_similar_bullets(text)
+        assert "Apple" in r and "Banana" in r and "Cherry" in r
+
+    def test_empty(self):
+        assert merge_similar_bullets("") == ""
+
+    def test_no_bullets(self):
+        text = "Just text\nMore text"
+        assert merge_similar_bullets(text) == text
+
+    def test_different_indent_levels(self):
+        text = "- Parent\n  - Child A\n  - Child A extended"
+        r = merge_similar_bullets(text)
+        assert "Parent" in r
+
+    def test_threshold(self):
+        text = "- item one\n- item two"
+        # These are 0.75 similar, default threshold is 0.80
+        r = merge_similar_bullets(text)
+        assert "item one" in r and "item two" in r
+
+    def test_three_similar(self):
+        text = "- Setup the server config\n- Setup the server configuration\n- Setup server config file"
+        r = merge_similar_bullets(text)
+        lines = [l for l in r.split('\n') if l.strip().startswith('-')]
+        assert len(lines) < 3
+
+
+class TestMergeShortBullets:
+    def test_merges_short(self):
+        text = "- A\n- B\n- C\n- D"
+        r = merge_short_bullets(text)
+        assert "A, B, C, D" in r
+        assert r.count('\n- ') == 0 or r.count('\n-') < 4
+
+    def test_preserves_long_bullets(self):
+        text = "- This is a long bullet point\n- Another long bullet point"
+        r = merge_short_bullets(text)
+        assert "This is a long" in r
+        assert r.count('\n') >= 1
+
+    def test_mixed(self):
+        text = "- A\n- B\n- C\n- This is long enough to not merge"
+        r = merge_short_bullets(text)
+        assert "A, B, C" in r
+        assert "This is long" in r
+
+    def test_fewer_than_three(self):
+        text = "- A\n- B"
+        r = merge_short_bullets(text)
+        # Only 2 short bullets, not enough to merge
+        assert "- A" in r and "- B" in r
+
+    def test_empty(self):
+        assert merge_short_bullets("") == ""
+
+    def test_no_bullets(self):
+        text = "plain text"
+        assert merge_short_bullets(text) == text
+
+    def test_max_merge(self):
+        text = "\n".join(f"- {chr(65+i)}" for i in range(10))
+        r = merge_short_bullets(text, max_merge=3)
+        # Should create multiple merged lines
+        lines = [l for l in r.split('\n') if l.strip()]
+        assert len(lines) < 10
+
+
+class TestIntegrationNewRules:
+    """Test that new rules work together in rule_compress pipeline."""
+
+    def test_real_memory_style(self):
+        """Simulate a real memory file with tables, emoji, Chinese punct."""
+        from compress_memory import rule_compress
+
+        text = """# 📊 项目状态
+
+## ⚠️ 铁律
+
+1. **主 session 不跑 exec** — 全部派 sub-agent
+2. **交易所 API 全走日本节点** — CCXT routed through remote-node
+
+## 节点
+
+| 节点 | 类型 | IP |
+|------|------|-----|
+| my-server | macOS | localhost |
+| remote-node | VPS | 10.0.1.2 |
+
+## 空标题
+
+## 数据
+
+- 下载数据
+- 下载数据到服务器
+- A
+- B
+- C
+- D
+"""
+        r = rule_compress(text)
+        # Emoji stripped
+        assert "📊" not in r
+        # Table compressed
+        assert "| 节点 |" not in r  # no table headers
+        assert "my-server" in r
+        # Empty section removed
+        assert "空标题" not in r  # removed (truly empty, no child)
+        # Short bullets merged (A, B, C, D are short single-char items)
+        assert "A, B, C, D" in r
+
+    def test_chinese_punctuation_in_pipeline(self):
+        from compress_memory import rule_compress
+
+        text = "# 笔记\n\n- 节点：remote-node，状态：在线"
+        r = rule_compress(text)
+        assert ":" in r  # Chinese colons converted
+        assert "," in r  # Chinese commas converted

package/skills/claw-compactor/tests/test_new_features.py

@@ -0,0 +1,77 @@
+"""Tests for new features: llm_compress_file, extract_key_facts, generate_auto_summary."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+from compress_memory import llm_compress_file
+from generate_summary_tiers import extract_key_facts, generate_auto_summary
+
+
+class TestLlmCompressFile:
+    def test_creates_prompt_file(self, tmp_path):
+        f = tmp_path / "test.md"
+        f.write_text("# Title\n\n" + "\n".join(f"- Entry {i}" for i in range(50)))
+        r = llm_compress_file(f)
+        prompt_path = Path(r["prompt_file"])
+        assert prompt_path.exists()
+        assert r["original_tokens"] > 0
+        assert r["prompt_tokens"] > r["rule_compressed_tokens"]
+        # Cleanup
+        prompt_path.unlink()
+
+    def test_stats(self, tmp_path):
+        f = tmp_path / "test.md"
+        f.write_text("# Simple\n\nContent\n")
+        r = llm_compress_file(f, target_pct=30)
+        assert r["target_pct"] == 30
+        assert "instruction" in r
+        Path(r["prompt_file"]).unlink()
+
+
+class TestExtractKeyFacts:
+    def test_key_value(self):
+        facts = extract_key_facts("- Name: Alex\n- IP: 192.168.1.1\n")
+        assert any("Alex" in f for f in facts)
+        assert any("192.168.1.1" in f for f in facts)
+
+    def test_important_markers(self):
+        facts = extract_key_facts("⚠️ Critical: do not delete\nNormal line\n")
+        assert any("Critical" in f for f in facts)
+
+    def test_empty(self):
+        assert extract_key_facts("") == []
+
+    def test_headers_skipped(self):
+        facts = extract_key_facts("# Header\n## Sub\nContent: value\n")
+        assert not any("Header" in f for f in facts)
+
+    def test_dedup(self):
+        facts = extract_key_facts("- A: 1\n- A: 1\n")
+        assert len(facts) == 1
+
+
+class TestGenerateAutoSummary:
+    def test_basic(self, tmp_path):
+        f = tmp_path / "test.md"
+        f.write_text("# Notes\n\n- Server: 192.168.1.1\n- Port: 8080\n")
+        summary = generate_auto_summary([f], budget=200)
+        assert "# Auto Summary" in summary
+        assert "192.168.1.1" in summary
+
+    def test_budget_limit(self, tmp_path):
+        f = tmp_path / "big.md"
+        lines = "\n".join(f"- Key{i}: Value{i} with some extra text" for i in range(200))
+        f.write_text(f"# Big\n\n{lines}\n")
+        summary = generate_auto_summary([f], budget=100)
+        # Should be within budget
+        from lib.tokens import estimate_tokens
+        assert estimate_tokens(summary) <= 120  # small margin
+
+    def test_empty_files(self, tmp_path):
+        f = tmp_path / "empty.md"
+        f.write_text("")
+        summary = generate_auto_summary([f])
+        assert "# Auto Summary" in summary