mumucc 0.4.4 → 0.4.10

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_corpus.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.corpus import Corpus
+
+
+class TestCorpus:
+    def test_init_from_plugins(self, sample_plugin_dir: Path, tmp_path: Path):
+        corpus_dir = tmp_path / "corpus"
+        corpus = Corpus.init_from_source(sample_plugin_dir.parent, corpus_dir)
+        assert corpus.size > 0
+        assert (corpus_dir / "index.json").exists()
+
+    def test_select_references(self, sample_plugin_dir: Path, tmp_path: Path):
+        corpus_dir = tmp_path / "corpus"
+        corpus = Corpus.init_from_source(sample_plugin_dir.parent, corpus_dir)
+        refs = corpus.select_references(category="development", n=3)
+        assert len(refs) <= 3
+
+    def test_list_skills(self, sample_plugin_dir: Path, tmp_path: Path):
+        corpus_dir = tmp_path / "corpus"
+        corpus = Corpus.init_from_source(sample_plugin_dir.parent, corpus_dir)
+        skills = corpus.list_skills()
+        assert len(skills) > 0

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_e2e.py

@@ -0,0 +1,47 @@
+"""End-to-end tests using real plugins from the claude-agents repo."""
+
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.engine import EvalEngine
+from plugin_eval.models import Depth, EvalConfig
+
+REPO_ROOT = Path(__file__).parent.parent.parent.parent  # plugins/plugin-eval/tests -> claude-agents
+
+
+@pytest.mark.skipif(
+    not (REPO_ROOT / "plugins" / "observability-monitoring").exists(),
+    reason="Real plugin directory not available",
+)
+class TestE2ERealPlugins:
+    def test_score_real_skill_quick(self):
+        skill_dir = (
+            REPO_ROOT / "plugins" / "observability-monitoring" / "skills" / "distributed-tracing"
+        )
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(skill_dir)
+
+        assert result.composite is not None
+        assert result.composite.score > 50  # known good skill
+        assert len(result.layers) == 1
+
+    def test_score_real_plugin_quick(self):
+        plugin_dir = REPO_ROOT / "plugins" / "observability-monitoring"
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_plugin(plugin_dir)
+
+        assert result.composite is not None
+        assert result.composite.score > 40
+
+    def test_score_agent_teams_quick(self):
+        """Test a coordination-style plugin (shorter skills)."""
+        skill_dir = REPO_ROOT / "plugins" / "agent-teams" / "skills" / "multi-reviewer-patterns"
+        if not skill_dir.exists():
+            pytest.skip("agent-teams plugin not available")
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(skill_dir)
+        assert result.composite.score > 30

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_elo.py

@@ -0,0 +1,48 @@
+import pytest
+
+from plugin_eval.elo import EloCalculator
+
+
+class TestEloCalculator:
+    def test_expected_score(self):
+        calc = EloCalculator(k_factor=32)
+        assert calc.expected(1500, 1500) == pytest.approx(0.5)
+        assert calc.expected(1600, 1500) > 0.5
+        assert calc.expected(1400, 1500) < 0.5
+
+    def test_update_win(self):
+        calc = EloCalculator(k_factor=32)
+        new_rating = calc.update(1500, 1500, actual=1.0)
+        assert new_rating > 1500
+        assert new_rating == pytest.approx(1516.0)
+
+    def test_update_loss(self):
+        calc = EloCalculator(k_factor=32)
+        new_rating = calc.update(1500, 1500, actual=0.0)
+        assert new_rating < 1500
+
+    def test_update_draw(self):
+        calc = EloCalculator(k_factor=32)
+        new_rating = calc.update(1500, 1500, actual=0.5)
+        assert new_rating == pytest.approx(1500.0)
+
+    def test_compute_rating_from_matchups(self):
+        calc = EloCalculator(k_factor=32)
+        matchups = [
+            (1540, 1.0),
+            (1500, 0.0),
+            (1460, 1.0),
+        ]
+        final = calc.compute_rating(1500, matchups)
+        assert final > 1500
+
+    def test_bootstrap_ci(self):
+        calc = EloCalculator(k_factor=32)
+        matchups = [
+            (1540, 1.0),
+            (1500, 0.0),
+            (1460, 1.0),
+            (1520, 0.5),
+        ]
+        rating, lower, upper = calc.compute_rating_with_ci(1500, matchups, seed=42)
+        assert lower <= rating <= upper or lower == upper

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_engine.py

@@ -0,0 +1,42 @@
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.engine import EvalEngine
+from plugin_eval.models import Depth, EvalConfig, PluginEvalResult
+
+
+class TestEvalEngine:
+    def test_quick_eval_skill(self, sample_skill_dir: Path):
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(sample_skill_dir)
+        assert isinstance(result, PluginEvalResult)
+        assert len(result.layers) == 1
+        assert result.layers[0].layer == "static"
+        assert result.composite is not None
+        assert result.composite.confidence_label == "Estimated"
+
+    def test_quick_eval_plugin(self, sample_plugin_dir: Path):
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_plugin(sample_plugin_dir)
+        assert isinstance(result, PluginEvalResult)
+        assert result.composite.score > 0
+
+    def test_composite_score_within_bounds(self, sample_skill_dir: Path):
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(sample_skill_dir)
+        assert 0 <= result.composite.score <= 100
+
+    def test_layer_blend_renormalization(self):
+        """When only L1 is available, L1 weights should renormalize to 1.0."""
+        engine = EvalEngine(EvalConfig(depth=Depth.QUICK))
+        blended = engine._blend_layer_scores(
+            static_scores={"triggering_accuracy": 0.9, "orchestration_fitness": 0.8},
+            judge_scores=None,
+            mc_scores=None,
+        )
+        assert blended["triggering_accuracy"] > 0
+        assert blended["orchestration_fitness"] > 0

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_judge.py

@@ -0,0 +1,58 @@
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from plugin_eval.layers.judge import JudgeAnalyzer, JudgeConfig
+
+
+class TestJudgeConfig:
+    def test_default_config(self):
+        config = JudgeConfig()
+        assert config.judges == 1
+        assert config.auth == "max"
+
+
+class TestJudgeAnalyzer:
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.judge.query_llm")
+    async def test_assess_triggering(self, mock_query, sample_skill_dir: Path):
+        mock_query.return_value = {
+            "predictions": [
+                {"prompt": "test logging", "should_trigger": True, "would_trigger": True},
+                {"prompt": "make coffee", "should_trigger": False, "would_trigger": False},
+            ],
+            "precision": 1.0,
+            "recall": 1.0,
+            "f1": 1.0,
+        }
+        analyzer = JudgeAnalyzer(JudgeConfig())
+        result = await analyzer.assess_triggering(sample_skill_dir)
+        assert result["f1"] == 1.0
+        mock_query.assert_called()
+
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.judge.query_llm")
+    async def test_assess_orchestration(self, mock_query, sample_skill_dir: Path):
+        mock_query.return_value = {
+            "score": 0.82,
+            "reasoning": "Clean worker role with structured outputs.",
+            "evidence": ["Output format documented", "No orchestration logic"],
+        }
+        analyzer = JudgeAnalyzer(JudgeConfig())
+        result = await analyzer.assess_orchestration(sample_skill_dir)
+        assert result["score"] == 0.82
+
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.judge.query_llm")
+    async def test_full_analysis(self, mock_query, sample_skill_dir: Path):
+        mock_query.side_effect = [
+            {"f1": 0.85, "precision": 0.90, "recall": 0.80, "predictions": []},
+            {"score": 0.82, "reasoning": "Good", "evidence": []},
+            {"score": 0.79, "simulations": []},
+            {"score": 0.88, "assessment": "well-scoped"},
+        ]
+        analyzer = JudgeAnalyzer(JudgeConfig())
+        result = await analyzer.analyze_skill(sample_skill_dir)
+        assert result.layer == "judge"
+        assert result.score > 0

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_models.py

@@ -0,0 +1,102 @@
+import pytest
+from pydantic import ValidationError
+
+from plugin_eval.models import (
+    AntiPattern,
+    Badge,
+    CompositeResult,
+    Depth,
+    DimensionScore,
+    EloMatchup,
+    EloResult,
+    EvalConfig,
+    LayerResult,
+    PluginEvalResult,
+    StaticSubScore,
+)
+
+
+class TestEvalConfig:
+    def test_default_config(self):
+        config = EvalConfig()
+        assert config.depth == Depth.STANDARD
+        assert config.concurrency == 4
+        assert config.auth == "max"
+
+    def test_custom_config(self):
+        config = EvalConfig(depth=Depth.DEEP, concurrency=8)
+        assert config.depth == Depth.DEEP
+        assert config.concurrency == 8
+
+    def test_concurrency_bounds(self):
+        with pytest.raises(ValidationError):
+            EvalConfig(concurrency=0)
+        with pytest.raises(ValidationError):
+            EvalConfig(concurrency=21)
+
+
+class TestDimensionScore:
+    def test_valid_score(self):
+        ds = DimensionScore(name="triggering_accuracy", weight=0.25, score=0.85)
+        assert ds.weighted_score == pytest.approx(0.2125)
+
+    def test_score_bounds(self):
+        with pytest.raises(ValidationError):
+            DimensionScore(name="x", weight=0.1, score=1.5)
+        with pytest.raises(ValidationError):
+            DimensionScore(name="x", weight=0.1, score=-0.1)
+
+    def test_optional_ci(self):
+        ds = DimensionScore(
+            name="triggering_accuracy",
+            weight=0.25,
+            score=0.85,
+            ci_lower=0.80,
+            ci_upper=0.90,
+        )
+        assert ds.ci_lower == 0.80
+
+
+class TestAntiPattern:
+    def test_anti_pattern(self):
+        ap = AntiPattern(flag="OVER_CONSTRAINED", description="Too many MUSTs", severity=0.05)
+        assert ap.flag == "OVER_CONSTRAINED"
+
+
+class TestLayerResult:
+    def test_layer_result(self):
+        lr = LayerResult(layer="static", score=0.91)
+        assert lr.score == 0.91
+
+
+class TestEloMatchup:
+    def test_matchup(self):
+        m = EloMatchup(
+            opponent="distributed-tracing",
+            opponent_elo=1540,
+            result="loss",
+            score=0.44,
+        )
+        assert m.result in ("win", "loss", "draw")
+
+    def test_invalid_result(self):
+        with pytest.raises(ValidationError):
+            EloMatchup(opponent="x", opponent_elo=1500, result="tie", score=0.5)
+
+
+class TestBadge:
+    def test_badge_from_scores_gold(self):
+        badge = Badge.from_scores(composite=85, elo=1520)
+        assert badge == Badge.GOLD
+
+    def test_badge_from_scores_platinum(self):
+        badge = Badge.from_scores(composite=92, elo=1650)
+        assert badge == Badge.PLATINUM
+
+    def test_badge_requires_both(self):
+        badge = Badge.from_scores(composite=95, elo=1200)
+        assert badge == Badge.NO_BADGE
+
+    def test_badge_no_elo(self):
+        badge = Badge.from_scores(composite=85, elo=None)
+        assert badge == Badge.GOLD

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_monte_carlo.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from plugin_eval.layers.monte_carlo import MonteCarloAnalyzer, MonteCarloConfig, SimResult
+
+
+class TestSimResult:
+    def test_sim_result(self):
+        sr = SimResult(activated=True, quality_score=0.8, tokens=2500, duration_ms=1200)
+        assert sr.activated is True
+        assert sr.errored is False
+
+
+class TestMonteCarloAnalyzer:
+    @pytest.mark.asyncio
+    @patch("plugin_eval.layers.monte_carlo.run_simulation")
+    async def test_run_with_mocked_sims(self, mock_sim, sample_skill_dir: Path):
+        mock_sim.return_value = SimResult(
+            activated=True, quality_score=0.82, tokens=2800, duration_ms=1500
+        )
+        config = MonteCarloConfig(n_runs=10, concurrency=2)
+        analyzer = MonteCarloAnalyzer(config)
+        result = await analyzer.analyze_skill(sample_skill_dir)
+        assert result.layer == "monte_carlo"
+        assert result.score > 0
+        assert "triggering" in result.sub_scores
+        assert "output_consistency" in result.sub_scores
+        assert "failure_rate" in result.sub_scores
+
+    def test_statistical_analysis(self):
+        """Test the statistical analysis on pre-computed sim results."""
+        analyzer = MonteCarloAnalyzer(MonteCarloConfig(n_runs=50))
+        results = [
+            SimResult(activated=True, quality_score=0.8 + i * 0.002, tokens=2500, duration_ms=1200)
+            for i in range(48)
+        ] + [
+            SimResult(activated=False, quality_score=0.0, tokens=500, duration_ms=200, errored=True),
+            SimResult(activated=True, quality_score=0.75, tokens=8000, duration_ms=5000),
+        ]
+        stats = analyzer._compute_statistics(results)
+        assert stats["triggering"]["activation_rate"] == pytest.approx(0.98)
+        assert stats["failure_rate"]["p_fail"] == pytest.approx(0.02)
+        assert stats["output_consistency"]["cv"] < 0.15

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_parser.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.parser import ParsedSkill, ParsedAgent, ParsedPlugin, parse_skill, parse_agent, parse_plugin
+
+
+class TestParseSkill:
+    def test_parse_valid_skill(self, sample_skill_dir: Path):
+        skill = parse_skill(sample_skill_dir)
+        assert skill.name == "test-skill"
+        assert "testing plugin-eval" in skill.description
+        assert skill.line_count > 0
+        assert skill.h2_count >= 2
+        assert skill.code_block_count >= 1
+        assert skill.has_references is True
+
+    def test_parse_poor_skill(self, poor_skill_dir: Path):
+        skill = parse_skill(poor_skill_dir)
+        assert skill.name == "poor-skill"
+        assert skill.must_never_always_count > 15
+        assert skill.has_references is True
+        assert len(skill.reference_files) == 1
+
+    def test_missing_skill_md_raises(self, tmp_path: Path):
+        empty = tmp_path / "empty-skill"
+        empty.mkdir()
+        with pytest.raises(FileNotFoundError):
+            parse_skill(empty)
+
+
+class TestParseAgent:
+    def test_parse_valid_agent(self, sample_plugin_dir: Path):
+        agent_path = sample_plugin_dir / "agents" / "test-agent.md"
+        agent = parse_agent(agent_path)
+        assert agent.name == "test-agent"
+        assert agent.model == "sonnet"
+        assert agent.has_tools_restriction is True
+        assert agent.has_proactive_trigger is True
+
+
+class TestParsePlugin:
+    def test_parse_valid_plugin(self, sample_plugin_dir: Path):
+        plugin = parse_plugin(sample_plugin_dir)
+        assert plugin.name == "test-plugin"
+        assert len(plugin.skills) == 1
+        assert len(plugin.agents) == 1

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_reporter.py

@@ -0,0 +1,32 @@
+import json
+from pathlib import Path
+
+from plugin_eval.engine import EvalEngine
+from plugin_eval.models import Depth, EvalConfig
+from plugin_eval.reporter import Reporter
+
+
+class TestReporter:
+    def test_json_output(self, sample_skill_dir: Path):
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(sample_skill_dir)
+
+        reporter = Reporter()
+        output = reporter.to_json(result)
+        parsed = json.loads(output)
+        assert "composite" in parsed
+        assert "layers" in parsed
+        assert parsed["composite"]["confidence_label"] == "Estimated"
+
+    def test_markdown_output(self, sample_skill_dir: Path):
+        config = EvalConfig(depth=Depth.QUICK)
+        engine = EvalEngine(config)
+        result = engine.evaluate_skill(sample_skill_dir)
+
+        reporter = Reporter()
+        output = reporter.to_markdown(result)
+        assert "# PluginEval Report" in output
+        assert "Overall Score" in output
+        assert "Layer Breakdown" in output
+        assert "Dimension Scores" in output

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_static.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+import pytest
+
+from plugin_eval.layers.static import StaticAnalyzer
+from plugin_eval.models import LayerResult
+
+
+class TestStaticAnalyzer:
+    def test_analyze_valid_skill(self, sample_skill_dir: Path):
+        analyzer = StaticAnalyzer()
+        result = analyzer.analyze_skill(sample_skill_dir)
+        assert isinstance(result, LayerResult)
+        assert result.layer == "static"
+        assert result.score > 0.5
+        assert len(result.anti_patterns) == 0
+
+    def test_analyze_poor_skill(self, poor_skill_dir: Path):
+        analyzer = StaticAnalyzer()
+        result = analyzer.analyze_skill(poor_skill_dir)
+        assert result.score < 0.7
+        flags = [ap.flag for ap in result.anti_patterns]
+        assert "OVER_CONSTRAINED" in flags
+        assert "MISSING_TRIGGER" in flags
+
+    def test_analyze_plugin(self, sample_plugin_dir: Path):
+        analyzer = StaticAnalyzer()
+        result = analyzer.analyze_plugin(sample_plugin_dir)
+        assert result.layer == "static"
+        assert result.score > 0.5
+        assert "skill_scores" in result.sub_scores
+        assert "agent_scores" in result.sub_scores
+
+    def test_anti_pattern_penalty(self):
+        analyzer = StaticAnalyzer()
+        assert analyzer._anti_pattern_penalty(0) == 1.0
+        assert analyzer._anti_pattern_penalty(2) == pytest.approx(0.9)
+        assert analyzer._anti_pattern_penalty(10) == 0.5
+        assert analyzer._anti_pattern_penalty(20) == 0.5
+
+    def test_description_pushiness_score(self):
+        analyzer = StaticAnalyzer()
+        good = "Test skill for evaluation. Use when testing plugin-eval. Use PROACTIVELY for quality checks."
+        weak = "A skill."
+        assert analyzer._description_pushiness(good) > analyzer._description_pushiness(weak)

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/tests/test_stats.py

@@ -0,0 +1,100 @@
+import pytest
+
+from plugin_eval.stats import (
+    bootstrap_ci,
+    clopper_pearson_ci,
+    cohens_kappa,
+    coefficient_of_variation,
+    wilson_score_ci,
+)
+
+
+class TestWilsonScore:
+    def test_perfect_activation(self):
+        lower, upper = wilson_score_ci(successes=50, trials=50, confidence=0.95)
+        assert lower > 0.90
+        assert upper == pytest.approx(1.0, abs=0.01)
+
+    def test_half_activation(self):
+        lower, upper = wilson_score_ci(successes=25, trials=50, confidence=0.95)
+        assert lower < 0.50
+        assert upper > 0.50
+        assert lower > 0.35
+        assert upper < 0.65
+
+    def test_zero_trials_raises(self):
+        with pytest.raises(ValueError):
+            wilson_score_ci(successes=0, trials=0)
+
+    def test_successes_exceed_trials_raises(self):
+        with pytest.raises(ValueError):
+            wilson_score_ci(successes=10, trials=5)
+
+
+class TestBootstrapCI:
+    def test_tight_data(self):
+        data = [0.80, 0.82, 0.81, 0.83, 0.79, 0.80, 0.82, 0.81]
+        lower, upper = bootstrap_ci(data, confidence=0.95, n_resamples=1000, seed=42)
+        assert lower > 0.78
+        assert upper < 0.84
+        assert lower < upper
+
+    def test_single_value(self):
+        lower, upper = bootstrap_ci([0.5], confidence=0.95, n_resamples=100, seed=42)
+        assert lower == pytest.approx(0.5)
+        assert upper == pytest.approx(0.5)
+
+    def test_empty_raises(self):
+        with pytest.raises(ValueError):
+            bootstrap_ci([], confidence=0.95)
+
+
+class TestClopperPearson:
+    def test_zero_failures(self):
+        lower, upper = clopper_pearson_ci(failures=0, trials=50, confidence=0.95)
+        assert lower == 0.0
+        assert upper < 0.10
+
+    def test_some_failures(self):
+        lower, upper = clopper_pearson_ci(failures=2, trials=50, confidence=0.95)
+        assert lower < 0.04
+        assert upper > 0.04
+        assert upper < 0.15
+
+    def test_zero_trials_raises(self):
+        with pytest.raises(ValueError):
+            clopper_pearson_ci(failures=0, trials=0)
+
+
+class TestCoefficientOfVariation:
+    def test_low_variation(self):
+        data = [0.80, 0.82, 0.81, 0.83, 0.79]
+        cv = coefficient_of_variation(data)
+        assert cv < 0.05
+
+    def test_high_variation(self):
+        data = [0.20, 0.90, 0.10, 0.95, 0.50]
+        cv = coefficient_of_variation(data)
+        assert cv > 0.40
+
+    def test_empty_raises(self):
+        with pytest.raises(ValueError):
+            coefficient_of_variation([])
+
+
+class TestCohensKappa:
+    def test_perfect_agreement(self):
+        rater1 = [1, 2, 3, 4, 5]
+        rater2 = [1, 2, 3, 4, 5]
+        k = cohens_kappa(rater1, rater2)
+        assert k == pytest.approx(1.0)
+
+    def test_no_agreement(self):
+        rater1 = [1, 2, 3, 4, 5]
+        rater2 = [5, 4, 3, 2, 1]
+        k = cohens_kappa(rater1, rater2)
+        assert k < 0.0
+
+    def test_mismatched_length_raises(self):
+        with pytest.raises(ValueError):
+            cohens_kappa([1, 2], [1, 2, 3])