mumucc 0.4.4 → 0.4.10

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/src/plugin_eval/layers/judge.py

@@ -0,0 +1,296 @@
+"""Layer 2: LLM Judge — semantic evaluation via Claude, model-tiered, async."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from plugin_eval.models import LayerResult
+from plugin_eval.parser import ParsedSkill, parse_skill
+
+# ---------------------------------------------------------------------------
+# Anchored rubrics
+# ---------------------------------------------------------------------------
+
+ORCHESTRATION_RUBRIC = """
+Score 0.0 — Poor: Skill acts as standalone agent; manages its own tool calls and sub-tasks.
+Score 0.25 — Below average: Skill has some orchestration logic mixed with worker tasks.
+Score 0.5 — Average: Skill delegates some tasks but still coordinates multi-step flows itself.
+Score 0.75 — Good: Skill is mostly a worker; inputs/outputs documented, minimal coordination.
+Score 1.0 — Excellent: Pure worker role; composable, clear contracts, no orchestration logic.
+""".strip()
+
+SCOPE_RUBRIC = """
+Score 0.0 — Too thin: Stub or trivial wrapper with near-zero unique value.
+Score 0.25 — Under-scoped: Covers only a narrow slice; misses obvious related tasks.
+Score 0.5 — Average: Reasonable scope but either too broad or somewhat narrow.
+Score 0.75 — Well-scoped: Covers one coherent domain; neither bloated nor sparse.
+Score 1.0 — Perfectly calibrated: Minimal surface area, maximum cohesion, ideal composability.
+""".strip()
+
+# ---------------------------------------------------------------------------
+# Model resolution
+# ---------------------------------------------------------------------------
+
+_MODEL_MAP: dict[str, str] = {
+    "haiku": "claude-haiku-4-5-20251001",
+    "sonnet": "claude-sonnet-4-6",
+    "opus": "claude-opus-4-6",
+}
+
+
+def _resolve_model(tier: str) -> str:
+    """Map a tier name to a full model ID."""
+    return _MODEL_MAP.get(tier, _MODEL_MAP["sonnet"])
+
+
+# ---------------------------------------------------------------------------
+# LLM query helper (abstracted for testability)
+# ---------------------------------------------------------------------------
+
+
+async def query_llm(prompt: str, system: str = "", model: str = "claude-sonnet-4-6") -> dict:
+    """Call Claude via the Agent SDK and return a parsed JSON dict.
+
+    Raises RuntimeError if claude-agent-sdk is not installed.
+    """
+    try:
+        from claude_agent_sdk import (  # type: ignore[import-untyped]
+            ClaudeAgentOptions,
+            ResultMessage,
+            query,
+        )
+    except ImportError as exc:
+        raise RuntimeError(
+            "claude-agent-sdk is required for LLM judge. Install with: pip install plugin-eval[llm]"
+        ) from exc
+
+    full_prompt = prompt
+    if system:
+        full_prompt = f"{system}\n\n{prompt}"
+
+    result_text = ""
+    async for message in query(
+        prompt=full_prompt,
+        options=ClaudeAgentOptions(
+            model=model,
+            allowed_tools=[],
+        ),
+    ):
+        if isinstance(message, ResultMessage):
+            # ResultMessage contains the final text
+            for block in getattr(message, "content", []):
+                if hasattr(block, "text"):
+                    result_text += block.text
+
+    # Try to parse JSON — handles raw JSON or JSON inside a markdown code block
+    stripped = result_text.strip()
+    fence_match = re.search(r"```(?:json)?\s*([\s\S]+?)\s*```", stripped)
+    if fence_match:
+        stripped = fence_match.group(1).strip()
+
+    try:
+        return json.loads(stripped)
+    except json.JSONDecodeError:
+        return {"raw": result_text, "score": 0.5}
+
+
+# ---------------------------------------------------------------------------
+# Config
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class JudgeConfig:
+    judges: int = 1
+    auth: str = "max"
+    concurrency: int = 4
+    model_tier: str = "auto"
+
+
+# ---------------------------------------------------------------------------
+# Analyzer
+# ---------------------------------------------------------------------------
+
+
+class JudgeAnalyzer:
+    """Semantic skill evaluation using Claude as a judge."""
+
+    def __init__(self, config: JudgeConfig) -> None:
+        self.config = config
+        self._sem = asyncio.Semaphore(config.concurrency)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def analyze_skill(self, skill_or_dir: Path | ParsedSkill) -> LayerResult:
+        """Run all 4 assessments concurrently and return a LayerResult."""
+        skill = skill_or_dir if isinstance(skill_or_dir, ParsedSkill) else parse_skill(skill_or_dir)
+        triggering, orchestration, output_quality, scope = await asyncio.gather(
+            self.assess_triggering(skill),
+            self.assess_orchestration(skill),
+            self.assess_output_quality(skill),
+            self.assess_scope(skill),
+        )
+
+        # Weighted composite: triggering 0.30, orchestration 0.30, output 0.25, scope 0.15
+        score = (
+            triggering.get("f1", 0.5) * 0.30
+            + orchestration.get("score", 0.5) * 0.30
+            + output_quality.get("score", 0.5) * 0.25
+            + scope.get("score", 0.5) * 0.15
+        )
+        score = max(0.0, min(1.0, score))
+
+        sub_scores: dict[str, float] = {
+            "triggering_accuracy": triggering.get("f1", 0.5),
+            "orchestration_fitness": orchestration.get("score", 0.5),
+            "output_quality": output_quality.get("score", 0.5),
+            "scope_calibration": scope.get("score", 0.5),
+        }
+
+        metadata: dict = {
+            "triggering": triggering,
+            "orchestration": orchestration,
+            "output_quality": output_quality,
+            "scope": scope,
+        }
+
+        return LayerResult(
+            layer="judge",
+            score=score,
+            sub_scores=sub_scores,
+            metadata=metadata,
+        )
+
+    # ------------------------------------------------------------------
+    # Individual assessments
+    # ------------------------------------------------------------------
+
+    async def assess_triggering(self, skill: Path | ParsedSkill) -> dict:
+        """Generate 10 synthetic prompts and classify triggering accuracy via Haiku."""
+        if isinstance(skill, Path):
+            skill = parse_skill(skill)
+        model = _resolve_model("haiku")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Given this skill description:
+
+<description>
+{skill.description}
+</description>
+
+Generate 10 synthetic user prompts: 5 that SHOULD trigger this skill and 5 that should NOT.
+For each prompt, also predict whether a typical Claude model would trigger this skill.
+
+Return JSON matching this schema:
+{{
+  "predictions": [
+    {{"prompt": "...", "should_trigger": true, "would_trigger": true}},
+    ...
+  ],
+  "precision": <float 0-1>,
+  "recall": <float 0-1>,
+  "f1": <float 0-1>
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)
+
+    async def assess_orchestration(self, skill: Path | ParsedSkill) -> dict:
+        """Rate orchestration fitness using an anchored rubric via Sonnet."""
+        if isinstance(skill, Path):
+            skill = parse_skill(skill)
+        model = _resolve_model("sonnet")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Evaluate this skill's orchestration fitness.
+
+A skill should be a pure WORKER — it should NOT orchestrate other tools or agents.
+It should accept clear inputs and produce clear outputs.
+
+Rubric:
+{ORCHESTRATION_RUBRIC}
+
+Skill content:
+<skill>
+{skill.raw_content[:3000]}
+</skill>
+
+Return JSON:
+{{
+  "score": <float 0.0-1.0 matching rubric>,
+  "reasoning": "<one sentence>",
+  "evidence": ["<quote or observation>", ...]
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)
+
+    async def assess_output_quality(self, skill: Path | ParsedSkill) -> dict:
+        """Simulate 3 tasks and judge output quality via Sonnet."""
+        if isinstance(skill, Path):
+            skill = parse_skill(skill)
+        model = _resolve_model("sonnet")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Simulate 3 realistic tasks this skill would handle, then evaluate the
+expected output quality based on the skill's instructions.
+
+Skill content:
+<skill>
+{skill.raw_content[:3000]}
+</skill>
+
+Return JSON:
+{{
+  "score": <float 0.0-1.0>,
+  "simulations": [
+    {{"task": "...", "expected_output": "...", "quality_notes": "..."}}
+  ]
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)
+
+    async def assess_scope(self, skill: Path | ParsedSkill) -> dict:
+        """Evaluate scope calibration using an anchored rubric via Sonnet."""
+        if isinstance(skill, Path):
+            skill = parse_skill(skill)
+        model = _resolve_model("sonnet")
+
+        system = (
+            "You are an expert evaluator of Claude Code skills. "
+            "Respond ONLY with valid JSON — no explanation, no markdown fences."
+        )
+        prompt = f"""Evaluate this skill's scope calibration.
+
+Rubric:
+{SCOPE_RUBRIC}
+
+Skill content:
+<skill>
+{skill.raw_content[:3000]}
+</skill>
+
+Return JSON:
+{{
+  "score": <float 0.0-1.0 matching rubric>,
+  "assessment": "<one sentence>"
+}}"""
+
+        async with self._sem:
+            return await query_llm(prompt, system=system, model=model)

package/defaults/plugin-seed/marketplaces/claude-code-workflows/plugins/plugin-eval/src/plugin_eval/layers/monte_carlo.py

@@ -0,0 +1,348 @@
+"""Layer 3: Monte Carlo simulation — statistical reliability testing via repeated runs."""
+
+from __future__ import annotations
+
+import asyncio
+import statistics
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+from plugin_eval.models import LayerResult
+from plugin_eval.parser import ParsedSkill, parse_skill
+from plugin_eval.stats import (
+    bootstrap_ci,
+    clopper_pearson_ci,
+    coefficient_of_variation,
+    wilson_score_ci,
+)
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SimResult:
+    """Result of a single simulation run."""
+
+    activated: bool
+    quality_score: float
+    tokens: int
+    duration_ms: int
+    errored: bool = False
+    prompt: str = ""
+
+
+@dataclass
+class MonteCarloConfig:
+    """Configuration for Monte Carlo simulation runs."""
+
+    n_runs: int = 50
+    concurrency: int = 4
+    auth: str = "max"
+    seed: int = 42
+    progress_callback: Callable[[int, int], None] | None = None
+
+
+# ---------------------------------------------------------------------------
+# Single simulation runner
+# ---------------------------------------------------------------------------
+
+
+async def run_simulation(skill_content: str, prompt: str, auth: str) -> SimResult:
+    """Run a single simulation via Agent SDK. Returns SimResult. On error, errored=True."""
+    try:
+        import claude_agent_sdk as sdk  # type: ignore[import-untyped]
+
+        result_text = ""
+        activated = False
+        tokens = 0
+
+        import time
+
+        start = time.monotonic()
+
+        async for event in sdk.stream(
+            prompt,
+            system=f"You are evaluating a skill. Apply the skill if appropriate.\n\n{skill_content}",
+        ):
+            if hasattr(event, "text"):
+                result_text += event.text
+                activated = True
+            if hasattr(event, "usage"):
+                tokens = getattr(event.usage, "total_tokens", 0)
+
+        duration_ms = int((time.monotonic() - start) * 1000)
+
+        # Estimate quality score from response length and coherence heuristic
+        quality_score = min(1.0, len(result_text) / 500) if activated else 0.0
+
+        return SimResult(
+            activated=activated,
+            quality_score=quality_score,
+            tokens=tokens,
+            duration_ms=duration_ms,
+            prompt=prompt,
+        )
+
+    except Exception:
+        return SimResult(
+            activated=False,
+            quality_score=0.0,
+            tokens=0,
+            duration_ms=0,
+            errored=True,
+            prompt=prompt,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Analyzer
+# ---------------------------------------------------------------------------
+
+
+class MonteCarloAnalyzer:
+    """Statistical reliability testing via repeated simulated runs."""
+
+    def __init__(self, config: MonteCarloConfig) -> None:
+        self.config = config
+        self._sem = asyncio.Semaphore(config.concurrency)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    async def analyze_skill(self, skill_or_dir: Path | ParsedSkill) -> LayerResult:
+        """Generate prompts, run N simulations, compute statistics, return LayerResult."""
+        skill = skill_or_dir if isinstance(skill_or_dir, ParsedSkill) else parse_skill(skill_or_dir)
+        skill_content = skill.raw_content
+
+        prompts = await self._generate_prompts(skill.name, skill.description)
+
+        # Repeat prompts to reach n_runs
+        repeated: list[str] = []
+        while len(repeated) < self.config.n_runs:
+            repeated.extend(prompts)
+        prompts_to_run = repeated[: self.config.n_runs]
+
+        results = await self._run_all(skill_content, prompts_to_run)
+        stats = self._compute_statistics(results)
+
+        triggering = stats["triggering"]
+        output_consistency = stats["output_consistency"]
+        failure_rate = stats["failure_rate"]
+        token_efficiency = stats["token_efficiency"]
+
+        activation_rate = triggering.get("activation_rate", 0.0)
+        cv = output_consistency.get("cv", 1.0)
+        p_fail = failure_rate.get("p_fail", 1.0)
+        efficiency_norm = token_efficiency.get("efficiency_norm", 0.0)
+
+        # Composite: 0.40 * trigger_reliability + 0.30 * (1-cv) + 0.20 * (1-p_fail) + 0.10 * efficiency_norm
+        score = (
+            0.40 * activation_rate
+            + 0.30 * (1.0 - min(1.0, cv))
+            + 0.20 * (1.0 - p_fail)
+            + 0.10 * efficiency_norm
+        )
+        score = max(0.0, min(1.0, score))
+
+        sub_scores: dict = {
+            "triggering": triggering,
+            "output_consistency": output_consistency,
+            "failure_rate": failure_rate,
+            "token_efficiency": token_efficiency,
+        }
+
+        metadata: dict = {
+            "n_runs": len(results),
+            "n_activated": sum(1 for r in results if r.activated),
+            "n_errored": sum(1 for r in results if r.errored),
+        }
+
+        return LayerResult(
+            layer="monte_carlo",
+            score=score,
+            sub_scores=sub_scores,
+            metadata=metadata,
+        )
+
+    # ------------------------------------------------------------------
+    # Prompt generation
+    # ------------------------------------------------------------------
+
+    async def _generate_prompts(self, name: str, description: str) -> list[str]:
+        """Use Haiku to generate 15 varied prompts. Falls back to basic variants."""
+        try:
+            from plugin_eval.layers.judge import query_llm
+
+            model = "claude-haiku-4-5-20251001"
+            system = (
+                "You are generating test prompts for a Claude Code skill. "
+                "Respond ONLY with a JSON array of strings — no explanation, no markdown fences."
+            )
+            prompt = (
+                f"Generate 15 varied user prompts that would trigger this skill:\n\n"
+                f"Name: {name}\n"
+                f"Description: {description}\n\n"
+                f'Return a JSON array of 15 strings. Example: ["prompt 1", "prompt 2", ...]'
+            )
+            result = await query_llm(prompt, system=system, model=model)
+            if isinstance(result, list) and len(result) >= 5:
+                return [str(p) for p in result[:15]]
+        except Exception:
+            pass
+
+        # Fallback: generate basic variants from description
+        return self._fallback_prompts(name, description)
+
+    def _fallback_prompts(self, name: str, description: str) -> list[str]:
+        """Generate basic prompt variants when LLM is unavailable."""
+        base = description.split(".")[0].strip() if description else name
+        variants = [
+            f"Please help me with {name}.",
+            f"I need to {base.lower()}.",
+            f"Can you {base.lower()}?",
+            f"Help me {base.lower()}.",
+            f"Run {name} for me.",
+            f"Use {name} to help with my task.",
+            f"I want to {base.lower()}.",
+            f"Execute {name}.",
+            f"Apply {name}.",
+            f"Invoke {name} now.",
+            f"Start {name}.",
+            f"Please execute {name} on this.",
+            f"I'd like to use {name}.",
+            f"Trigger {name}.",
+            f"Activate {name} for this request.",
+        ]
+        return variants[:15]
+
+    # ------------------------------------------------------------------
+    # Running simulations
+    # ------------------------------------------------------------------
+
+    async def _run_all(self, skill_content: str, prompts: list[str]) -> list[SimResult]:
+        """Run all simulations with semaphore throttling."""
+        completed = 0
+        total = len(prompts)
+
+        async def run_one(prompt: str) -> SimResult:
+            nonlocal completed
+            async with self._sem:
+                result = await run_simulation(skill_content, prompt, self.config.auth)
+                completed += 1
+                if self.config.progress_callback:
+                    self.config.progress_callback(completed, total)
+                return result
+
+        tasks = [run_one(p) for p in prompts]
+        return list(await asyncio.gather(*tasks))
+
+    # ------------------------------------------------------------------
+    # Statistical analysis
+    # ------------------------------------------------------------------
+
+    def _compute_statistics(self, results: list[SimResult]) -> dict:
+        """Compute statistical measures from simulation results."""
+        n = len(results)
+        if n == 0:
+            return {
+                "triggering": {"activation_rate": 0.0},
+                "output_consistency": {"mean_quality": 0.0, "std_dev": 0.0, "cv": 1.0},
+                "failure_rate": {"p_fail": 1.0},
+                "token_efficiency": {"median": 0, "efficiency_norm": 0.0},
+            }
+
+        # --- Triggering ---
+        n_activated = sum(1 for r in results if r.activated)
+        activation_rate = n_activated / n
+        wi_lower, wi_upper = wilson_score_ci(n_activated, n)
+
+        triggering: dict = {
+            "activation_rate": activation_rate,
+            "wilson_lower": wi_lower,
+            "wilson_upper": wi_upper,
+            "n_activated": n_activated,
+            "n_total": n,
+        }
+
+        # --- Output consistency (non-errored, activated runs) ---
+        quality_scores = [r.quality_score for r in results if r.activated and not r.errored]
+        if quality_scores:
+            mean_quality = statistics.mean(quality_scores)
+            std_dev = statistics.pstdev(quality_scores) if len(quality_scores) > 1 else 0.0
+            cv = coefficient_of_variation(quality_scores) if len(quality_scores) > 1 else 0.0
+            if len(quality_scores) > 1:
+                bs_lower, bs_upper = bootstrap_ci(quality_scores, seed=self.config.seed)
+            else:
+                bs_lower, bs_upper = quality_scores[0], quality_scores[0]
+        else:
+            mean_quality = 0.0
+            std_dev = 0.0
+            cv = 0.0
+            bs_lower, bs_upper = 0.0, 0.0
+
+        output_consistency: dict = {
+            "mean_quality": mean_quality,
+            "std_dev": std_dev,
+            "cv": cv,
+            "bootstrap_lower": bs_lower,
+            "bootstrap_upper": bs_upper,
+        }
+
+        # --- Failure rate ---
+        n_errored = sum(1 for r in results if r.errored)
+        p_fail = n_errored / n
+        cp_lower, cp_upper = clopper_pearson_ci(n_errored, n)
+
+        failure_rate: dict = {
+            "p_fail": p_fail,
+            "cp_lower": cp_lower,
+            "cp_upper": cp_upper,
+            "n_errored": n_errored,
+        }
+
+        # --- Token efficiency ---
+        all_tokens = [r.tokens for r in results if not r.errored]
+        if all_tokens:
+            sorted_tokens = sorted(all_tokens)
+            mid = len(sorted_tokens) // 2
+            if len(sorted_tokens) % 2 == 0:
+                median_tokens = (sorted_tokens[mid - 1] + sorted_tokens[mid]) / 2
+            else:
+                median_tokens = float(sorted_tokens[mid])
+
+            q1_idx = len(sorted_tokens) // 4
+            q3_idx = (3 * len(sorted_tokens)) // 4
+            q1 = float(sorted_tokens[q1_idx])
+            q3 = float(sorted_tokens[min(q3_idx, len(sorted_tokens) - 1)])
+            iqr = q3 - q1
+
+            # Count outliers: tokens > Q3 + 1.5*IQR
+            outlier_threshold = q3 + 1.5 * iqr
+            n_outliers = sum(1 for t in all_tokens if t > outlier_threshold)
+
+            # Normalize efficiency: lower tokens = better, cap at 8000 tokens
+            TOKEN_CAP = 8000.0
+            efficiency_norm = max(0.0, 1.0 - median_tokens / TOKEN_CAP)
+        else:
+            median_tokens = 0.0
+            iqr = 0.0
+            n_outliers = 0
+            efficiency_norm = 0.0
+
+        token_efficiency: dict = {
+            "median": median_tokens,
+            "iqr": iqr,
+            "outlier_count": n_outliers,
+            "efficiency_norm": efficiency_norm,
+        }
+
+        return {
+            "triggering": triggering,
+            "output_consistency": output_consistency,
+            "failure_rate": failure_rate,
+            "token_efficiency": token_efficiency,
+        }