@bastani/atomic 0.6.8-0 → 0.7.0-1

package/.agents/skills/advanced-evaluation/scripts/evaluation_example.py

@@ -1,392 +0,0 @@
-"""Advanced Evaluation Example
-
-Use when: building LLM-as-judge evaluation pipelines, comparing model outputs
-with position-bias mitigation, or generating domain-specific scoring rubrics.
-
-This module demonstrates the three core evaluation patterns from the
-advanced-evaluation skill: direct scoring, pairwise comparison with position
-swapping, and rubric generation. All functions use pseudocode-style examples
-that work across Python environments without specific dependencies.
-"""
-
-from __future__ import annotations
-
-from typing import Any
-
-__all__ = [
-    "direct_scoring_example",
-    "pairwise_comparison_example",
-    "rubric_generation_example",
-]
-
-
-# =============================================================================
-# DIRECT SCORING EXAMPLE
-# =============================================================================
-
-
-def direct_scoring_example() -> dict[str, Any]:
-    """Rate a single response against defined criteria using direct scoring.
-
-    Use when: evaluating objective criteria like factual accuracy, instruction
-    following, or toxicity where a clear ground truth or rubric exists.
-
-    Returns:
-        Dictionary containing per-criterion scores, evidence, justifications,
-        and a weighted summary.
-    """
-
-    # Input
-    prompt: str = "Explain quantum entanglement to a high school student"
-    response: str = (
-        "Quantum entanglement is like having two magical coins that are connected. "
-        "When you flip one and it lands on heads, the other instantly shows tails, "
-        'no matter how far apart they are. Scientists call this "spooky action at a distance."'
-    )
-
-    criteria: list[dict[str, Any]] = [
-        {"name": "Accuracy", "description": "Scientific correctness", "weight": 0.4},
-        {"name": "Clarity", "description": "Understandable for audience", "weight": 0.3},
-        {"name": "Engagement", "description": "Interesting and memorable", "weight": 0.3},
-    ]
-
-    # System prompt for the evaluator
-    system_prompt: str = (
-        "You are an expert evaluator. Assess the response against each criterion.\n\n"
-        "For each criterion:\n"
-        "1. Find specific evidence in the response\n"
-        "2. Score according to the rubric (1-5 scale)\n"
-        "3. Justify your score with evidence\n"
-        "4. Suggest one specific improvement\n\n"
-        "Be objective and consistent. Base scores on explicit evidence."
-    )
-
-    # User prompt structure
-    user_prompt: str = f"""## Original Prompt
-{prompt}
-
-## Response to Evaluate
-{response}
-
-## Criteria
-1. **Accuracy** (weight: 0.4): Scientific correctness
-2. **Clarity** (weight: 0.3): Understandable for audience
-3. **Engagement** (weight: 0.3): Interesting and memorable
-
-## Output Format
-Respond with valid JSON:
-{{
-  "scores": [
-    {{
-      "criterion": "Accuracy",
-      "score": 4,
-      "evidence": ["quote or observation"],
-      "justification": "why this score",
-      "improvement": "specific suggestion"
-    }}
-  ],
-  "summary": {{
-    "assessment": "overall quality summary",
-    "strengths": ["strength 1"],
-    "weaknesses": ["weakness 1"]
-  }}
-}}"""
-
-    # Expected output structure
-    expected_output: dict[str, Any] = {
-        "scores": [
-            {
-                "criterion": "Accuracy",
-                "score": 4,
-                "evidence": ["Correctly uses analogy", "Mentions spooky action at a distance"],
-                "justification": "Core concept is correct, analogy is appropriate",
-                "improvement": "Could mention it's a quantum mechanical phenomenon",
-            },
-            {
-                "criterion": "Clarity",
-                "score": 5,
-                "evidence": ["Simple coin analogy", "No jargon"],
-                "justification": "Appropriate for high school level",
-                "improvement": "None needed",
-            },
-            {
-                "criterion": "Engagement",
-                "score": 4,
-                "evidence": ["Magical coins", "Spooky action quote"],
-                "justification": "Memorable imagery and Einstein quote",
-                "improvement": "Could add a real-world application",
-            },
-        ],
-        "summary": {
-            "assessment": "Good explanation suitable for the target audience",
-            "strengths": ["Clear analogy", "Age-appropriate language"],
-            "weaknesses": ["Could be more comprehensive"],
-        },
-    }
-
-    # Calculate weighted score
-    total_weight: float = sum(c["weight"] for c in criteria)
-    weighted_score: float = sum(
-        s["score"] * next(c["weight"] for c in criteria if c["name"] == s["criterion"])
-        for s in expected_output["scores"]
-    ) / total_weight
-
-    print(f"Weighted Score: {weighted_score:.2f}/5")
-    return expected_output
-
-
-# =============================================================================
-# PAIRWISE COMPARISON WITH POSITION BIAS MITIGATION
-# =============================================================================
-
-
-def pairwise_comparison_example() -> dict[str, Any]:
-    """Compare two responses with position-swapped bias mitigation.
-
-    Use when: evaluating subjective preferences like tone, style, or
-    persuasiveness where pairwise comparison achieves higher human-judge
-    agreement than direct scoring.
-
-    Returns:
-        Dictionary containing the winner, confidence score, and whether
-        position consistency was achieved across both passes.
-    """
-
-    prompt: str = "Explain machine learning to a beginner"
-
-    response_a: str = (
-        "Machine learning is a subset of artificial intelligence that enables "
-        "systems to learn and improve from experience without being explicitly "
-        "programmed. It uses statistical techniques to give computers the ability "
-        "to identify patterns in data."
-    )
-
-    response_b: str = (
-        "Imagine teaching a dog a new trick. You show the dog what to do, give "
-        "treats when it's right, and eventually it learns. Machine learning works "
-        "similarly - we show computers lots of examples, tell them when they're "
-        "right, and they learn to recognize patterns on their own."
-    )
-
-    criteria: list[str] = ["clarity", "accessibility", "accuracy"]
-
-    # System prompt emphasizing bias awareness
-    system_prompt: str = (
-        "You are an expert evaluator comparing two AI responses.\n\n"
-        "CRITICAL INSTRUCTIONS:\n"
-        "- Do NOT prefer responses because they are longer\n"
-        "- Do NOT prefer responses based on position (first vs second)\n"
-        "- Focus ONLY on quality according to the specified criteria\n"
-        "- Ties are acceptable when responses are genuinely equivalent"
-    )
-
-    # Build evaluation prompt for a given ordering
-    def evaluate_pass(
-        first_response: str,
-        second_response: str,
-        first_label: str,
-        second_label: str,
-    ) -> str:
-        """Build evaluation prompt for one pass of position-swapped comparison.
-
-        Use when: constructing the prompt for a single evaluation pass before
-        swapping response positions for bias mitigation.
-        """
-        return f"""## Original Prompt
-{prompt}
-
-## Response {first_label}
-{first_response}
-
-## Response {second_label}
-{second_response}
-
-## Comparison Criteria
-{', '.join(criteria)}
-
-## Output Format
-{{
-  "comparison": [
-    {{"criterion": "clarity", "winner": "A|B|TIE", "reasoning": "..."}}
-  ],
-  "result": {{
-    "winner": "A|B|TIE",
-    "confidence": 0.0-1.0,
-    "reasoning": "overall reasoning"
-  }}
-}}"""
-
-    # Position bias mitigation protocol
-    print("Pass 1: A in first position")
-    pass1_result: dict[str, Any] = {"winner": "B", "confidence": 0.8}
-
-    print("Pass 2: B in first position (swapped)")
-    pass2_result: dict[str, Any] = {"winner": "A", "confidence": 0.75}  # A because B was first
-
-    # Map pass2 result back (swap labels)
-    def map_winner(winner: str) -> str:
-        """Map winner label after position swap."""
-        return {"A": "B", "B": "A", "TIE": "TIE"}[winner]
-
-    pass2_mapped: str = map_winner(pass2_result["winner"])
-    print(f"Pass 2 mapped winner: {pass2_mapped}")
-
-    # Check consistency
-    consistent: bool = pass1_result["winner"] == pass2_mapped
-
-    final_result: dict[str, Any]
-    if consistent:
-        final_result = {
-            "winner": pass1_result["winner"],
-            "confidence": (pass1_result["confidence"] + pass2_result["confidence"]) / 2,
-            "position_consistent": True,
-        }
-    else:
-        final_result = {
-            "winner": "TIE",
-            "confidence": 0.5,
-            "position_consistent": False,
-            "bias_detected": True,
-        }
-
-    print(f"\nFinal Result: {final_result}")
-    return final_result
-
-
-# =============================================================================
-# RUBRIC GENERATION
-# =============================================================================
-
-
-def rubric_generation_example() -> dict[str, Any]:
-    """Generate a domain-specific scoring rubric for consistent evaluation.
-
-    Use when: establishing evaluation standards for a new criterion, reducing
-    scoring variance (rubrics cut variance by 40-60%), or onboarding new
-    evaluators to an existing evaluation pipeline.
-
-    Returns:
-        Dictionary containing score levels, characteristics, examples,
-        scoring guidelines, and edge case handling.
-    """
-
-    criterion_name: str = "Code Readability"
-    criterion_description: str = "How easy the code is to understand and maintain"
-    domain: str = "software engineering"
-    scale: str = "1-5"
-    strictness: str = "balanced"
-
-    system_prompt: str = (
-        f"You are an expert in creating evaluation rubrics.\n"
-        f"Create clear, actionable rubrics with distinct boundaries between levels.\n\n"
-        f"Strictness: {strictness}\n"
-        f"- lenient: Lower bar for passing scores\n"
-        f"- balanced: Fair, typical expectations\n"
-        f"- strict: High standards, critical evaluation"
-    )
-
-    user_prompt: str = f"""Create a scoring rubric for:
-
-**Criterion**: {criterion_name}
-**Description**: {criterion_description}
-**Scale**: {scale}
-**Domain**: {domain}
-
-Generate:
-1. Clear descriptions for each score level
-2. Specific characteristics that define each level
-3. Brief example text for each level
-4. General scoring guidelines
-5. Edge cases with guidance"""
-
-    # Expected rubric structure
-    rubric: dict[str, Any] = {
-        "criterion": criterion_name,
-        "scale": {"min": 1, "max": 5},
-        "levels": [
-            {
-                "score": 1,
-                "label": "Poor",
-                "description": "Code is difficult to understand without significant effort",
-                "characteristics": [
-                    "No meaningful variable or function names",
-                    "No comments or documentation",
-                    "Deeply nested or convoluted logic",
-                ],
-                "example": "def f(x): return x[0]*x[1]+x[2]",
-            },
-            {
-                "score": 3,
-                "label": "Adequate",
-                "description": "Code is understandable with some effort",
-                "characteristics": [
-                    "Most variables have meaningful names",
-                    "Basic comments for complex sections",
-                    "Logic is followable but could be cleaner",
-                ],
-                "example": (
-                    "def calc_total(items): # calculate sum\n"
-                    "    total = 0\n"
-                    "    for i in items: total += i\n"
-                    "    return total"
-                ),
-            },
-            {
-                "score": 5,
-                "label": "Excellent",
-                "description": "Code is immediately clear and maintainable",
-                "characteristics": [
-                    "All names are descriptive and consistent",
-                    "Comprehensive documentation",
-                    "Clean, modular structure",
-                ],
-                "example": (
-                    "def calculate_total_price(items: List[Item]) -> Decimal:\n"
-                    "    '''Calculate the total price of all items.'''\n"
-                    "    return sum(item.price for item in items)"
-                ),
-            },
-        ],
-        "scoring_guidelines": [
-            "Focus on readability, not cleverness",
-            "Consider the intended audience (team skill level)",
-            "Consistency matters more than style preference",
-        ],
-        "edge_cases": [
-            {
-                "situation": "Code uses domain-specific abbreviations",
-                "guidance": "Score based on readability for domain experts, not general audience",
-            },
-            {
-                "situation": "Code is auto-generated",
-                "guidance": "Apply same standards but note in evaluation",
-            },
-        ],
-    }
-
-    print("Generated Rubric:")
-    for level in rubric["levels"]:
-        print(f"  {level['score']}: {level['label']} - {level['description']}")
-
-    return rubric
-
-
-# =============================================================================
-# MAIN
-# =============================================================================
-
-if __name__ == "__main__":
-    print("=" * 60)
-    print("DIRECT SCORING EXAMPLE")
-    print("=" * 60)
-    direct_scoring_example()
-
-    print("\n" + "=" * 60)
-    print("PAIRWISE COMPARISON EXAMPLE")
-    print("=" * 60)
-    pairwise_comparison_example()
-
-    print("\n" + "=" * 60)
-    print("RUBRIC GENERATION EXAMPLE")
-    print("=" * 60)
-    rubric_generation_example()