@patricio0312rev/skillset 0.1.0

package/templates/ai-engineering/llm-debugger/ SKILL.md

@@ -0,0 +1,283 @@
+---
+name: llm-debugger
+description: Diagnoses LLM output failures including hallucinations, constraint violations, format errors, and reasoning issues. Provides root cause classification, prompt fixes, tool improvements, and new test cases. Use for "debugging AI", "fixing prompts", "quality issues", or "output errors".
+---
+
+# LLM Debugger
+
+Systematically diagnose and fix LLM output issues.
+
+## Failure Taxonomy
+
+```python
+class FailureType(Enum):
+    HALLUCINATION = "hallucination"
+    FORMAT_VIOLATION = "format_violation"
+    CONSTRAINT_BREAK = "constraint_break"
+    REASONING_ERROR = "reasoning_error"
+    TOOL_MISUSE = "tool_misuse"
+    REFUSAL = "unexpected_refusal"
+    INCOMPLETE = "incomplete_output"
+```
+
+## Root Cause Analysis
+
+```python
+def diagnose_failure(input: str, output: str, expected: dict) -> dict:
+    """Identify why LLM output failed"""
+
+    issues = []
+
+    # Check format
+    if expected.get("format") == "json":
+        try:
+            json.loads(output)
+        except:
+            issues.append({
+                "type": FailureType.FORMAT_VIOLATION,
+                "details": "Invalid JSON output"
+            })
+
+    # Check required fields
+    if expected.get("required_fields"):
+        for field in expected["required_fields"]:
+            if field not in output:
+                issues.append({
+                    "type": FailureType.INCOMPLETE,
+                    "details": f"Missing required field: {field}"
+                })
+
+    # Check constraints
+    if expected.get("max_length"):
+        if len(output) > expected["max_length"]:
+            issues.append({
+                "type": FailureType.CONSTRAINT_BREAK,
+                "details": f"Output too long: {len(output)} > {expected['max_length']}"
+            })
+
+    # Check for hallucination indicators
+    if contains_hallucination_markers(output):
+        issues.append({
+            "type": FailureType.HALLUCINATION,
+            "details": "Contains fabricated information"
+        })
+
+    return {
+        "has_issues": len(issues) > 0,
+        "issues": issues,
+        "primary_issue": issues[0] if issues else None
+    }
+
+def contains_hallucination_markers(output: str) -> bool:
+    """Detect common hallucination patterns"""
+    markers = [
+        r'According to.*that doesn\'t exist',
+        r'In \d{4}.*before that year',
+        r'contradicts itself',
+    ]
+    return any(re.search(marker, output, re.IGNORECASE) for marker in markers)
+```
+
+## Prompt Fixes
+
+````python
+PROMPT_FIXES = {
+    FailureType.FORMAT_VIOLATION: """
+Add strict format instructions:
+"Output MUST be valid JSON with this exact structure:
+```json
+{{"field1": "value", "field2": 123}}
+````
+
+Do not add any text before or after the JSON."
+""",
+
+    FailureType.HALLUCINATION: """
+
+Add grounding instructions:
+"Base your response ONLY on the provided context.
+If information is not in the context, say 'I don't have that information.'
+Never make up facts or details."
+""",
+
+    FailureType.CONSTRAINT_BREAK: """
+
+Add explicit constraints:
+"Your response must be:
+
+- Maximum 200 words
+- No code examples
+- Professional tone only"
+  """,
+      FailureType.REASONING_ERROR: """
+  Add step-by-step reasoning:
+  "Think through this step by step:
+
+1. First, identify...
+2. Then, consider...
+3. Finally, conclude..."
+   """,
+   }
+
+def suggest_prompt_fix(diagnosis: dict, current_prompt: str) -> str:
+"""Generate improved prompt based on diagnosis"""
+primary_issue = diagnosis["primary_issue"]
+if not primary_issue:
+return current_prompt
+
+    fix = PROMPT_FIXES[primary_issue["type"]]
+
+    return f"""{current_prompt}
+
+IMPORTANT INSTRUCTIONS:
+{fix}
+"""
+
+````
+
+## Tool Improvements
+
+```python
+def suggest_tool_fixes(diagnosis: dict, tool_schema: dict) -> dict:
+    """Improve tool schema to prevent misuse"""
+
+    fixes = {}
+
+    for issue in diagnosis["issues"]:
+        if issue["type"] == FailureType.TOOL_MISUSE:
+            # Make required parameters more explicit
+            fixes["parameters"] = {
+                **tool_schema["parameters"],
+                "description": "REQUIRED: " + tool_schema["parameters"].get("description", "")
+            }
+
+            # Add examples
+            fixes["examples"] = [
+                {
+                    "params": {"query": "example search"},
+                    "description": "Use for finding information"
+                }
+            ]
+
+    return {**tool_schema, **fixes}
+````
+
+## Test Case Generation
+
+```python
+def generate_test_cases(diagnosis: dict, failed_input: str, failed_output: str) -> List[dict]:
+    """Create regression test cases from failures"""
+
+    test_cases = []
+
+    # Test case for the specific failure
+    test_cases.append({
+        "input": failed_input,
+        "expected_behavior": "Should not " + diagnosis["primary_issue"]["details"],
+        "validation": lambda output: not has_same_issue(output, diagnosis),
+    })
+
+    # Edge cases based on failure type
+    if diagnosis["primary_issue"]["type"] == FailureType.FORMAT_VIOLATION:
+        test_cases.append({
+            "input": failed_input,
+            "validation": lambda output: is_valid_json(output),
+            "description": "Output must be valid JSON"
+        })
+
+    # Similar inputs that might trigger same issue
+    similar_inputs = generate_similar_inputs(failed_input)
+    for inp in similar_inputs:
+        test_cases.append({
+            "input": inp,
+            "validation": lambda output: not has_same_issue(output, diagnosis),
+        })
+
+    return test_cases
+```
+
+## Debugging Workflow
+
+```python
+def debug_llm_output(
+    prompt: str,
+    output: str,
+    expected: dict,
+    context: dict = {}
+) -> dict:
+    """Complete debugging workflow"""
+
+    # 1. Diagnose issue
+    diagnosis = diagnose_failure(prompt, output, expected)
+
+    if not diagnosis["has_issues"]:
+        return {"status": "ok", "diagnosis": diagnosis}
+
+    # 2. Generate fixes
+    fixed_prompt = suggest_prompt_fix(diagnosis, prompt)
+
+    # 3. Create test cases
+    test_cases = generate_test_cases(diagnosis, prompt, output)
+
+    # 4. Test fix
+    test_output = llm(fixed_prompt)
+    fix_works = diagnose_failure(fixed_prompt, test_output, expected)["has_issues"] == False
+
+    return {
+        "status": "issues_found",
+        "diagnosis": diagnosis,
+        "fixes": {
+            "prompt": fixed_prompt,
+            "fix_verified": fix_works,
+        },
+        "test_cases": test_cases,
+        "recommendations": generate_recommendations(diagnosis, context)
+    }
+```
+
+## Interactive Debugging
+
+```python
+def interactive_debug():
+    """Interactive debugging session"""
+    print("LLM Debugger - Interactive Mode")
+
+    prompt = input("Enter prompt: ")
+    expected_output = input("Expected output format (json/text): ")
+
+    output = llm(prompt)
+    print(f"\nGenerated output:\n{output}\n")
+
+    if input("Does this look correct? (y/n): ").lower() == 'n':
+        diagnosis = diagnose_failure(prompt, output, {"format": expected_output})
+
+        print("\nDiagnosis:")
+        for issue in diagnosis["issues"]:
+            print(f"- {issue['type'].value}: {issue['details']}")
+
+        print("\nSuggested fix:")
+        print(suggest_prompt_fix(diagnosis, prompt))
+```
+
+## Best Practices
+
+1. **Reproduce consistently**: Multiple runs
+2. **Isolate variables**: Test prompt changes one at a time
+3. **Check examples**: Are few-shot examples clear?
+4. **Validate constraints**: Are they explicit enough?
+5. **Test edge cases**: Boundary conditions
+6. **Log everything**: Inputs, outputs, issues
+7. **Iterate systematically**: Track what fixes work
+
+## Output Checklist
+
+- [ ] Failure taxonomy defined
+- [ ] Root cause analyzer
+- [ ] Prompt fix suggestions
+- [ ] Tool schema improvements
+- [ ] Test case generator
+- [ ] Fix verification
+- [ ] Debugging workflow
+- [ ] Recommendations engine
+- [ ] Interactive debugger
+- [ ] Logging system

package/templates/ai-engineering/prompt-regression-tester/ SKILL.md

@@ -0,0 +1,216 @@
+---
+name: prompt-regression-tester
+description: Compares old vs new prompts across test cases with diff summaries, stability metrics, breakage analysis, and fix suggestions. Use for "prompt testing", "A/B testing prompts", "prompt versioning", or "quality regression".
+---
+
+# Prompt Regression Tester
+
+Systematically test prompt changes to prevent regressions.
+
+## Test Case Format
+
+```json
+{
+  "test_cases": [
+    {
+      "id": "test_001",
+      "input": "Summarize this article",
+      "context": "Article text here...",
+      "expected_behavior": "Concise 2-3 sentence summary",
+      "baseline_output": "Output from v1.0 prompt",
+      "must_include": ["main point", "conclusion"],
+      "must_not_include": ["opinion", "speculation"]
+    }
+  ]
+}
+```
+
+## Comparison Framework
+
+```python
+def compare_prompts(old_prompt, new_prompt, test_cases):
+    results = {
+        "test_cases": [],
+        "summary": {
+            "total": len(test_cases),
+            "improvements": 0,
+            "regressions": 0,
+            "unchanged": 0,
+        },
+        "breakages": []
+    }
+
+    for test in test_cases:
+        old_output = llm(old_prompt.format(**test))
+        new_output = llm(new_prompt.format(**test))
+
+        comparison = {
+            "test_id": test["id"],
+            "old_output": old_output,
+            "new_output": new_output,
+            "diff": compute_diff(old_output, new_output),
+            "scores": {
+                "old": score_output(old_output, test),
+                "new": score_output(new_output, test),
+            },
+            "verdict": classify_change(old_output, new_output, test)
+        }
+
+        results["test_cases"].append(comparison)
+        results["summary"][comparison["verdict"]] += 1
+
+        if comparison["verdict"] == "regressions":
+            results["breakages"].append(analyze_breakage(comparison, test))
+
+    return results
+```
+
+## Stability Metrics
+
+```python
+def calculate_stability_metrics(results):
+    return {
+        "output_stability": measure_output_consistency(results),
+        "format_stability": check_format_preservation(results),
+        "constraint_adherence": check_constraints(results),
+        "behavioral_consistency": measure_behavior_delta(results),
+    }
+
+def measure_output_consistency(results):
+    """How similar are outputs between versions?"""
+    similarities = []
+    for result in results["test_cases"]:
+        sim = semantic_similarity(
+            result["old_output"],
+            result["new_output"]
+        )
+        similarities.append(sim)
+    return sum(similarities) / len(similarities)
+```
+
+## Breakage Analysis
+
+```python
+def analyze_breakage(comparison, test_case):
+    """Identify why the new prompt failed"""
+    reasons = []
+
+    new_out = comparison["new_output"]
+
+    # Missing required content
+    for keyword in test_case.get("must_include", []):
+        if keyword.lower() not in new_out.lower():
+            reasons.append(f"Missing required content: '{keyword}'")
+
+    # Contains forbidden content
+    for keyword in test_case.get("must_not_include", []):
+        if keyword.lower() in new_out.lower():
+            reasons.append(f"Contains forbidden content: '{keyword}'")
+
+    # Format violations
+    if not check_format(new_out, test_case.get("expected_format")):
+        reasons.append("Output format violation")
+
+    # Length issues
+    expected_length = test_case.get("expected_length")
+    if expected_length:
+        actual_length = len(new_out.split())
+        if abs(actual_length - expected_length) > expected_length * 0.3:
+            reasons.append(f"Length deviation: expected ~{expected_length}, got {actual_length}")
+
+    return {
+        "test_id": test_case["id"],
+        "reasons": reasons,
+        "old_output": comparison["old_output"][:100],
+        "new_output": comparison["new_output"][:100],
+    }
+```
+
+## Fix Suggestions
+
+```python
+def suggest_fixes(breakages):
+    """Generate fix suggestions based on breakage patterns"""
+    suggestions = []
+
+    # Group breakages by reason
+    reason_groups = {}
+    for breakage in breakages:
+        for reason in breakage["reasons"]:
+            if reason not in reason_groups:
+                reason_groups[reason] = []
+            reason_groups[reason].append(breakage["test_id"])
+
+    # Generate suggestions
+    for reason, test_ids in reason_groups.items():
+        if "Missing required content" in reason:
+            suggestions.append({
+                "issue": reason,
+                "affected_tests": test_ids,
+                "suggestion": "Add explicit instruction in prompt to include this content",
+                "example": f"Make sure to mention {reason.split(':')[1]} in your response."
+            })
+        elif "format violation" in reason:
+            suggestions.append({
+                "issue": reason,
+                "affected_tests": test_ids,
+                "suggestion": "Add stricter format constraints to prompt",
+                "example": "Output must follow this exact format: ..."
+            })
+
+    return suggestions
+```
+
+## Report Generation
+
+```markdown
+# Prompt Regression Report
+
+## Summary
+
+- **Total tests:** 50
+- **Improvements:** 5 (10%)
+- **Regressions:** 3 (6%)
+- **Unchanged:** 42 (84%)
+
+## Stability Metrics
+
+- **Output stability:** 0.87
+- **Format stability:** 0.95
+- **Constraint adherence:** 0.94
+
+## Regressions (3)
+
+### test_005: Missing required content
+
+**Old output:** "The main benefit is cost savings..."
+**New output:** "This approach provides flexibility..."
+**Issue:** Missing required keyword 'cost'
+**Fix:** Add explicit instruction: "Mention cost implications in your response"
+
+## Recommended Actions
+
+1. Revert changes that caused regressions (tests: 005, 012, 023)
+2. Add format constraints for JSON output
+3. Run full test suite before deployment
+```
+
+## Best Practices
+
+- Test with diverse inputs
+- Compare across multiple runs (LLMs are stochastic)
+- Track metrics over time
+- Automate in CI/CD
+- Review all regressions before deploy
+- Maintain test case library
+
+## Output Checklist
+
+- [ ] Test cases defined (30+)
+- [ ] Comparison runner
+- [ ] Stability metrics
+- [ ] Breakage analyzer
+- [ ] Fix suggestions
+- [ ] Diff visualizer
+- [ ] Automated report
+- [ ] CI integration