@patricio0312rev/skillset 0.1.0

package/templates/ai-engineering/evaluation-harness/ SKILL.md

@@ -0,0 +1,219 @@
+---
+name: evaluation-harness
+description: Builds repeatable evaluation systems with golden datasets, scoring rubrics, pass/fail thresholds, and regression reports. Use for "LLM evaluation", "testing AI systems", "quality assurance", or "model benchmarking".
+---
+
+# Evaluation Harness
+
+Build systematic evaluation frameworks for LLM applications.
+
+## Golden Dataset Format
+
+```json
+[
+  {
+    "id": "test_001",
+    "category": "code_generation",
+    "input": "Write a Python function to reverse a string",
+    "expected_output": "def reverse_string(s: str) -> str:\n    return s[::-1]",
+    "rubric": {
+      "correctness": 1.0,
+      "style": 0.8,
+      "documentation": 0.5
+    },
+    "metadata": {
+      "difficulty": "easy",
+      "tags": ["python", "strings"]
+    }
+  }
+]
+```
+
+## Scoring Rubrics
+
+```python
+from typing import Dict, Any
+
+def score_exact_match(actual: str, expected: str) -> float:
+    """Binary score: 1.0 if exact match, 0.0 otherwise"""
+    return 1.0 if actual.strip() == expected.strip() else 0.0
+
+def score_semantic_similarity(actual: str, expected: str) -> float:
+    """Cosine similarity of embeddings"""
+    actual_emb = get_embedding(actual)
+    expected_emb = get_embedding(expected)
+    return cosine_similarity(actual_emb, expected_emb)
+
+def score_contains_keywords(actual: str, keywords: List[str]) -> float:
+    """Percentage of required keywords present"""
+    found = sum(1 for kw in keywords if kw.lower() in actual.lower())
+    return found / len(keywords)
+
+def score_with_llm(actual: str, expected: str, rubric: Dict[str, float]) -> Dict[str, float]:
+    """Use LLM as judge"""
+    prompt = f"""
+    Grade this output on a scale of 0-1 for each criterion:
+
+    Expected: {expected}
+    Actual: {actual}
+
+    Criteria: {', '.join(rubric.keys())}
+
+    Return JSON with scores.
+    """
+    return json.loads(llm(prompt))
+```
+
+## Test Runner
+
+```python
+class EvaluationHarness:
+    def __init__(self, dataset_path: str):
+        self.dataset = self.load_dataset(dataset_path)
+        self.results = []
+
+    def run_evaluation(self, model_fn):
+        for test_case in self.dataset:
+            # Generate output
+            actual = model_fn(test_case["input"])
+
+            # Score
+            scores = self.score_output(
+                actual,
+                test_case["expected_output"],
+                test_case["rubric"]
+            )
+
+            # Record result
+            self.results.append({
+                "test_id": test_case["id"],
+                "category": test_case["category"],
+                "scores": scores,
+                "passed": self.check_threshold(scores, test_case),
+                "actual_output": actual,
+            })
+
+        return self.generate_report()
+
+    def score_output(self, actual, expected, rubric):
+        return {
+            "exact_match": score_exact_match(actual, expected),
+            "semantic_similarity": score_semantic_similarity(actual, expected),
+            **score_with_llm(actual, expected, rubric)
+        }
+
+    def check_threshold(self, scores, test_case):
+        min_scores = test_case.get("min_scores", {})
+        for metric, threshold in min_scores.items():
+            if scores.get(metric, 0) < threshold:
+                return False
+        return True
+```
+
+## Thresholds & Pass Criteria
+
+```python
+# Define thresholds per category
+THRESHOLDS = {
+    "code_generation": {
+        "correctness": 0.9,
+        "style": 0.7,
+    },
+    "summarization": {
+        "semantic_similarity": 0.8,
+        "brevity": 0.7,
+    },
+    "classification": {
+        "exact_match": 1.0,
+    }
+}
+
+def check_test_passed(result: Dict) -> bool:
+    category = result["category"]
+    thresholds = THRESHOLDS.get(category, {})
+
+    for metric, threshold in thresholds.items():
+        if result["scores"].get(metric, 0) < threshold:
+            return False
+
+    return True
+```
+
+## Regression Report
+
+```python
+def generate_regression_report(baseline_results, current_results):
+    report = {
+        "summary": {},
+        "regressions": [],
+        "improvements": [],
+        "unchanged": 0
+    }
+
+    for baseline, current in zip(baseline_results, current_results):
+        assert baseline["test_id"] == current["test_id"]
+
+        baseline_passed = baseline["passed"]
+        current_passed = current["passed"]
+
+        if baseline_passed and not current_passed:
+            report["regressions"].append({
+                "test_id": baseline["test_id"],
+                "category": baseline["category"],
+                "baseline_scores": baseline["scores"],
+                "current_scores": current["scores"],
+            })
+        elif not baseline_passed and current_passed:
+            report["improvements"].append(baseline["test_id"])
+        else:
+            report["unchanged"] += 1
+
+    report["summary"] = {
+        "total_tests": len(baseline_results),
+        "regressions": len(report["regressions"]),
+        "improvements": len(report["improvements"]),
+        "unchanged": report["unchanged"],
+    }
+
+    return report
+```
+
+## Continuous Evaluation
+
+```python
+# Run evaluation on every commit
+def ci_evaluation():
+    harness = EvaluationHarness("golden_dataset.json")
+    results = harness.run_evaluation(production_model)
+
+    # Check for regressions
+    baseline = load_baseline("baseline_results.json")
+    report = generate_regression_report(baseline, results)
+
+    # Fail CI if regressions
+    if report["summary"]["regressions"] > 0:
+        print(f"❌ {report['summary']['regressions']} regressions detected!")
+        sys.exit(1)
+
+    print("✅ All tests passed!")
+```
+
+## Best Practices
+
+1. **Representative dataset**: Cover edge cases
+2. **Multiple metrics**: Don't rely on one score
+3. **Human validation**: Review LLM judge scores
+4. **Version datasets**: Track changes over time
+5. **Automate in CI**: Catch regressions early
+6. **Regular updates**: Add new test cases
+
+## Output Checklist
+
+- [ ] Golden dataset created (50+ examples)
+- [ ] Multiple scoring functions
+- [ ] Pass/fail thresholds defined
+- [ ] Test runner implemented
+- [ ] Regression comparison
+- [ ] Report generation
+- [ ] CI integration
+- [ ] Baseline established

package/templates/ai-engineering/guardrails-safety-filter-builder/ SKILL.md

@@ -0,0 +1,226 @@
+---
+name: guardrails-safety-filter-builder
+description: Implements content safety filters with PII redaction, policy constraints, prompt injection detection, and safe refusal templates. Use when adding "content moderation", "safety filters", "PII protection", or "guardrails".
+---
+
+# Guardrails & Safety Filter Builder
+
+Build comprehensive safety systems for LLM applications.
+
+## Safety Layers
+
+1. **Input filtering**: Block malicious prompts
+2. **Output filtering**: Redact sensitive data
+3. **Topic constraints**: Policy-based refusals
+4. **PII detection**: Mask personal information
+5. **Prompt injection**: Detect manipulation attempts
+
+## PII Detection & Redaction
+
+```python
+import re
+from presidio_analyzer import AnalyzerEngine
+from presidio_anonymizer import AnonymizerEngine
+
+analyzer = AnalyzerEngine()
+anonymizer = AnonymizerEngine()
+
+def redact_pii(text: str) -> str:
+    # Detect PII
+    results = analyzer.analyze(
+        text=text,
+        language='en',
+        entities=["EMAIL_ADDRESS", "PHONE_NUMBER", "CREDIT_CARD", "SSN"]
+    )
+
+    # Anonymize
+    anonymized = anonymizer.anonymize(text, results)
+    return anonymized.text
+
+# Example: "My email is john@example.com" → "My email is <EMAIL_ADDRESS>"
+```
+
+## Prompt Injection Detection
+
+```python
+def detect_prompt_injection(user_input: str) -> bool:
+    """Detect common prompt injection patterns"""
+    patterns = [
+        r'ignore (previous|above) instructions',
+        r'disregard (all|any) (prior|previous)',
+        r'you are now',
+        r'new instructions',
+        r'system:',
+        r'override',
+    ]
+
+    for pattern in patterns:
+        if re.search(pattern, user_input, re.IGNORECASE):
+            return True
+
+    return False
+
+# Block if detected
+if detect_prompt_injection(user_input):
+    return "I cannot process that request."
+```
+
+## Topic Constraints
+
+```python
+# Define allowed/disallowed topics
+POLICY = {
+    "allowed_topics": [
+        "product_features",
+        "troubleshooting",
+        "billing",
+        "account_management"
+    ],
+    "disallowed_topics": [
+        "medical_advice",
+        "legal_advice",
+        "financial_advice",
+        "politics",
+        "violence"
+    ],
+    "requires_disclaimer": [
+        "security_practices",
+        "data_privacy"
+    ]
+}
+
+# Classify topic
+def classify_topic(query: str) -> str:
+    classification_prompt = f"""
+    Classify this query into one of these topics:
+    {', '.join(POLICY['allowed_topics'] + POLICY['disallowed_topics'])}
+
+    Query: {query}
+
+    Return only the topic name.
+    """
+    return llm(classification_prompt)
+
+# Check policy
+def check_policy(query: str) -> dict:
+    topic = classify_topic(query)
+
+    if topic in POLICY["disallowed_topics"]:
+        return {
+            "allowed": False,
+            "reason": f"Cannot provide {topic}",
+            "refusal": REFUSAL_TEMPLATES[topic]
+        }
+
+    return {"allowed": True, "topic": topic}
+```
+
+## Refusal Templates
+
+```python
+REFUSAL_TEMPLATES = {
+    "medical_advice": """
+        I cannot provide medical advice. Please consult with a healthcare
+        professional for medical concerns.
+    """,
+    "legal_advice": """
+        I cannot provide legal advice. For legal matters, please consult
+        with a qualified attorney.
+    """,
+    "out_of_scope": """
+        I'm designed to help with product documentation and support.
+        This question is outside my area of expertise.
+    """,
+}
+
+def refuse_safely(reason: str) -> str:
+    return REFUSAL_TEMPLATES.get(reason, REFUSAL_TEMPLATES["out_of_scope"])
+```
+
+## Output Validation
+
+```python
+def validate_output(output: str) -> dict:
+    """Check output before returning to user"""
+    issues = []
+
+    # Check for PII
+    pii_results = analyzer.analyze(output, language='en')
+    if pii_results:
+        issues.append("Contains PII")
+        output = redact_pii(output)
+
+    # Check for banned phrases
+    banned_phrases = ["password", "api key", "secret"]
+    for phrase in banned_phrases:
+        if phrase.lower() in output.lower():
+            issues.append(f"Contains banned phrase: {phrase}")
+
+    # Check toxicity
+    toxicity_score = toxicity_classifier(output)
+    if toxicity_score > 0.7:
+        issues.append("High toxicity detected")
+
+    return {
+        "safe": len(issues) == 0,
+        "issues": issues,
+        "sanitized_output": output
+    }
+```
+
+## Complete Guardrail Pipeline
+
+```python
+def apply_guardrails(user_input: str) -> dict:
+    # 1. Input validation
+    if detect_prompt_injection(user_input):
+        return {
+            "allowed": False,
+            "response": "Invalid request detected."
+        }
+
+    # 2. Policy check
+    policy_check = check_policy(user_input)
+    if not policy_check["allowed"]:
+        return {
+            "allowed": False,
+            "response": policy_check["refusal"]
+        }
+
+    # 3. Generate response
+    response = llm(user_input)
+
+    # 4. Output validation
+    validation = validate_output(response)
+    if not validation["safe"]:
+        return {
+            "allowed": True,
+            "response": validation["sanitized_output"],
+            "warnings": validation["issues"]
+        }
+
+    return {
+        "allowed": True,
+        "response": response
+    }
+```
+
+## Best Practices
+
+- Layer multiple defenses
+- Log all blocked requests
+- Provide helpful refusals
+- Redact, don't reject when possible
+- Regular pattern updates
+- Human review of edge cases
+
+## Output Checklist
+
+- [ ] PII detection implemented
+- [ ] Prompt injection detection
+- [ ] Topic classification
+- [ ] Policy constraints defined
+- [ ] Refusal templates written
+- [ ] Output validation
+- [ ] Logging/monitoring
+- [ ] Test cases for bypasses