codehava-agent-kit 1.0.1 → 2.0.0

package/templates/.agent/skills/prompt-engineering-patterns/scripts/optimize-prompt.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Prompt Optimization Script
+
+Automatically test and optimize prompts using A/B testing and metrics tracking.
+"""
+
+import json
+import time
+from typing import List, Dict, Any
+from dataclasses import dataclass
+from concurrent.futures import ThreadPoolExecutor
+import numpy as np
+
+
+@dataclass
+class TestCase:
+    input: Dict[str, Any]
+    expected_output: str
+    metadata: Dict[str, Any] = None
+
+
+class PromptOptimizer:
+    def __init__(self, llm_client, test_suite: List[TestCase]):
+        self.client = llm_client
+        self.test_suite = test_suite
+        self.results_history = []
+        self.executor = ThreadPoolExecutor()
+
+    def shutdown(self):
+        """Shutdown the thread pool executor."""
+        self.executor.shutdown(wait=True)
+
+    def evaluate_prompt(self, prompt_template: str, test_cases: List[TestCase] = None) -> Dict[str, float]:
+        """Evaluate a prompt template against test cases in parallel."""
+        if test_cases is None:
+            test_cases = self.test_suite
+
+        metrics = {
+            'accuracy': [],
+            'latency': [],
+            'token_count': [],
+            'success_rate': []
+        }
+
+        def process_test_case(test_case):
+            start_time = time.time()
+
+            # Render prompt with test case inputs
+            prompt = prompt_template.format(**test_case.input)
+
+            # Get LLM response
+            response = self.client.complete(prompt)
+
+            # Measure latency
+            latency = time.time() - start_time
+
+            # Calculate individual metrics
+            token_count = len(prompt.split()) + len(response.split())
+            success = 1 if response else 0
+            accuracy = self.calculate_accuracy(response, test_case.expected_output)
+
+            return {
+                'latency': latency,
+                'token_count': token_count,
+                'success_rate': success,
+                'accuracy': accuracy
+            }
+
+        # Run test cases in parallel
+        results = list(self.executor.map(process_test_case, test_cases))
+
+        # Aggregate metrics
+        for result in results:
+            metrics['latency'].append(result['latency'])
+            metrics['token_count'].append(result['token_count'])
+            metrics['success_rate'].append(result['success_rate'])
+            metrics['accuracy'].append(result['accuracy'])
+
+        return {
+            'avg_accuracy': np.mean(metrics['accuracy']),
+            'avg_latency': np.mean(metrics['latency']),
+            'p95_latency': np.percentile(metrics['latency'], 95),
+            'avg_tokens': np.mean(metrics['token_count']),
+            'success_rate': np.mean(metrics['success_rate'])
+        }
+
+    def calculate_accuracy(self, response: str, expected: str) -> float:
+        """Calculate accuracy score between response and expected output."""
+        # Simple exact match
+        if response.strip().lower() == expected.strip().lower():
+            return 1.0
+
+        # Partial match using word overlap
+        response_words = set(response.lower().split())
+        expected_words = set(expected.lower().split())
+
+        if not expected_words:
+            return 0.0
+
+        overlap = len(response_words & expected_words)
+        return overlap / len(expected_words)
+
+    def optimize(self, base_prompt: str, max_iterations: int = 5) -> Dict[str, Any]:
+        """Iteratively optimize a prompt."""
+        current_prompt = base_prompt
+        best_prompt = base_prompt
+        best_score = 0
+        current_metrics = None
+
+        for iteration in range(max_iterations):
+            print(f"\nIteration {iteration + 1}/{max_iterations}")
+
+            # Evaluate current prompt
+            # Bolt Optimization: Avoid re-evaluating if we already have metrics from previous iteration
+            if current_metrics:
+                metrics = current_metrics
+            else:
+                metrics = self.evaluate_prompt(current_prompt)
+
+            print(f"Accuracy: {metrics['avg_accuracy']:.2f}, Latency: {metrics['avg_latency']:.2f}s")
+
+            # Track results
+            self.results_history.append({
+                'iteration': iteration,
+                'prompt': current_prompt,
+                'metrics': metrics
+            })
+
+            # Update best if improved
+            if metrics['avg_accuracy'] > best_score:
+                best_score = metrics['avg_accuracy']
+                best_prompt = current_prompt
+
+            # Stop if good enough
+            if metrics['avg_accuracy'] > 0.95:
+                print("Achieved target accuracy!")
+                break
+
+            # Generate variations for next iteration
+            variations = self.generate_variations(current_prompt, metrics)
+
+            # Test variations and pick best
+            best_variation = current_prompt
+            best_variation_score = metrics['avg_accuracy']
+            best_variation_metrics = metrics
+
+            for variation in variations:
+                var_metrics = self.evaluate_prompt(variation)
+                if var_metrics['avg_accuracy'] > best_variation_score:
+                    best_variation_score = var_metrics['avg_accuracy']
+                    best_variation = variation
+                    best_variation_metrics = var_metrics
+
+            current_prompt = best_variation
+            current_metrics = best_variation_metrics
+
+        return {
+            'best_prompt': best_prompt,
+            'best_score': best_score,
+            'history': self.results_history
+        }
+
+    def generate_variations(self, prompt: str, current_metrics: Dict) -> List[str]:
+        """Generate prompt variations to test."""
+        variations = []
+
+        # Variation 1: Add explicit format instruction
+        variations.append(prompt + "\n\nProvide your answer in a clear, concise format.")
+
+        # Variation 2: Add step-by-step instruction
+        variations.append("Let's solve this step by step.\n\n" + prompt)
+
+        # Variation 3: Add verification step
+        variations.append(prompt + "\n\nVerify your answer before responding.")
+
+        # Variation 4: Make more concise
+        concise = self.make_concise(prompt)
+        if concise != prompt:
+            variations.append(concise)
+
+        # Variation 5: Add examples (if none present)
+        if "example" not in prompt.lower():
+            variations.append(self.add_examples(prompt))
+
+        return variations[:3]  # Return top 3 variations
+
+    def make_concise(self, prompt: str) -> str:
+        """Remove redundant words to make prompt more concise."""
+        replacements = [
+            ("in order to", "to"),
+            ("due to the fact that", "because"),
+            ("at this point in time", "now"),
+            ("in the event that", "if"),
+        ]
+
+        result = prompt
+        for old, new in replacements:
+            result = result.replace(old, new)
+
+        return result
+
+    def add_examples(self, prompt: str) -> str:
+        """Add example section to prompt."""
+        return f"""{prompt}
+
+Example:
+Input: Sample input
+Output: Sample output
+"""
+
+    def compare_prompts(self, prompt_a: str, prompt_b: str) -> Dict[str, Any]:
+        """A/B test two prompts."""
+        print("Testing Prompt A...")
+        metrics_a = self.evaluate_prompt(prompt_a)
+
+        print("Testing Prompt B...")
+        metrics_b = self.evaluate_prompt(prompt_b)
+
+        return {
+            'prompt_a_metrics': metrics_a,
+            'prompt_b_metrics': metrics_b,
+            'winner': 'A' if metrics_a['avg_accuracy'] > metrics_b['avg_accuracy'] else 'B',
+            'improvement': abs(metrics_a['avg_accuracy'] - metrics_b['avg_accuracy'])
+        }
+
+    def export_results(self, filename: str):
+        """Export optimization results to JSON."""
+        with open(filename, 'w') as f:
+            json.dump(self.results_history, f, indent=2)
+
+
+def main():
+    # Example usage
+    test_suite = [
+        TestCase(
+            input={'text': 'This movie was amazing!'},
+            expected_output='Positive'
+        ),
+        TestCase(
+            input={'text': 'Worst purchase ever.'},
+            expected_output='Negative'
+        ),
+        TestCase(
+            input={'text': 'It was okay, nothing special.'},
+            expected_output='Neutral'
+        )
+    ]
+
+    # Mock LLM client for demonstration
+    class MockLLMClient:
+        def complete(self, prompt):
+            # Simulate LLM response
+            if 'amazing' in prompt:
+                return 'Positive'
+            elif 'worst' in prompt.lower():
+                return 'Negative'
+            else:
+                return 'Neutral'
+
+    optimizer = PromptOptimizer(MockLLMClient(), test_suite)
+
+    try:
+        base_prompt = "Classify the sentiment of: {text}\nSentiment:"
+
+        results = optimizer.optimize(base_prompt)
+
+        print("\n" + "="*50)
+        print("Optimization Complete!")
+        print(f"Best Accuracy: {results['best_score']:.2f}")
+        print(f"Best Prompt:\n{results['best_prompt']}")
+
+        optimizer.export_results('optimization_results.json')
+    finally:
+        optimizer.shutdown()
+
+
+if __name__ == '__main__':
+    main()

package/templates/.agent/skills/prompt-library/SKILL.md

@@ -0,0 +1,325 @@
+---
+name: prompt-library
+description: "A comprehensive collection of battle-tested prompts inspired by [awesome-chatgpt-prompts](https://github.com/f/awesome-chatgpt-prompts) and community best practices."
+risk: unknown
+source: community
+date_added: "2026-02-27"
+---
+
+# 📝 Prompt Library
+
+> A comprehensive collection of battle-tested prompts inspired by [awesome-chatgpt-prompts](https://github.com/f/awesome-chatgpt-prompts) and community best practices.
+
+## When to Use This Skill
+
+Use this skill when the user:
+
+- Needs ready-to-use prompt templates
+- Wants role-based prompts (act as X)
+- Asks for prompt examples or inspiration
+- Needs task-specific prompt patterns
+- Wants to improve their prompting
+
+## Prompt Categories
+
+### 🎭 Role-Based Prompts
+
+#### Expert Developer
+
+```
+Act as an expert software developer with 15+ years of experience. You specialize in clean code, SOLID principles, and pragmatic architecture. When reviewing code:
+1. Identify bugs and potential issues
+2. Suggest performance improvements
+3. Recommend better patterns
+4. Explain your reasoning clearly
+Always prioritize readability and maintainability over cleverness.
+```
+
+#### Code Reviewer
+
+```
+Act as a senior code reviewer. Your role is to:
+1. Check for bugs, edge cases, and error handling
+2. Evaluate code structure and organization
+3. Assess naming conventions and readability
+4. Identify potential security issues
+5. Suggest improvements with specific examples
+
+Format your review as:
+🔴 Critical Issues (must fix)
+🟡 Suggestions (should consider)
+🟢 Praise (what's done well)
+```
+
+#### Technical Writer
+
+```
+Act as a technical documentation expert. Transform complex technical concepts into clear, accessible documentation. Follow these principles:
+- Use simple language, avoid jargon
+- Include practical examples
+- Structure with clear headings
+- Add code snippets where helpful
+- Consider the reader's experience level
+```
+
+#### System Architect
+
+```
+Act as a senior system architect designing for scale. Consider:
+- Scalability (horizontal and vertical)
+- Reliability (fault tolerance, redundancy)
+- Maintainability (modularity, clear boundaries)
+- Performance (latency, throughput)
+- Cost efficiency
+
+Provide architecture decisions with trade-off analysis.
+```
+
+### 🛠️ Task-Specific Prompts
+
+#### Debug This Code
+
+```
+Debug the following code. Your analysis should include:
+
+1. **Problem Identification**: What exactly is failing?
+2. **Root Cause**: Why is it failing?
+3. **Fix**: Provide corrected code
+4. **Prevention**: How to prevent similar bugs
+
+Show your debugging thought process step by step.
+```
+
+#### Explain Like I'm 5 (ELI5)
+
+```
+Explain [CONCEPT] as if I'm 5 years old. Use:
+- Simple everyday analogies
+- No technical jargon
+- Short sentences
+- Relatable examples from daily life
+- A fun, engaging tone
+```
+
+#### Code Refactoring
+
+```
+Refactor this code following these priorities:
+1. Readability first
+2. Remove duplication (DRY)
+3. Single responsibility per function
+4. Meaningful names
+5. Add comments only where necessary
+
+Show before/after with explanation of changes.
+```
+
+#### Write Tests
+
+```
+Write comprehensive tests for this code:
+1. Happy path scenarios
+2. Edge cases
+3. Error conditions
+4. Boundary values
+
+Use [FRAMEWORK] testing conventions. Include:
+- Descriptive test names
+- Arrange-Act-Assert pattern
+- Mocking where appropriate
+```
+
+#### API Documentation
+
+```
+Generate API documentation for this endpoint including:
+- Endpoint URL and method
+- Request parameters (path, query, body)
+- Request/response examples
+- Error codes and meanings
+- Authentication requirements
+- Rate limits if applicable
+
+Format as OpenAPI/Swagger or Markdown.
+```
+
+### 📊 Analysis Prompts
+
+#### Code Complexity Analysis
+
+```
+Analyze the complexity of this codebase:
+
+1. **Cyclomatic Complexity**: Identify complex functions
+2. **Coupling**: Find tightly coupled components
+3. **Cohesion**: Assess module cohesion
+4. **Dependencies**: Map critical dependencies
+5. **Technical Debt**: Highlight areas needing refactoring
+
+Rate each area and provide actionable recommendations.
+```
+
+#### Performance Analysis
+
+```
+Analyze this code for performance issues:
+
+1. **Time Complexity**: Big O analysis
+2. **Space Complexity**: Memory usage patterns
+3. **I/O Bottlenecks**: Database, network, disk
+4. **Algorithmic Issues**: Inefficient patterns
+5. **Quick Wins**: Easy optimizations
+
+Prioritize findings by impact.
+```
+
+#### Security Review
+
+```
+Perform a security review of this code:
+
+1. **Input Validation**: Check all inputs
+2. **Authentication/Authorization**: Access control
+3. **Data Protection**: Sensitive data handling
+4. **Injection Vulnerabilities**: SQL, XSS, etc.
+5. **Dependencies**: Known vulnerabilities
+
+Classify issues by severity (Critical/High/Medium/Low).
+```
+
+### 🎨 Creative Prompts
+
+#### Brainstorm Features
+
+```
+Brainstorm features for [PRODUCT]:
+
+For each feature, provide:
+- Name and one-line description
+- User value proposition
+- Implementation complexity (Low/Med/High)
+- Dependencies on other features
+
+Generate 10 ideas, then rank top 3 by impact/effort ratio.
+```
+
+#### Name Generator
+
+```
+Generate names for [PROJECT/FEATURE]:
+
+Provide 10 options in these categories:
+- Descriptive (what it does)
+- Evocative (how it feels)
+- Acronyms (memorable abbreviations)
+- Metaphorical (analogies)
+
+For each, explain the reasoning and check domain availability patterns.
+```
+
+### 🔄 Transformation Prompts
+
+#### Migrate Code
+
+```
+Migrate this code from [SOURCE] to [TARGET]:
+
+1. Identify equivalent constructs
+2. Handle incompatible features
+3. Preserve functionality exactly
+4. Follow target language idioms
+5. Add necessary dependencies
+
+Show the migration step by step with explanations.
+```
+
+#### Convert Format
+
+```
+Convert this [SOURCE_FORMAT] to [TARGET_FORMAT]:
+
+Requirements:
+- Preserve all data
+- Use idiomatic target format
+- Handle edge cases
+- Validate the output
+- Provide sample verification
+```
+
+## Prompt Engineering Techniques
+
+### Chain of Thought (CoT)
+
+```
+Let's solve this step by step:
+1. First, I'll understand the problem
+2. Then, I'll identify the key components
+3. Next, I'll work through the logic
+4. Finally, I'll verify the solution
+
+[Your question here]
+```
+
+### Few-Shot Learning
+
+```
+Here are some examples of the task:
+
+Example 1:
+Input: [example input 1]
+Output: [example output 1]
+
+Example 2:
+Input: [example input 2]
+Output: [example output 2]
+
+Now complete this:
+Input: [actual input]
+Output:
+```
+
+### Persona Pattern
+
+```
+You are [PERSONA] with [TRAITS].
+Your communication style is [STYLE].
+You prioritize [VALUES].
+
+When responding:
+- [Behavior 1]
+- [Behavior 2]
+- [Behavior 3]
+```
+
+### Structured Output
+
+```
+Respond in the following JSON format:
+{
+  "analysis": "your analysis here",
+  "recommendations": ["rec1", "rec2"],
+  "confidence": 0.0-1.0,
+  "caveats": ["caveat1"]
+}
+```
+
+## Prompt Improvement Checklist
+
+When crafting prompts, ensure:
+
+- [ ] **Clear objective**: What exactly do you want?
+- [ ] **Context provided**: Background information included?
+- [ ] **Format specified**: How should output be structured?
+- [ ] **Examples given**: Are there reference examples?
+- [ ] **Constraints defined**: Any limitations or requirements?
+- [ ] **Success criteria**: How do you measure good output?
+
+## Resources
+
+- [awesome-chatgpt-prompts](https://github.com/f/awesome-chatgpt-prompts)
+- [prompts.chat](https://prompts.chat)
+- [Learn Prompting](https://learnprompting.org/)
+
+---
+
+> 💡 **Tip**: The best prompts are specific, provide context, and include examples of desired output.