@freshworks/shiftleft-tools 1.1.8

package/templates/postman/scripts/report-generators/mutation-report.sh

@@ -0,0 +1,672 @@
+#!/bin/bash
+#
+# Mutation Testing Report Generator
+# ==================================
+# Runs PIT mutation tests and generates a comprehensive HTML report.
+#
+# Usage:
+#   ./mutation-report.sh [OPTIONS]
+#
+# Options:
+#   -r, --run        Force re-run mutation tests (even if results exist)
+#   -o, --output     Output directory (default: postman/reports)
+#   -h, --help       Show this help message
+#
+# Examples:
+#   ./mutation-report.sh              # Generate report (runs tests if needed)
+#   ./mutation-report.sh --run        # Force re-run tests and generate report
+#   ./mutation-report.sh -o ./reports # Custom output directory
+#
+# Prerequisites:
+#   - Maven installed and configured
+#   - Python 3.x installed
+#
+# Report includes:
+#   - Overall mutation score and line coverage
+#   - Test strength metrics
+#   - Mutator breakdown with kill rates
+#   - Package-level analysis
+#   - Class-level details
+#   - Survived mutations for review
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Helper functions
+log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
+log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }
+
+# Configuration
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+INSTALLATION_DIR="$PROJECT_ROOT/installation"
+PIT_REPORTS_DIR="$INSTALLATION_DIR/target/pit-reports"
+MUTATIONS_XML="$PIT_REPORTS_DIR/mutations.xml"
+OUTPUT_DIR="$PROJECT_ROOT/postman/reports"
+
+# Parse arguments
+RUN_TESTS=false
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help)
+            head -30 "$0" | tail -27
+            exit 0
+            ;;
+        -r|--run)
+            RUN_TESTS=true
+            shift
+            ;;
+        -o|--output)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        *)
+            echo -e "${RED}Unknown option: $1${NC}"
+            exit 1
+            ;;
+    esac
+done
+
+echo -e "${BLUE}=== Mutation Testing Report Generator ===${NC}"
+
+# Check if we should run mutation tests
+if [[ "$RUN_TESTS" == false ]] && [[ ! -f "$MUTATIONS_XML" ]]; then
+    echo ""
+    log_warn "mutations.xml not found. Running mutation tests..."
+    RUN_TESTS=true
+fi
+
+if [[ "$RUN_TESTS" == true ]]; then
+    echo ""
+    log_info "Running mutation tests (this may take 2-3 minutes)..."
+    echo ""
+    
+    cd "$INSTALLATION_DIR"
+    mvn org.pitest:pitest-maven:mutationCoverage 2>&1 | tee /tmp/pit-output.txt | grep -E "(INFO|Generated|Killed|Coverage|BUILD|>>)" || true
+    
+    # Check if build succeeded
+    if grep -q "BUILD SUCCESS" /tmp/pit-output.txt; then
+        echo ""
+        log_success "Mutation tests completed!"
+        echo ""
+    elif grep -q "BUILD FAILURE" /tmp/pit-output.txt; then
+        echo ""
+        log_error "Mutation tests failed. Check /tmp/pit-output.txt for details."
+        exit 1
+    fi
+fi
+
+# Verify mutations.xml exists after run
+if [[ ! -f "$MUTATIONS_XML" ]]; then
+    echo -e "${RED}Error: mutations.xml not found at $MUTATIONS_XML${NC}"
+    echo -e "${RED}Mutation tests may have failed. Check the output above.${NC}"
+    exit 1
+fi
+
+# Create output directory
+mkdir -p "$OUTPUT_DIR"
+
+TIMESTAMP=$(date +"%Y%m%d-%H%M%S")
+HTML_OUTPUT="$OUTPUT_DIR/mutation-report-$TIMESTAMP.html"
+
+echo -e "${GREEN}Parsing mutations.xml...${NC}"
+
+# Use unified report generator
+python3 "$SCRIPT_DIR/../lib/report_generator.py" mutation "$MUTATIONS_XML" "$HTML_OUTPUT"
+
+# Skip inline Python script - now handled by separate module
+: << 'SKIP_INLINE_PYTHON'
+# Python script to parse mutations.xml and generate report
+python3 << 'PYTHON_SCRIPT' - "$MUTATIONS_XML" "$HTML_OUTPUT"
+import sys
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from datetime import datetime
+
+mutations_xml = sys.argv[1]
+html_output = sys.argv[2]
+
+# Parse XML
+tree = ET.parse(mutations_xml)
+root = tree.getroot()
+
+# Collect statistics
+total = 0
+killed = 0
+survived = 0
+no_coverage = 0
+timed_out = 0
+tests_run = 0
+
+# By mutator
+mutator_stats = defaultdict(lambda: {'killed': 0, 'survived': 0, 'no_coverage': 0, 'total': 0})
+
+# By class
+class_stats = defaultdict(lambda: {'killed': 0, 'survived': 0, 'no_coverage': 0, 'total': 0})
+
+# By package
+package_stats = defaultdict(lambda: {'killed': 0, 'survived': 0, 'no_coverage': 0, 'total': 0})
+
+# Survived mutations for review
+survived_mutations = []
+
+for mutation in root.findall('mutation'):
+    total += 1
+    status = mutation.get('status')
+    mutator = mutation.find('mutator').text.split('.')[-1]
+    mutated_class = mutation.find('mutatedClass').text
+    source_file = mutation.find('sourceFile').text
+    method = mutation.find('mutatedMethod').text
+    line = mutation.find('lineNumber').text
+    description = mutation.find('description').text
+    tests = int(mutation.get('numberOfTestsRun', 0))
+    tests_run += tests
+    
+    # Package
+    package = '.'.join(mutated_class.split('.')[:-1])
+    
+    # Update stats
+    mutator_stats[mutator]['total'] += 1
+    class_stats[mutated_class]['total'] += 1
+    package_stats[package]['total'] += 1
+    
+    if status == 'KILLED':
+        killed += 1
+        mutator_stats[mutator]['killed'] += 1
+        class_stats[mutated_class]['killed'] += 1
+        package_stats[package]['killed'] += 1
+    elif status == 'SURVIVED':
+        survived += 1
+        mutator_stats[mutator]['survived'] += 1
+        class_stats[mutated_class]['survived'] += 1
+        package_stats[package]['survived'] += 1
+        survived_mutations.append({
+            'class': mutated_class.split('.')[-1],
+            'method': method,
+            'line': line,
+            'mutator': mutator,
+            'description': description
+        })
+    elif status == 'NO_COVERAGE':
+        no_coverage += 1
+        mutator_stats[mutator]['no_coverage'] += 1
+        class_stats[mutated_class]['no_coverage'] += 1
+        package_stats[package]['no_coverage'] += 1
+    elif status == 'TIMED_OUT':
+        timed_out += 1
+
+# Calculate percentages
+mutation_score = (killed / total * 100) if total > 0 else 0
+covered = total - no_coverage
+test_strength = (killed / covered * 100) if covered > 0 else 0
+tests_per_mutation = tests_run / total if total > 0 else 0
+
+# Sort mutators by kill rate
+mutator_sorted = sorted(
+    mutator_stats.items(),
+    key=lambda x: (x[1]['killed'] / x[1]['total'] if x[1]['total'] > 0 else 0),
+    reverse=True
+)
+
+# Sort packages by mutation score
+package_sorted = sorted(
+    package_stats.items(),
+    key=lambda x: (x[1]['killed'] / x[1]['total'] if x[1]['total'] > 0 else 0),
+    reverse=True
+)
+
+# Sort classes by number of survived mutations (most problematic first)
+class_sorted = sorted(
+    class_stats.items(),
+    key=lambda x: x[1]['survived'],
+    reverse=True
+)[:15]  # Top 15 classes with most survived
+
+def get_status_class(percentage):
+    if percentage >= 80:
+        return 'excellent'
+    elif percentage >= 70:
+        return 'good'
+    elif percentage >= 60:
+        return 'moderate'
+    else:
+        return 'poor'
+
+def get_status_icon(percentage):
+    if percentage >= 80:
+        return '✅'
+    elif percentage >= 70:
+        return '🟢'
+    elif percentage >= 60:
+        return '🟡'
+    else:
+        return '🔴'
+
+# Generate HTML - matching API coverage matrix style
+html = f'''<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Mutation Testing Report</title>
+    <style>
+        * {{ margin: 0; padding: 0; box-sizing: border-box; }}
+        
+        body {{
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+            background: #f8fafc;
+            color: #1e293b;
+            line-height: 1.6;
+            padding: 2rem;
+        }}
+        
+        .container {{ max-width: 1200px; margin: 0 auto; }}
+        
+        h1 {{
+            font-size: 1.75rem;
+            font-weight: 700;
+            color: #0f172a;
+            margin-bottom: 0.5rem;
+        }}
+        
+        .timestamp {{ color: #64748b; font-size: 0.875rem; margin-bottom: 2rem; }}
+        
+        h2 {{
+            font-size: 1.1rem;
+            font-weight: 600;
+            color: #0f172a;
+            margin: 2rem 0 1rem;
+            padding-bottom: 0.5rem;
+            border-bottom: 2px solid #e2e8f0;
+        }}
+        
+        h3 {{
+            font-size: 1rem;
+            font-weight: 600;
+            color: #64748b;
+            margin: 1.5rem 0 1rem;
+        }}
+        
+        .summary-grid {{
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+            gap: 1rem;
+            margin-bottom: 2rem;
+        }}
+        
+        .summary-card {{
+            background: white;
+            border: 1px solid #e2e8f0;
+            border-radius: 10px;
+            padding: 1.25rem;
+            text-align: center;
+        }}
+        
+        .summary-card .value {{ font-size: 2rem; font-weight: 700; color: #3b82f6; }}
+        .summary-card .label {{ color: #64748b; font-size: 0.8rem; margin-top: 0.25rem; }}
+        .summary-card .detail {{ color: #94a3b8; font-size: 0.75rem; margin-top: 0.25rem; }}
+        .summary-card.success .value {{ color: #10b981; }}
+        .summary-card.warning .value {{ color: #f59e0b; }}
+        .summary-card.error .value {{ color: #ef4444; }}
+        
+        table {{
+            width: 100%;
+            border-collapse: collapse;
+            background: white;
+            border: 1px solid #e2e8f0;
+            border-radius: 10px;
+            overflow: hidden;
+            margin-bottom: 2rem;
+            font-size: 0.8rem;
+        }}
+        
+        th, td {{ 
+            padding: 0.6rem 0.5rem; 
+            text-align: center; 
+            border-bottom: 1px solid #e2e8f0;
+            vertical-align: middle;
+        }}
+        th {{
+            background: #f1f5f9;
+            font-weight: 600;
+            font-size: 0.65rem;
+            text-transform: uppercase;
+            color: #64748b;
+        }}
+        
+        td.left {{ text-align: left; }}
+        
+        tr:hover {{ background: #f8fafc; }}
+        
+        .progress-bar {{
+            height: 8px;
+            background: #e2e8f0;
+            border-radius: 4px;
+            overflow: hidden;
+            min-width: 100px;
+        }}
+        
+        .progress-fill {{
+            height: 100%;
+            border-radius: 4px;
+        }}
+        
+        .progress-fill.success {{ background: #10b981; }}
+        .progress-fill.good {{ background: #22c55e; }}
+        .progress-fill.warning {{ background: #f59e0b; }}
+        .progress-fill.error {{ background: #ef4444; }}
+        
+        .two-columns {{
+            display: grid;
+            grid-template-columns: 1fr 1fr;
+            gap: 1.5rem;
+            align-items: start;
+        }}
+        
+        @media (max-width: 900px) {{
+            .two-columns {{ grid-template-columns: 1fr; }}
+        }}
+        
+        .two-columns > div {{
+            display: flex;
+            flex-direction: column;
+        }}
+        
+        .two-columns .card {{
+            flex: 1;
+            min-height: 400px;
+            max-height: 400px;
+            overflow-y: auto;
+        }}
+        
+        .card {{
+            background: white;
+            border: 1px solid #e2e8f0;
+            border-radius: 10px;
+            padding: 1rem;
+            margin-bottom: 1rem;
+        }}
+        
+        .survived-list {{
+            max-height: none;
+            overflow-y: auto;
+        }}
+        
+        .survived-item {{
+            padding: 0.75rem;
+            border-bottom: 1px solid #e2e8f0;
+            font-size: 0.85rem;
+        }}
+        
+        .survived-item:last-child {{ border-bottom: none; }}
+        
+        .survived-class {{ font-weight: 600; color: #0f172a; }}
+        .survived-method {{ color: #3b82f6; }}
+        .survived-description {{ color: #64748b; font-style: italic; margin-top: 0.25rem; font-size: 0.8rem; }}
+        
+        .definitions {{
+            background: white;
+            border: 1px solid #e2e8f0;
+            border-radius: 10px;
+            padding: 1.25rem;
+            margin-top: 2rem;
+        }}
+        
+        .definitions h3 {{ margin-top: 0; color: #0f172a; }}
+        .definitions dl {{ margin: 0; }}
+        .definitions dt {{ font-weight: 600; margin-top: 0.75rem; color: #0f172a; }}
+        .definitions dd {{ color: #64748b; margin-left: 1rem; font-size: 0.9rem; }}
+        
+        footer {{ margin-top: 2rem; text-align: center; color: #94a3b8; font-size: 0.8rem; }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>🧬 Mutation Testing Report</h1>
+        <p class="timestamp">Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}</p>
+        
+        <div class="summary-grid">
+            <div class="summary-card {'success' if mutation_score >= 80 else 'warning' if mutation_score >= 60 else 'error'}">
+                <div class="value">{mutation_score:.1f}%</div>
+                <div class="label">Mutation Score</div>
+                <div class="detail">{killed:,} / {total:,} killed</div>
+            </div>
+            <div class="summary-card {'success' if test_strength >= 80 else 'warning' if test_strength >= 60 else 'error'}">
+                <div class="value">{test_strength:.1f}%</div>
+                <div class="label">Test Strength</div>
+                <div class="detail">{killed:,} / {covered:,} covered</div>
+            </div>
+            <div class="summary-card {'error' if survived > 100 else 'warning' if survived > 50 else ''}">
+                <div class="value">{survived}</div>
+                <div class="label">Survived</div>
+                <div class="detail">Mutations not killed</div>
+            </div>
+            <div class="summary-card {'error' if no_coverage > 100 else 'warning' if no_coverage > 50 else ''}">
+                <div class="value">{no_coverage}</div>
+                <div class="label">No Coverage</div>
+                <div class="detail">Need more tests</div>
+            </div>
+            <div class="summary-card">
+                <div class="value">{tests_per_mutation:.1f}</div>
+                <div class="label">Tests/Mutation</div>
+                <div class="detail">{tests_run:,} tests run</div>
+            </div>
+        </div>
+        
+        <h2>Mutator Breakdown</h2>
+        <table>
+            <thead>
+                <tr>
+                    <th style="text-align: left;">Mutator</th>
+                    <th>Kill Rate</th>
+                    <th style="width: 150px;">Progress</th>
+                    <th>Killed</th>
+                    <th>Survived</th>
+                    <th>No Coverage</th>
+                    <th>Total</th>
+                    <th>Status</th>
+                </tr>
+            </thead>
+            <tbody>
+'''
+
+# Friendly names for mutators
+mutator_friendly_names = {
+    'NegateConditionals': 'Flip Conditions (if → if not)',
+    'VoidMethodCall': 'Remove Method Calls',
+    'NullReturns': 'Return Null Instead',
+    'EmptyObjectReturns': 'Return Empty Object',
+    'BooleanTrueReturnVals': 'Return True Instead',
+    'BooleanFalseReturnVals': 'Return False Instead',
+    'PrimitiveReturns': 'Return 0/Default Value',
+    'ConditionalsBoundary': 'Change < to ≤, > to ≥',
+    'Math': 'Change Math Operators (+−×÷)',
+    'Increments': 'Change ++ to −−',
+    'InvertNegs': 'Remove Negative Signs',
+}
+
+for mutator, stats in mutator_sorted:
+    rate = (stats['killed'] / stats['total'] * 100) if stats['total'] > 0 else 0
+    status_class = 'success' if rate >= 80 else 'good' if rate >= 70 else 'warning' if rate >= 50 else 'error'
+    # Get friendly name or clean up the mutator name
+    short_name = mutator.replace('Mutator', '')
+    friendly_name = mutator_friendly_names.get(short_name, short_name)
+    html += f'''                <tr>
+                    <td class="left">{friendly_name}</td>
+                    <td><strong>{rate:.1f}%</strong></td>
+                    <td>
+                        <div class="progress-bar">
+                            <div class="progress-fill {status_class}" style="width: {rate}%"></div>
+                        </div>
+                    </td>
+                    <td>{stats['killed']}</td>
+                    <td>{stats['survived']}</td>
+                    <td>{stats['no_coverage']}</td>
+                    <td>{stats['total']}</td>
+                    <td>{get_status_icon(rate)}</td>
+                </tr>
+'''
+
+html += '''            </tbody>
+        </table>
+        
+        <h2>Package Breakdown</h2>
+        <table>
+            <thead>
+                <tr>
+                    <th style="text-align: left;">Package</th>
+                    <th>Kill Rate</th>
+                    <th style="width: 150px;">Progress</th>
+                    <th>Killed</th>
+                    <th>Survived</th>
+                    <th>No Coverage</th>
+                    <th>Total</th>
+                </tr>
+            </thead>
+            <tbody>
+'''
+
+for package, stats in package_sorted:
+    rate = (stats['killed'] / stats['total'] * 100) if stats['total'] > 0 else 0
+    status_class = 'success' if rate >= 80 else 'good' if rate >= 70 else 'warning' if rate >= 50 else 'error'
+    short_pkg = package.replace('com.freshworks.marketplace.installation.', '')
+    html += f'''                <tr>
+                    <td class="left">{short_pkg}</td>
+                    <td><strong>{rate:.1f}%</strong></td>
+                    <td>
+                        <div class="progress-bar">
+                            <div class="progress-fill {status_class}" style="width: {rate}%"></div>
+                        </div>
+                    </td>
+                    <td>{stats['killed']}</td>
+                    <td>{stats['survived']}</td>
+                    <td>{stats['no_coverage']}</td>
+                    <td>{stats['total']}</td>
+                </tr>
+'''
+
+html += '''            </tbody>
+        </table>
+        
+        <div class="two-columns">
+            <div>
+                <h2>Classes Needing Attention</h2>
+                <div class="card">
+                    <p style="color: #64748b; margin-bottom: 1rem; font-size: 0.85rem;">
+                        Classes with most survived mutations
+                    </p>
+                    <table>
+                        <thead>
+                            <tr>
+                                <th style="text-align: left;">Class</th>
+                                <th>Survived</th>
+                                <th>Total</th>
+                                <th>Kill Rate</th>
+                            </tr>
+                        </thead>
+                        <tbody>
+'''
+
+for class_name, stats in class_sorted:
+    if stats['survived'] == 0:
+        continue
+    rate = (stats['killed'] / stats['total'] * 100) if stats['total'] > 0 else 0
+    short_class = class_name.split('.')[-1]
+    html += f'''                            <tr>
+                                <td class="left">{short_class}</td>
+                                <td style="color: #ef4444; font-weight: 600;">{stats['survived']}</td>
+                                <td>{stats['total']}</td>
+                                <td>{rate:.1f}%</td>
+                            </tr>
+'''
+
+html += '''                        </tbody>
+                    </table>
+                </div>
+            </div>
+            
+            <div>
+                <h2>Sample Survived Mutations</h2>
+                <div class="card survived-list">
+'''
+
+for mut in survived_mutations[:20]:
+    html += f'''                    <div class="survived-item">
+                        <span class="survived-class">{mut['class']}</span>.<span class="survived-method">{mut['method']}</span> (line {mut['line']})
+                        <div class="survived-description">{mut['description']}</div>
+                    </div>
+'''
+
+html += '''                </div>
+            </div>
+        </div>
+        
+        <div class="definitions">
+            <h3>Definitions</h3>
+            <dl>
+                <dt>Mutation Score</dt>
+                <dd>Percentage of mutations killed by tests. Target: ≥80%</dd>
+                
+                <dt>Test Strength</dt>
+                <dd>Kill rate among covered mutations (excludes NO_COVERAGE). Shows test quality.</dd>
+                
+                <dt>Killed</dt>
+                <dd>Mutations detected by at least one test - good!</dd>
+                
+                <dt>Survived</dt>
+                <dd>Mutations not detected - tests need improvement.</dd>
+                
+                <dt>No Coverage</dt>
+                <dd>Code not covered by any test - need more tests.</dd>
+                
+                <dt>Common Mutators</dt>
+                <dd>
+                    <strong>NegateConditionals:</strong> Changes if(x) to if(!x)<br>
+                    <strong>VoidMethodCall:</strong> Removes void method calls<br>
+                    <strong>NullReturns:</strong> Returns null instead of actual value<br>
+                    <strong>BooleanReturns:</strong> Swaps true/false return values<br>
+                    <strong>ConditionalsBoundary:</strong> Changes &lt; to &lt;=, etc.
+                </dd>
+            </dl>
+        </div>
+        
+        <footer>
+            Generated by Mutation Report Generator | Data from PIT Mutation Testing
+        </footer>
+    </div>
+</body>
+</html>
+'''
+
+# Write HTML
+with open(html_output, 'w') as f:
+    f.write(html)
+
+print(f"Total mutations: {total}")
+print(f"Killed: {killed} ({mutation_score:.1f}%)")
+print(f"Survived: {survived}")
+print(f"No Coverage: {no_coverage}")
+print(f"Test Strength: {test_strength:.1f}%")
+print(f"Tests per Mutation: {tests_per_mutation:.1f}")
+PYTHON_SCRIPT
+SKIP_INLINE_PYTHON
+
+echo ""
+echo -e "${GREEN}✅ Report generated: $HTML_OUTPUT${NC}"
+
+# Cleanup old reports (keep only last one)
+cd "$OUTPUT_DIR"
+ls -t mutation-report-*.html 2>/dev/null | tail -n +2 | xargs -I {} rm -f {} 2>/dev/null || true
+echo -e "${BLUE}Cleaned up old reports (kept only latest)${NC}"
+
+echo ""
+echo -e "${YELLOW}To view the report:${NC}"
+echo "  open $HTML_OUTPUT"