@salesforce/afv-skills 1.8.0 → 1.9.0

package/skills/running-apex-tests/hooks/scripts/parse-test-results.py

@@ -0,0 +1,364 @@
+#!/usr/bin/env python3
+"""
+Parse Apex test results and format for Claude auto-fix loop.
+
+This hook parses the JSON output from `sf apex run test` and provides
+structured feedback that enables Claude to automatically fix failing tests.
+
+Environment Variables:
+    TOOL_OUTPUT: The stdout from the Bash command
+    TOOL_INPUT: The command that was executed
+
+Output:
+    Formatted test results with failure analysis and fix suggestions
+"""
+
+import json
+import os
+import sys
+import re
+from pathlib import Path
+from datetime import datetime
+
+# Only process sf apex run test commands
+def should_process():
+    """Check if this is an apex test command we should process."""
+    tool_input = os.environ.get('TOOL_INPUT', '')
+    return 'sf apex run test' in tool_input or 'sf apex get test' in tool_input
+
+def parse_test_results(output: str) -> dict:
+    """
+    Parse test results from sf CLI JSON output.
+
+    Returns:
+        dict with summary, failures, and coverage data
+    """
+    try:
+        # Try to parse as JSON (if --result-format json was used)
+        data = json.loads(output)
+        return parse_json_results(data)
+    except json.JSONDecodeError:
+        # Parse human-readable output
+        return parse_text_results(output)
+
+def parse_json_results(data: dict) -> dict:
+    """Parse JSON format test results."""
+    result = data.get('result', data)
+
+    summary = {
+        'passed': 0,
+        'failed': 0,
+        'skipped': 0,
+        'total': 0,
+        'duration_ms': 0,
+        'coverage_percent': 0
+    }
+
+    failures = []
+    coverage = []
+
+    # Parse test results
+    tests = result.get('tests', [])
+    for test in tests:
+        outcome = test.get('Outcome', test.get('outcome', '')).lower()
+        if outcome == 'pass':
+            summary['passed'] += 1
+        elif outcome == 'fail':
+            summary['failed'] += 1
+            failures.append({
+                'class': test.get('ApexClass', {}).get('Name', test.get('className', 'Unknown')),
+                'method': test.get('MethodName', test.get('methodName', 'Unknown')),
+                'message': test.get('Message', test.get('message', '')),
+                'stack_trace': test.get('StackTrace', test.get('stackTrace', '')),
+                'run_time': test.get('RunTime', test.get('runTime', 0))
+            })
+        elif outcome == 'skip':
+            summary['skipped'] += 1
+
+    summary['total'] = summary['passed'] + summary['failed'] + summary['skipped']
+
+    # Parse coverage
+    coverage_data = result.get('coverage', {}).get('coverage', [])
+    if not coverage_data:
+        coverage_data = result.get('codecoverage', [])
+
+    total_lines = 0
+    covered_lines = 0
+
+    for cov in coverage_data:
+        class_name = cov.get('name', cov.get('apexClassOrTriggerName', 'Unknown'))
+        num_lines = cov.get('totalLines', cov.get('numLinesCovered', 0) + cov.get('numLinesUncovered', 0))
+        num_covered = cov.get('coveredLines', cov.get('numLinesCovered', 0))
+        if isinstance(num_covered, list):
+            num_covered = len(num_covered)
+
+        uncovered = cov.get('uncoveredLines', [])
+        if isinstance(uncovered, int):
+            uncovered = []
+
+        pct = (num_covered / num_lines * 100) if num_lines > 0 else 0
+
+        coverage.append({
+            'class': class_name,
+            'total_lines': num_lines,
+            'covered_lines': num_covered,
+            'uncovered_lines': uncovered[:10] if uncovered else [],  # Limit to first 10
+            'percent': round(pct, 1)
+        })
+
+        total_lines += num_lines
+        covered_lines += num_covered if isinstance(num_covered, int) else 0
+
+    summary['coverage_percent'] = round(covered_lines / total_lines * 100, 1) if total_lines > 0 else 0
+
+    return {
+        'summary': summary,
+        'failures': failures,
+        'coverage': coverage
+    }
+
+def parse_text_results(output: str) -> dict:
+    """Parse human-readable test output."""
+    summary = {
+        'passed': 0,
+        'failed': 0,
+        'skipped': 0,
+        'total': 0,
+        'duration_ms': 0,
+        'coverage_percent': 0
+    }
+
+    failures = []
+
+    # Look for pass/fail patterns
+    pass_match = re.search(r'(\d+)\s+(?:test[s]?\s+)?pass(?:ed|ing)?', output, re.IGNORECASE)
+    fail_match = re.search(r'(\d+)\s+(?:test[s]?\s+)?fail(?:ed|ing|ure)?', output, re.IGNORECASE)
+
+    if pass_match:
+        summary['passed'] = int(pass_match.group(1))
+    if fail_match:
+        summary['failed'] = int(fail_match.group(1))
+
+    summary['total'] = summary['passed'] + summary['failed']
+
+    # Look for failure details
+    failure_pattern = re.compile(
+        r'([\w]+)\.([\w]+)\s*[-:]\s*(.*?)(?=\n\n|\n[A-Z]|$)',
+        re.MULTILINE | re.DOTALL
+    )
+
+    for match in failure_pattern.finditer(output):
+        if 'fail' in match.group(3).lower() or 'error' in match.group(3).lower():
+            failures.append({
+                'class': match.group(1),
+                'method': match.group(2),
+                'message': match.group(3).strip(),
+                'stack_trace': ''
+            })
+
+    return {
+        'summary': summary,
+        'failures': failures,
+        'coverage': []
+    }
+
+def analyze_failure(failure: dict) -> dict:
+    """
+    Analyze a test failure and suggest fix strategy.
+
+    Returns:
+        dict with error_type, root_cause, and suggested_fix
+    """
+    message = failure.get('message', '')
+    stack_trace = failure.get('stack_trace', '')
+
+    analysis = {
+        'error_type': 'Unknown',
+        'root_cause': 'Unable to determine root cause',
+        'suggested_fix': 'Review the test and code under test',
+        'auto_fixable': False
+    }
+
+    # Assertion failures
+    if 'AssertException' in message or 'Assertion Failed' in message:
+        analysis['error_type'] = 'Assertion Failure'
+
+        # Extract expected vs actual
+        expected_match = re.search(r'[Ee]xpected[:\s]+(\S+)', message)
+        actual_match = re.search(r'[Aa]ctual[:\s]+(\S+)', message)
+
+        if expected_match and actual_match:
+            analysis['root_cause'] = f"Expected {expected_match.group(1)} but got {actual_match.group(1)}"
+            analysis['suggested_fix'] = "Check if the test expectation is correct, or if the code logic needs fixing"
+        else:
+            analysis['root_cause'] = "Test assertion did not match expected outcome"
+            analysis['suggested_fix'] = "Review the assertion and verify expected vs actual values"
+
+        analysis['auto_fixable'] = True
+
+    # Null pointer
+    elif 'NullPointerException' in message:
+        analysis['error_type'] = 'Null Pointer Exception'
+
+        # Try to extract line number
+        line_match = re.search(r'[Ll]ine[:\s]+(\d+)', stack_trace or message)
+        if line_match:
+            analysis['root_cause'] = f"Null reference at line {line_match.group(1)}"
+        else:
+            analysis['root_cause'] = "Attempting to access a property or method on a null reference"
+
+        analysis['suggested_fix'] = "Add null check before accessing the object, or ensure test data setup creates required records"
+        analysis['auto_fixable'] = True
+
+    # DML exceptions
+    elif 'DmlException' in message:
+        analysis['error_type'] = 'DML Exception'
+
+        if 'REQUIRED_FIELD_MISSING' in message:
+            analysis['root_cause'] = "Required field not populated in test data"
+            analysis['suggested_fix'] = "Add the missing required field to TestDataFactory or test setup"
+        elif 'FIELD_CUSTOM_VALIDATION_EXCEPTION' in message:
+            analysis['root_cause'] = "Record fails validation rule"
+            analysis['suggested_fix'] = "Modify test data to meet validation rule requirements"
+        elif 'DUPLICATE_VALUE' in message:
+            analysis['root_cause'] = "Unique field constraint violation"
+            analysis['suggested_fix'] = "Use unique values in test data (e.g., add timestamp or random suffix)"
+        else:
+            analysis['root_cause'] = "DML operation failed"
+            analysis['suggested_fix'] = "Review the DML error message and adjust test data accordingly"
+
+        analysis['auto_fixable'] = True
+
+    # Query exceptions
+    elif 'QueryException' in message:
+        analysis['error_type'] = 'Query Exception'
+        analysis['root_cause'] = "SOQL query returned no results or too many results"
+        analysis['suggested_fix'] = "Ensure test data exists before querying, or handle empty results"
+        analysis['auto_fixable'] = True
+
+    # Limit exceptions
+    elif 'LimitException' in message:
+        analysis['error_type'] = 'Governor Limit Exception'
+
+        if 'Too many SOQL' in message:
+            analysis['root_cause'] = "SOQL query limit exceeded (100 queries)"
+            analysis['suggested_fix'] = "Bulkify queries - query before loops, use maps for lookups"
+        elif 'Too many DML' in message:
+            analysis['root_cause'] = "DML statement limit exceeded (150 statements)"
+            analysis['suggested_fix'] = "Bulkify DML - collect records in list, single DML after loop"
+        else:
+            analysis['root_cause'] = "Governor limit exceeded"
+            analysis['suggested_fix'] = "Review code for bulkification issues"
+
+        analysis['auto_fixable'] = True
+
+    # Mixed DML
+    elif 'MIXED_DML_OPERATION' in message:
+        analysis['error_type'] = 'Mixed DML Exception'
+        analysis['root_cause'] = "Setup and non-setup objects modified in same transaction"
+        analysis['suggested_fix'] = "Use System.runAs() to separate User operations from data operations"
+        analysis['auto_fixable'] = True
+
+    return analysis
+
+def format_output(results: dict) -> str:
+    """Format test results for Claude consumption."""
+    summary = results['summary']
+    failures = results['failures']
+    coverage = results['coverage']
+
+    lines = []
+    lines.append("=" * 60)
+    lines.append("📊 APEX TEST RESULTS")
+    lines.append("=" * 60)
+    lines.append("")
+
+    # Summary
+    status_icon = "✅" if summary['failed'] == 0 else "❌"
+    lines.append(f"{status_icon} SUMMARY")
+    lines.append("-" * 60)
+    lines.append(f"   Passed:   {summary['passed']}")
+    lines.append(f"   Failed:   {summary['failed']}")
+    lines.append(f"   Skipped:  {summary['skipped']}")
+    lines.append(f"   Total:    {summary['total']}")
+
+    if summary['coverage_percent'] > 0:
+        cov_icon = "✅" if summary['coverage_percent'] >= 75 else "⚠️"
+        lines.append(f"   Coverage: {summary['coverage_percent']}% {cov_icon}")
+
+    lines.append("")
+
+    # Failures with analysis
+    if failures:
+        lines.append("❌ FAILED TESTS")
+        lines.append("-" * 60)
+
+        for i, failure in enumerate(failures, 1):
+            analysis = analyze_failure(failure)
+
+            lines.append(f"\n{i}. {failure['class']}.{failure['method']}")
+            lines.append(f"   Error Type: {analysis['error_type']}")
+            lines.append(f"   Message: {failure['message'][:200]}...")
+            lines.append(f"   Root Cause: {analysis['root_cause']}")
+            lines.append(f"   Suggested Fix: {analysis['suggested_fix']}")
+
+            if analysis['auto_fixable']:
+                lines.append("   🤖 AUTO-FIXABLE: Yes - Claude can attempt automatic fix")
+
+        lines.append("")
+        lines.append("=" * 60)
+        lines.append("🤖 AGENTIC FIX INSTRUCTIONS")
+        lines.append("=" * 60)
+        lines.append("")
+        lines.append("To automatically fix these failures:")
+        lines.append("1. Read the failing test class")
+        lines.append("2. Read the class under test")
+        lines.append("3. Apply the suggested fix")
+        lines.append("4. Re-run: sf apex run test --tests [ClassName].[methodName]")
+        lines.append("")
+
+    # Coverage details (if below threshold)
+    low_coverage = [c for c in coverage if c['percent'] < 75]
+    if low_coverage:
+        lines.append("⚠️ LOW COVERAGE CLASSES (<75%)")
+        lines.append("-" * 60)
+
+        for cov in sorted(low_coverage, key=lambda x: x['percent']):
+            lines.append(f"   {cov['class']}: {cov['percent']}%")
+            if cov.get('uncovered_lines'):
+                lines.append(f"      Uncovered lines: {cov['uncovered_lines']}")
+
+        lines.append("")
+
+    lines.append("=" * 60)
+
+    return "\n".join(lines)
+
+def main():
+    """Main entry point."""
+    if not should_process():
+        # Not an apex test command, exit silently
+        sys.exit(0)
+
+    output = os.environ.get('TOOL_OUTPUT', '')
+
+    if not output:
+        sys.exit(0)
+
+    # Check if this looks like test output
+    if 'test' not in output.lower() and 'coverage' not in output.lower():
+        sys.exit(0)
+
+    try:
+        results = parse_test_results(output)
+
+        # Only output if there were tests or failures
+        if results['summary']['total'] > 0 or results['failures']:
+            formatted = format_output(results)
+            print(formatted)
+    except Exception as e:
+        # Silently fail - don't block on parsing errors
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()

package/skills/running-apex-tests/references/cli-commands.md

@@ -0,0 +1,289 @@
+<!-- Parent: running-apex-tests/SKILL.md -->
+# Salesforce CLI Test Commands Reference
+
+## Quick Reference
+
+| Task | Command |
+|------|---------|
+| Run single test class | `sf apex run test --class-names MyTest` |
+| Run all local tests | `sf apex run test --test-level RunLocalTests` |
+| Run with coverage | `sf apex run test --class-names MyTest --code-coverage` |
+| Get JSON output | `sf apex run test --class-names MyTest --result-format json` |
+| Run specific methods | `sf apex run test --tests MyTest.method1 --tests MyTest.method2` |
+
+## Test Execution
+
+### Run Single Test Class
+
+```bash
+sf apex run test \
+  --class-names AccountServiceTest \
+  --target-org my-sandbox \
+  --code-coverage \
+  --result-format human
+```
+
+### Run Multiple Test Classes
+
+```bash
+sf apex run test \
+  --class-names AccountServiceTest \
+  --class-names ContactServiceTest \
+  --class-names LeadServiceTest \
+  --target-org my-sandbox \
+  --code-coverage
+```
+
+### Run Specific Test Methods
+
+```bash
+sf apex run test \
+  --tests AccountServiceTest.testCreate \
+  --tests AccountServiceTest.testUpdate \
+  --target-org my-sandbox
+```
+
+### Run All Local Tests
+
+```bash
+sf apex run test \
+  --test-level RunLocalTests \
+  --target-org my-sandbox \
+  --code-coverage \
+  --output-dir test-results
+```
+
+### Run Test Suite
+
+```bash
+sf apex run test \
+  --suite-names RegressionSuite \
+  --target-org my-sandbox \
+  --code-coverage
+```
+
+## Test Levels
+
+| Level | Description |
+|-------|-------------|
+| `RunSpecifiedTests` | Only specified tests (default when using --class-names) |
+| `RunLocalTests` | All tests except managed packages |
+| `RunAllTestsInOrg` | All tests including managed packages |
+
+## Output Formats
+
+### Human Readable (Default)
+
+```bash
+sf apex run test --class-names MyTest --result-format human
+```
+
+### JSON (For Parsing)
+
+```bash
+sf apex run test --class-names MyTest --result-format json
+```
+
+### JUnit XML (For CI/CD)
+
+```bash
+sf apex run test --class-names MyTest --result-format junit
+```
+
+### TAP (Test Anything Protocol)
+
+```bash
+sf apex run test --class-names MyTest --result-format tap
+```
+
+## Code Coverage
+
+### Basic Coverage
+
+```bash
+sf apex run test \
+  --class-names MyTest \
+  --code-coverage \
+  --target-org my-sandbox
+```
+
+### Detailed Line-by-Line Coverage
+
+```bash
+sf apex run test \
+  --class-names MyTest \
+  --code-coverage \
+  --detailed-coverage \
+  --target-org my-sandbox
+```
+
+### Save Results to Directory
+
+```bash
+sf apex run test \
+  --class-names MyTest \
+  --code-coverage \
+  --output-dir ./test-results \
+  --target-org my-sandbox
+```
+
+Output files:
+- `test-run-id.json` - Test results
+- `test-run-id-codecoverage.json` - Coverage data
+
+## Async Test Execution
+
+### Run Tests Asynchronously
+
+```bash
+sf apex run test \
+  --test-level RunLocalTests \
+  --target-org my-sandbox \
+  --async
+```
+
+Returns a test run ID for checking status later.
+
+### Check Async Test Status
+
+```bash
+sf apex get test \
+  --test-run-id 707xx0000000000AAA \
+  --target-org my-sandbox
+```
+
+### Wait for Test Completion
+
+```bash
+sf apex run test \
+  --test-level RunLocalTests \
+  --target-org my-sandbox \
+  --wait 10  # Wait up to 10 minutes
+```
+
+## Debug Logs During Tests
+
+### Enable Debug Logs
+
+```bash
+# List current log levels
+sf apex list log --target-org my-sandbox
+
+# Tail logs in real-time
+sf apex tail log --target-org my-sandbox --color
+```
+
+### Get Specific Log
+
+```bash
+sf apex get log \
+  --log-id 07Lxx0000000000AAA \
+  --target-org my-sandbox
+```
+
+## Useful Flags
+
+| Flag | Description |
+|------|-------------|
+| `--code-coverage` | Include coverage in results |
+| `--detailed-coverage` | Line-by-line coverage (slower) |
+| `--result-format` | Output format (human, json, junit, tap) |
+| `--output-dir` | Save results to directory |
+| `--synchronous` | Wait for completion (default) |
+| `--wait` | Max minutes to wait |
+| `--async` | Return immediately with run ID |
+| `--verbose` | Show additional details |
+| `--concise` | Suppress passing test details (show only failures) |
+| `--poll-interval <seconds>` | Customize polling interval (v2.116.6+) |
+
+## Common Patterns
+
+### Full Test Run with Coverage Report
+
+```bash
+sf apex run test \
+  --test-level RunLocalTests \
+  --code-coverage \
+  --result-format json \
+  --output-dir ./test-results \
+  --target-org my-sandbox \
+  --wait 30
+```
+
+### Quick Validation (Single Test)
+
+```bash
+sf apex run test \
+  --tests AccountServiceTest.testCreate \
+  --target-org my-sandbox
+```
+
+### CI/CD Pipeline Pattern
+
+```bash
+# Run tests with JUnit output for CI tools
+sf apex run test \
+  --test-level RunLocalTests \
+  --result-format junit \
+  --output-dir ./test-results \
+  --code-coverage \
+  --target-org ci-sandbox \
+  --wait 60
+
+# Check exit code
+if [ $? -ne 0 ]; then
+  echo "Tests failed!"
+  exit 1
+fi
+```
+
+### Coverage Validation
+
+```bash
+# Run tests and check minimum coverage
+sf apex run test \
+  --test-level RunLocalTests \
+  --code-coverage \
+  --result-format json \
+  --output-dir ./test-results \
+  --target-org my-sandbox
+
+# Parse coverage from JSON (requires jq)
+coverage=$(jq '.result.summary.orgWideCoverage' ./test-results/*.json | tr -d '"' | tr -d '%')
+if [ "$coverage" -lt 75 ]; then
+  echo "Coverage $coverage% is below 75% threshold!"
+  exit 1
+fi
+```
+
+## Troubleshooting
+
+### Test Timeout
+
+```bash
+# Increase wait time for long-running tests
+sf apex run test \
+  --test-level RunAllTestsInOrg \
+  --wait 120 \
+  --target-org my-sandbox
+```
+
+### No Test Results
+
+Check if tests exist:
+```bash
+sf data query --query "SELECT Id, Name FROM ApexClass WHERE Name LIKE '%Test%'" --target-org my-sandbox
+```
+
+### Permission Errors
+
+Ensure user has "Author Apex" permission and API access.
+
+### Async Test Not Completing
+
+Check system status:
+```bash
+sf apex get test \
+  --test-run-id 707xx0000000000AAA \
+  --target-org my-sandbox
+```