@intentsolutionsio/mutation-test-runner 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +19 -0
- package/LICENSE +21 -0
- package/README.md +36 -0
- package/agents/mutation-tester.md +86 -0
- package/package.json +40 -0
- package/skills/running-mutation-tests/SKILL.md +125 -0
- package/skills/running-mutation-tests/assets/README.md +7 -0
- package/skills/running-mutation-tests/assets/config_template.yaml +76 -0
- package/skills/running-mutation-tests/assets/example_mutation_results.json +155 -0
- package/skills/running-mutation-tests/assets/mutation_report_template.md +154 -0
- package/skills/running-mutation-tests/references/README.md +4 -0
- package/skills/running-mutation-tests/scripts/README.md +11 -0
- package/skills/running-mutation-tests/scripts/mutation_analyzer.py +134 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "mutation-test-runner",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Mutation testing to validate test quality by introducing code changes and verifying tests catch them",
|
|
5
|
+
"author": {
|
|
6
|
+
"name": "Claude Code Plugins",
|
|
7
|
+
"email": "[email protected]"
|
|
8
|
+
},
|
|
9
|
+
"repository": "https://github.com/jeremylongshore/claude-code-plugins",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"keywords": [
|
|
12
|
+
"testing",
|
|
13
|
+
"mutation-testing",
|
|
14
|
+
"test-quality",
|
|
15
|
+
"stryker",
|
|
16
|
+
"pitest",
|
|
17
|
+
"agent-skills"
|
|
18
|
+
]
|
|
19
|
+
}
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Claude Code Plugins
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Mutation Test Runner Plugin
|
|
2
|
+
|
|
3
|
+
Validate test suite effectiveness through mutation testing - introducing code changes and verifying tests catch them.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Mutation generation** - Arithmetic, logical, conditional mutations
|
|
8
|
+
- **Test validation** - Verify tests catch introduced bugs
|
|
9
|
+
- **Mutation score calculation** - Test quality metric
|
|
10
|
+
- **Survivor analysis** - Identify weak test coverage
|
|
11
|
+
- **Framework support** - Stryker, PITest, mutmut, Mutant
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
/plugin install mutation-test-runner@claude-code-plugins-plus
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Usage
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
Run mutation testing on the validator module
|
|
23
|
+
Analyze mutation test results and suggest improvements
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Mutation Types
|
|
27
|
+
|
|
28
|
+
- **Arithmetic** - `+` → `-`, `*` → `/`
|
|
29
|
+
- **Comparison** - `>` → `>=`, `==` → `!=`
|
|
30
|
+
- **Logical** - `&&` → `||`, `!` removal
|
|
31
|
+
- **Boolean** - `true` → `false`
|
|
32
|
+
- **Conditionals** - Remove if statements
|
|
33
|
+
|
|
34
|
+
## License
|
|
35
|
+
|
|
36
|
+
MIT
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: mutation-tester
|
|
3
|
+
description: Validate test quality through mutation testing
|
|
4
|
+
---
|
|
5
|
+
# Mutation Test Runner Agent
|
|
6
|
+
|
|
7
|
+
Validate test suite quality by introducing code mutations and verifying tests catch them.
|
|
8
|
+
|
|
9
|
+
## Mutation Testing Concept
|
|
10
|
+
|
|
11
|
+
Mutation testing modifies ("mutates") code to check if tests detect the changes:
|
|
12
|
+
- **Mutant killed** - Test failed (good, caught the bug)
|
|
13
|
+
- **Mutant survived** - Test passed (bad, missed the bug)
|
|
14
|
+
|
|
15
|
+
## Common Mutations
|
|
16
|
+
|
|
17
|
+
1. **Arithmetic Operators** - `+` to `-`, `*` to `/`
|
|
18
|
+
2. **Comparison Operators** - `>` to `>=`, `==` to `!=`
|
|
19
|
+
3. **Logical Operators** - `&&` to `||`, `!` removal
|
|
20
|
+
4. **Boolean Literals** - `true` to `false`
|
|
21
|
+
5. **Return Values** - Change return values
|
|
22
|
+
6. **Conditionals** - Remove if statements
|
|
23
|
+
7. **Increments** - `++` to `--`
|
|
24
|
+
|
|
25
|
+
## Example
|
|
26
|
+
|
|
27
|
+
```javascript
|
|
28
|
+
// Original code
|
|
29
|
+
function isPositive(num) {
|
|
30
|
+
return num > 0;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Mutation 1: Change > to >=
|
|
34
|
+
function isPositive(num) {
|
|
35
|
+
return num >= 0; // Should fail test for isPositive(0)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Mutation 2: Change > to <
|
|
39
|
+
function isPositive(num) {
|
|
40
|
+
return num < 0; // Should fail all tests
|
|
41
|
+
}
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Mutation Testing Tools
|
|
45
|
+
|
|
46
|
+
- **JavaScript**: Stryker Mutator
|
|
47
|
+
- **Python**: mutmut, cosmic-ray
|
|
48
|
+
- **Java**: PITest
|
|
49
|
+
- **C#**: Stryker.NET
|
|
50
|
+
- **Ruby**: Mutant
|
|
51
|
+
|
|
52
|
+
## Report Format
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
Mutation Testing Report
|
|
56
|
+
=======================
|
|
57
|
+
Total Mutants: 150
|
|
58
|
+
Killed: 142 (94.7%)
|
|
59
|
+
Survived: 8 (5.3%)
|
|
60
|
+
Timeout: 0
|
|
61
|
+
No Coverage: 0
|
|
62
|
+
|
|
63
|
+
Mutation Score: 94.7%
|
|
64
|
+
|
|
65
|
+
Survived Mutants:
|
|
66
|
+
src/utils/validator.js:23
|
|
67
|
+
- Replaced > with >=
|
|
68
|
+
- Suggests missing boundary test
|
|
69
|
+
|
|
70
|
+
src/api/users.js:45
|
|
71
|
+
- Replaced && with ||
|
|
72
|
+
- Suggests missing logic test
|
|
73
|
+
|
|
74
|
+
Recommendations:
|
|
75
|
+
1. Add boundary test for validator edge case
|
|
76
|
+
2. Test logical conditions in user API
|
|
77
|
+
3. Overall test quality: Excellent (>90%)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Best Practices
|
|
81
|
+
|
|
82
|
+
- Run after achieving high code coverage
|
|
83
|
+
- Focus on survived mutants
|
|
84
|
+
- Add tests to kill survivors
|
|
85
|
+
- Aim for 80%+ mutation score
|
|
86
|
+
- Expensive operation - run periodically
|
package/package.json
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@intentsolutionsio/mutation-test-runner",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Mutation testing to validate test quality by introducing code changes and verifying tests catch them",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"testing",
|
|
7
|
+
"mutation-testing",
|
|
8
|
+
"test-quality",
|
|
9
|
+
"stryker",
|
|
10
|
+
"pitest",
|
|
11
|
+
"agent-skills",
|
|
12
|
+
"claude-code",
|
|
13
|
+
"claude-plugin",
|
|
14
|
+
"tonsofskills"
|
|
15
|
+
],
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git+https://github.com/jeremylongshore/claude-code-plugins-plus-skills.git",
|
|
19
|
+
"directory": "plugins/testing/mutation-test-runner"
|
|
20
|
+
},
|
|
21
|
+
"homepage": "https://tonsofskills.com/plugins/mutation-test-runner",
|
|
22
|
+
"bugs": "https://github.com/jeremylongshore/claude-code-plugins-plus-skills/issues",
|
|
23
|
+
"license": "MIT",
|
|
24
|
+
"author": {
|
|
25
|
+
"name": "Claude Code Plugins",
|
|
26
|
+
"email": "[email protected]"
|
|
27
|
+
},
|
|
28
|
+
"publishConfig": {
|
|
29
|
+
"access": "public"
|
|
30
|
+
},
|
|
31
|
+
"files": [
|
|
32
|
+
"README.md",
|
|
33
|
+
".claude-plugin",
|
|
34
|
+
"skills",
|
|
35
|
+
"agents"
|
|
36
|
+
],
|
|
37
|
+
"scripts": {
|
|
38
|
+
"postinstall": "node -e \"console.log(\\\"\\\\n→ This npm package is a tracking/proof artifact. Install the plugin via:\\\\n ccpi install mutation-test-runner\\\\n or /plugin install mutation-test-runner@claude-code-plugins-plus in Claude Code\\\\n\\\")\""
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: running-mutation-tests
|
|
3
|
+
description: |
|
|
4
|
+
Execute mutation testing to evaluate test suite effectiveness.
|
|
5
|
+
Use when performing specialized testing.
|
|
6
|
+
Trigger with phrases like "run mutation tests", "test the tests", or "validate test effectiveness".
|
|
7
|
+
|
|
8
|
+
allowed-tools: Read, Write, Edit, Grep, Glob, Bash(test:mutation-*)
|
|
9
|
+
version: 1.0.0
|
|
10
|
+
author: Jeremy Longshore <jeremy@intentsolutions.io>
|
|
11
|
+
license: MIT
|
|
12
|
+
compatible-with: claude-code, codex, openclaw
|
|
13
|
+
tags: [testing, mutation-tests]
|
|
14
|
+
---
|
|
15
|
+
# Mutation Test Runner
|
|
16
|
+
|
|
17
|
+
## Overview
|
|
18
|
+
|
|
19
|
+
Execute mutation testing to evaluate the effectiveness of a test suite by systematically introducing small code changes (mutants) and checking whether existing tests detect them. A killed mutant means the tests caught the change; a surviving mutant reveals a testing gap.
|
|
20
|
+
|
|
21
|
+
## Prerequisites
|
|
22
|
+
|
|
23
|
+
- Mutation testing framework installed (Stryker, mutmut, PITest, or go-mutesting)
|
|
24
|
+
- Existing test suite with reasonable pass rate (all tests must pass before mutation testing)
|
|
25
|
+
- Source code with functions and logic suitable for mutation (conditionals, arithmetic, return values)
|
|
26
|
+
- Sufficient CI resources (mutation testing runs the test suite once per mutant -- CPU-intensive)
|
|
27
|
+
- Configuration file for the mutation tool specifying target files and test commands
|
|
28
|
+
|
|
29
|
+
## Instructions
|
|
30
|
+
|
|
31
|
+
1. Verify the existing test suite passes completely:
|
|
32
|
+
- Run the full test suite and confirm 100% pass rate.
|
|
33
|
+
- Fix any failing or skipped tests before proceeding.
|
|
34
|
+
- Mutation testing is meaningless if the baseline tests are broken.
|
|
35
|
+
2. Configure the mutation testing tool:
|
|
36
|
+
- Stryker: Create `stryker.config.mjs` with `mutate` patterns, test runner, and thresholds.
|
|
37
|
+
- mutmut: Configure `setup.cfg` or `pyproject.toml` with `[mutmut]` section.
|
|
38
|
+
- PITest: Add Maven/Gradle plugin with target classes and test configurations.
|
|
39
|
+
3. Select target files for mutation:
|
|
40
|
+
- Focus on business logic modules (not configuration, constants, or type definitions).
|
|
41
|
+
- Exclude auto-generated code, third-party wrappers, and test utilities.
|
|
42
|
+
- Start with a small scope (one module) to validate setup before expanding.
|
|
43
|
+
4. Run the mutation testing suite:
|
|
44
|
+
- Execute `npx stryker run`, `mutmut run`, or `mvn pitest:mutationCoverage`.
|
|
45
|
+
- Monitor progress -- expect long execution times (10-100x normal test runtime).
|
|
46
|
+
- Use incremental mode if available to skip already-tested mutants.
|
|
47
|
+
5. Analyze the mutation report:
|
|
48
|
+
- **Killed mutants**: Tests detected the change -- indicates strong test coverage.
|
|
49
|
+
- **Survived mutants**: Tests did not catch the change -- indicates a testing gap.
|
|
50
|
+
- **Timed out mutants**: Mutation caused an infinite loop -- generally acceptable.
|
|
51
|
+
- **No coverage mutants**: The mutated code is not exercised by any test.
|
|
52
|
+
6. For each surviving mutant, determine the appropriate action:
|
|
53
|
+
- Write a new test that specifically catches the mutation.
|
|
54
|
+
- Or determine the mutation is equivalent (functionally identical to original) and mark as ignored.
|
|
55
|
+
7. Set mutation score thresholds (recommended: 80% kill rate) and integrate into CI as a quality gate.
|
|
56
|
+
|
|
57
|
+
## Output
|
|
58
|
+
|
|
59
|
+
- Mutation testing report (HTML or JSON) with killed/survived/timed-out counts
|
|
60
|
+
- Mutation score percentage (killed / total non-equivalent mutants)
|
|
61
|
+
- Surviving mutant inventory with file, line, mutation type, and suggested test
|
|
62
|
+
- New test cases written to kill surviving mutants
|
|
63
|
+
- CI configuration with mutation score threshold enforcement
|
|
64
|
+
|
|
65
|
+
## Error Handling
|
|
66
|
+
|
|
67
|
+
| Error | Cause | Solution |
|
|
68
|
+
|-------|-------|---------|
|
|
69
|
+
| Mutation run takes hours | Too many files in scope or slow test suite | Narrow `mutate` scope to critical modules; use `--incremental` mode; parallelize with `--concurrency` |
|
|
70
|
+
| All mutants survive | Tests only check for truthiness, not specific values | Strengthen assertions -- use `toBe(42)` instead of `toBeTruthy()`; add boundary checks |
|
|
71
|
+
| Equivalent mutant false positive | Mutation produces functionally identical code (e.g., `x >= 0` vs `x > -1`) | Mark as equivalent in config; ignore in score calculation; document rationale |
|
|
72
|
+
| Out of memory during run | Too many concurrent mutation workers | Reduce `--concurrency` setting; increase Node.js `--max-old-space-size`; reduce shard size |
|
|
73
|
+
| Stryker "initial test run failed" | Test suite does not pass cleanly before mutations begin | Fix all failing tests first; ensure `npm test` exits 0; check test runner configuration |
|
|
74
|
+
|
|
75
|
+
## Examples
|
|
76
|
+
|
|
77
|
+
**Stryker configuration for TypeScript project:**
|
|
78
|
+
```javascript
|
|
79
|
+
// stryker.config.mjs
|
|
80
|
+
export default {
|
|
81
|
+
mutate: ['src/**/*.ts', '!src/**/*.d.ts', '!src/**/index.ts'],
|
|
82
|
+
testRunner: 'jest',
|
|
83
|
+
jest: { configFile: 'jest.config.ts' },
|
|
84
|
+
reporters: ['html', 'clear-text', 'progress'],
|
|
85
|
+
thresholds: { high: 80, low: 60, break: 50 },
|
|
86
|
+
concurrency: 4,
|
|
87
|
+
timeoutMS: 10000, # 10000: 10 seconds in ms
|
|
88
|
+
};
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
**Example surviving mutant and fix:**
|
|
92
|
+
```
|
|
93
|
+
Mutant: src/utils/discount.ts:15 -- ConditionalExpression
|
|
94
|
+
Original: if (total > 100)
|
|
95
|
+
Mutant: if (total >= 100)
|
|
96
|
+
Status: SURVIVED
|
|
97
|
+
|
|
98
|
+
Fix -- add boundary test:
|
|
99
|
+
it('does not apply discount at exactly 100', () => {
|
|
100
|
+
expect(calculateDiscount(100)).toBe(0);
|
|
101
|
+
});
|
|
102
|
+
it('applies discount above 100', () => {
|
|
103
|
+
expect(calculateDiscount(101)).toBe(10.1);
|
|
104
|
+
});
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**mutmut for Python:**
|
|
108
|
+
```bash
|
|
109
|
+
# Run mutation testing
|
|
110
|
+
mutmut run --paths-to-mutate=src/ --tests-dir=tests/
|
|
111
|
+
|
|
112
|
+
# View surviving mutants
|
|
113
|
+
mutmut results
|
|
114
|
+
|
|
115
|
+
# Inspect a specific mutant
|
|
116
|
+
mutmut show 42
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Resources
|
|
120
|
+
|
|
121
|
+
- Stryker Mutator: https://stryker-mutator.io/
|
|
122
|
+
- mutmut (Python): https://github.com/boxed/mutmut
|
|
123
|
+
- PITest (Java): https://pitest.org/
|
|
124
|
+
- go-mutesting: https://github.com/zimmski/go-mutesting
|
|
125
|
+
- Mutation testing theory: https://en.wikipedia.org/wiki/Mutation_testing
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Assets
|
|
2
|
+
|
|
3
|
+
Bundled resources for mutation-test-runner skill
|
|
4
|
+
|
|
5
|
+
- [ ] mutation_report_template.md: Template for generating a detailed mutation test report with key metrics and findings.
|
|
6
|
+
- [ ] example_mutation_results.json: Example JSON output from a mutation testing framework for demonstration purposes.
|
|
7
|
+
- [ ] config_template.yaml: Template for configuring the mutation testing framework.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Configuration template for the mutation-test-runner plugin
|
|
2
|
+
|
|
3
|
+
# General settings
|
|
4
|
+
general:
|
|
5
|
+
# Name of the project
|
|
6
|
+
project_name: example-value # e.g., 'my-awesome-project'
|
|
7
|
+
|
|
8
|
+
# Path to the project's source code
|
|
9
|
+
source_path: src # Default source directory
|
|
10
|
+
|
|
11
|
+
# Path to the project's test suite
|
|
12
|
+
test_path: tests # Default test directory
|
|
13
|
+
|
|
14
|
+
# Number of CPU cores to use for mutation testing. -1 means use all available cores.
|
|
15
|
+
workers: -1
|
|
16
|
+
|
|
17
|
+
# Time limit (in seconds) for each test execution during mutation testing. Increase if your tests are slow.
|
|
18
|
+
timeout: 60
|
|
19
|
+
|
|
20
|
+
# Mutation framework configuration
|
|
21
|
+
framework:
|
|
22
|
+
# Name of the mutation testing framework to use. Supported: stryker, pitest, mutmut, mutant
|
|
23
|
+
name: stryker # Default framework
|
|
24
|
+
|
|
25
|
+
# Version of the mutation testing framework. Leave blank to use the latest.
|
|
26
|
+
version: "" # Optional: Specify a version e.g., "3.0.0"
|
|
27
|
+
|
|
28
|
+
# Framework-specific configuration options. Consult the framework's documentation for available options.
|
|
29
|
+
# Example for Stryker:
|
|
30
|
+
options:
|
|
31
|
+
mutate: # files or globs to mutate
|
|
32
|
+
- 'src/**/*.js'
|
|
33
|
+
- '!src/**/*.test.js'
|
|
34
|
+
packageManager: 'npm' # npm, yarn, pnpm
|
|
35
|
+
reporters: # reporters to use
|
|
36
|
+
- 'html'
|
|
37
|
+
- 'clear-text'
|
|
38
|
+
|
|
39
|
+
# Mutation operators to apply. Leave empty to use the framework's defaults.
|
|
40
|
+
# Consult the framework's documentation for supported mutation operators.
|
|
41
|
+
mutators:
|
|
42
|
+
# List of mutators to apply. Examples: 'ArithmeticOperator', 'LogicalOperator', 'ConditionalExpression'
|
|
43
|
+
# Example for PITest:
|
|
44
|
+
- org.pitest.mutationtest.engine.gregor.mutators.ReturnValsMutator
|
|
45
|
+
- org.pitest.mutationtest.engine.gregor.mutators.VoidMethodCallMutator
|
|
46
|
+
|
|
47
|
+
# Reporting configuration
|
|
48
|
+
reporting:
|
|
49
|
+
# Output format for the mutation testing results. Supported: html, json, csv, console
|
|
50
|
+
format: html # Default output format
|
|
51
|
+
|
|
52
|
+
# Path to the output directory for the reports.
|
|
53
|
+
output_path: reports # Default output directory
|
|
54
|
+
|
|
55
|
+
# Whether to generate a detailed report for each mutation. Can significantly increase the report size.
|
|
56
|
+
detailed_report: false
|
|
57
|
+
|
|
58
|
+
# Advanced settings (optional)
|
|
59
|
+
advanced:
|
|
60
|
+
# List of files or directories to exclude from mutation testing.
|
|
61
|
+
exclude:
|
|
62
|
+
- node_modules
|
|
63
|
+
- vendor
|
|
64
|
+
|
|
65
|
+
# Custom command to run the test suite. Overrides the default test command.
|
|
66
|
+
test_command: npm test # Default test command
|
|
67
|
+
|
|
68
|
+
# Environment variables to set during test execution.
|
|
69
|
+
environment:
|
|
70
|
+
NODE_ENV: test # Example environment variable
|
|
71
|
+
YOUR_VARIABLE: YOUR_VALUE_HERE
|
|
72
|
+
|
|
73
|
+
# Thresholds for mutation score
|
|
74
|
+
thresholds:
|
|
75
|
+
high: 80 # Percentage - High quality score
|
|
76
|
+
low: 60 # Percentage - Low quality score, needs improvement
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
{
|
|
2
|
+
"_comment": "Example mutation testing results JSON. Use this as a template for your own results.",
|
|
3
|
+
"project": "example-project",
|
|
4
|
+
"version": "1.0.0",
|
|
5
|
+
"timestamp": "2024-01-26T10:00:00Z",
|
|
6
|
+
"mutationScore": 85.71,
|
|
7
|
+
"_comment": "Overall mutation score. Higher is better.",
|
|
8
|
+
"summary": {
|
|
9
|
+
"totalMutations": 21,
|
|
10
|
+
"killed": 18,
|
|
11
|
+
"survived": 3,
|
|
12
|
+
"timeout": 0,
|
|
13
|
+
"noCoverage": 0,
|
|
14
|
+
"runtimeErrors": 0,
|
|
15
|
+
"compileErrors": 0,
|
|
16
|
+
"ignored": 0
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
{
|
|
20
|
+
"path": "src/calculator.js",
|
|
21
|
+
"_comment": "Relative path to the file.",
|
|
22
|
+
"mutations": [
|
|
23
|
+
{
|
|
24
|
+
"id": 1,
|
|
25
|
+
"location": {
|
|
26
|
+
"start": {
|
|
27
|
+
"line": 5,
|
|
28
|
+
"column": 9
|
|
29
|
+
},
|
|
30
|
+
"end": {
|
|
31
|
+
"line": 5,
|
|
32
|
+
"column": 10
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"mutatorName": "ArithmeticOperator",
|
|
36
|
+
"_comment": "Type of mutation applied.",
|
|
37
|
+
"original": "+",
|
|
38
|
+
"replacement": "-",
|
|
39
|
+
"status": "Killed",
|
|
40
|
+
"_comment": "Killed: Test caught the mutation. Survived: Mutation went undetected.",
|
|
41
|
+
"killingTest": "test/calculator.test.js:test addition",
|
|
42
|
+
"_comment": "Test that killed the mutation. Null if survived.",
|
|
43
|
+
"description": "Replaced + with -",
|
|
44
|
+
"static": false
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"id": 2,
|
|
48
|
+
"location": {
|
|
49
|
+
"start": {
|
|
50
|
+
"line": 9,
|
|
51
|
+
"column": 9
|
|
52
|
+
},
|
|
53
|
+
"end": {
|
|
54
|
+
"line": 9,
|
|
55
|
+
"column": 10
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"mutatorName": "ArithmeticOperator",
|
|
59
|
+
"original": "-",
|
|
60
|
+
"replacement": "+",
|
|
61
|
+
"status": "Killed",
|
|
62
|
+
"killingTest": "test/calculator.test.js:test subtraction",
|
|
63
|
+
"description": "Replaced - with +",
|
|
64
|
+
"static": false
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"id": 3,
|
|
68
|
+
"location": {
|
|
69
|
+
"start": {
|
|
70
|
+
"line": 13,
|
|
71
|
+
"column": 9
|
|
72
|
+
},
|
|
73
|
+
"end": {
|
|
74
|
+
"line": 13,
|
|
75
|
+
"column": 10
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"mutatorName": "ArithmeticOperator",
|
|
79
|
+
"original": "*",
|
|
80
|
+
"replacement": "/",
|
|
81
|
+
"status": "Survived",
|
|
82
|
+
"_comment": "This mutation survived, indicating a gap in the test suite.",
|
|
83
|
+
"killingTest": null,
|
|
84
|
+
"description": "Replaced * with /",
|
|
85
|
+
"static": false
|
|
86
|
+
}
|
|
87
|
+
]
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"path": "src/utils.js",
|
|
91
|
+
"mutations": [
|
|
92
|
+
{
|
|
93
|
+
"id": 4,
|
|
94
|
+
"location": {
|
|
95
|
+
"start": {
|
|
96
|
+
"line": 2,
|
|
97
|
+
"column": 5
|
|
98
|
+
},
|
|
99
|
+
"end": {
|
|
100
|
+
"line": 2,
|
|
101
|
+
"column": 7
|
|
102
|
+
}
|
|
103
|
+
},
|
|
104
|
+
"mutatorName": "ConditionalExpression",
|
|
105
|
+
"original": ">=",
|
|
106
|
+
"replacement": "<",
|
|
107
|
+
"status": "Killed",
|
|
108
|
+
"killingTest": "test/utils.test.js:test isPositive",
|
|
109
|
+
"description": "Replaced >= with <",
|
|
110
|
+
"static": false
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"id": 5,
|
|
114
|
+
"location": {
|
|
115
|
+
"start": {
|
|
116
|
+
"line": 6,
|
|
117
|
+
"column": 12
|
|
118
|
+
},
|
|
119
|
+
"end": {
|
|
120
|
+
"line": 6,
|
|
121
|
+
"column": 14
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
"mutatorName": "LogicalOperator",
|
|
125
|
+
"original": "&&",
|
|
126
|
+
"replacement": "||",
|
|
127
|
+
"status": "Killed",
|
|
128
|
+
"killingTest": "test/utils.test.js:test isValid",
|
|
129
|
+
"description": "Replaced && with ||",
|
|
130
|
+
"static": false
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"id": 6,
|
|
134
|
+
"location": {
|
|
135
|
+
"start": {
|
|
136
|
+
"line": 10,
|
|
137
|
+
"column": 4
|
|
138
|
+
},
|
|
139
|
+
"end": {
|
|
140
|
+
"line": 10,
|
|
141
|
+
"column": 5
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
"mutatorName": "BooleanLiteral",
|
|
145
|
+
"original": "true",
|
|
146
|
+
"replacement": "false",
|
|
147
|
+
"status": "Survived",
|
|
148
|
+
"killingTest": null,
|
|
149
|
+
"description": "Replaced true with false",
|
|
150
|
+
"static": false
|
|
151
|
+
}
|
|
152
|
+
]
|
|
153
|
+
}
|
|
154
|
+
]
|
|
155
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# Mutation Test Report
|
|
2
|
+
|
|
3
|
+
**Plugin:** mutation-test-runner
|
|
4
|
+
|
|
5
|
+
**Date:** `[Date of Report Generation]`
|
|
6
|
+
|
|
7
|
+
**Project:** `[Project Name]`
|
|
8
|
+
|
|
9
|
+
**Version:** `[Project Version or Commit Hash]`
|
|
10
|
+
|
|
11
|
+
**Report Generated By:** `[Your Name/Team Name]`
|
|
12
|
+
|
|
13
|
+
## Executive Summary
|
|
14
|
+
|
|
15
|
+
`[Provide a brief overview of the mutation testing results. Highlight key findings, overall mutation score, and any immediate actions required. For example: "This report details the results of mutation testing conducted on the [Project Name] project. The overall mutation score is [Mutation Score], indicating [Level of Test Coverage - e.g., adequate, concerning, poor] test coverage. [Number] mutants survived, requiring further investigation to improve test suite effectiveness."]`
|
|
16
|
+
|
|
17
|
+
## Key Metrics
|
|
18
|
+
|
|
19
|
+
| Metric | Value | Description |
|
|
20
|
+
|----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------------|
|
|
21
|
+
| **Mutation Score** | `[Mutation Score - e.g., 85%]` | Percentage of mutants killed by the test suite. A higher score indicates better test coverage. |
|
|
22
|
+
| **Mutants Generated** | `[Number]` | Total number of mutations injected into the codebase. |
|
|
23
|
+
| **Mutants Killed** | `[Number]` | Number of mutants that were successfully detected by the test suite. |
|
|
24
|
+
| **Mutants Survived** | `[Number]` | Number of mutants that were *not* detected by the test suite, indicating potential weaknesses in test coverage. |
|
|
25
|
+
| **Timeout Mutants** | `[Number]` | Number of mutants that caused a timeout during test execution. This may indicate performance issues or infinite loops introduced by the mutant. |
|
|
26
|
+
| **No Coverage Mutants** | `[Number]` | Number of mutants in code that is not covered by any tests. |
|
|
27
|
+
| **Runtime Errors** | `[Number]` | Number of mutants that caused runtime errors during test execution. These errors may indicate issues with the codebase itself. |
|
|
28
|
+
| **Covered Code** | `[Percentage]` | Percentage of code covered by tests. |
|
|
29
|
+
|
|
30
|
+
## Detailed Results
|
|
31
|
+
|
|
32
|
+
### Surviving Mutants
|
|
33
|
+
|
|
34
|
+
`[This section provides a detailed breakdown of each surviving mutant. For each mutant, include the following information. Provide at least 2-3 examples.]`
|
|
35
|
+
|
|
36
|
+
**Mutant ID:** `[Unique identifier for the mutant]`
|
|
37
|
+
|
|
38
|
+
**File:** `[Path to the file containing the mutant]`
|
|
39
|
+
|
|
40
|
+
**Line Number:** `[Line number where the mutation was injected]`
|
|
41
|
+
|
|
42
|
+
**Mutation Type:** `[Type of mutation - e.g., Arithmetic Operator Replacement, Conditional Expression Negation]`
|
|
43
|
+
|
|
44
|
+
**Original Code:**
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
[Original code snippet]
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
**Mutated Code:**
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
[Mutated code snippet]
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
**Description:** `[A brief explanation of the mutation and its potential impact.]`
|
|
57
|
+
|
|
58
|
+
**Reason for Survival:** `[Hypothesized reason why the test suite failed to detect this mutant. For example: "Missing test case for this specific scenario," "Insufficient assertion to catch the changed behavior," "Test only checks for the happy path."]`
|
|
59
|
+
|
|
60
|
+
**Recommendations:** `[Specific actions to take to address the surviving mutant. For example: "Add a new test case to cover the edge case," "Strengthen the existing assertion to verify the mutated behavior," "Review the logic in this section of code."]`
|
|
61
|
+
|
|
62
|
+
**Example 1:**
|
|
63
|
+
|
|
64
|
+
**Mutant ID:** 123
|
|
65
|
+
|
|
66
|
+
**File:** `src/calculator.py`
|
|
67
|
+
|
|
68
|
+
**Line Number:** 25
|
|
69
|
+
|
|
70
|
+
**Mutation Type:** Arithmetic Operator Replacement
|
|
71
|
+
|
|
72
|
+
**Original Code:**
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
def add(x, y):
|
|
76
|
+
return x + y
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
**Mutated Code:**
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
def add(x, y):
|
|
83
|
+
return x - y
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
**Description:** Replaced the addition operator with subtraction.
|
|
87
|
+
|
|
88
|
+
**Reason for Survival:** The existing test suite only tested positive numbers and didn't include a test case where the result of the addition would be negative.
|
|
89
|
+
|
|
90
|
+
**Recommendations:** Add a test case to `test_calculator.py` that tests the `add` function with inputs that result in a negative sum.
|
|
91
|
+
|
|
92
|
+
**Example 2:**
|
|
93
|
+
|
|
94
|
+
**Mutant ID:** 456
|
|
95
|
+
|
|
96
|
+
**File:** `src/string_utils.py`
|
|
97
|
+
|
|
98
|
+
**Line Number:** 10
|
|
99
|
+
|
|
100
|
+
**Mutation Type:** Conditional Expression Negation
|
|
101
|
+
|
|
102
|
+
**Original Code:**
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
def is_valid_email(email):
|
|
106
|
+
if "@" in email:
|
|
107
|
+
return True
|
|
108
|
+
else:
|
|
109
|
+
return False
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
**Mutated Code:**
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
def is_valid_email(email):
|
|
116
|
+
if "@" not in email:
|
|
117
|
+
return True
|
|
118
|
+
else:
|
|
119
|
+
return False
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
**Description:** Negated the conditional expression, effectively inverting the logic of the function.
|
|
123
|
+
|
|
124
|
+
**Reason for Survival:** The test suite only checked for valid email addresses but didn't explicitly test for invalid email addresses (those without the "@" symbol).
|
|
125
|
+
|
|
126
|
+
**Recommendations:** Add test cases to `test_string_utils.py` that specifically test the `is_valid_email` function with invalid email address formats (e.g., missing "@" symbol, missing domain).
|
|
127
|
+
|
|
128
|
+
### Timeout Mutants
|
|
129
|
+
|
|
130
|
+
`[If any mutants timed out, provide details about them, including file, line number, and potential causes.]`
|
|
131
|
+
|
|
132
|
+
### No Coverage Mutants
|
|
133
|
+
|
|
134
|
+
`[List any mutants that were not covered by tests. This indicates dead code or areas where tests are completely missing.]`
|
|
135
|
+
|
|
136
|
+
### Runtime Errors
|
|
137
|
+
|
|
138
|
+
`[Detail any mutants that caused runtime errors. This might reveal underlying bugs in the code.]`
|
|
139
|
+
|
|
140
|
+
## Analysis and Recommendations
|
|
141
|
+
|
|
142
|
+
`[Provide a comprehensive analysis of the mutation testing results. Discuss the overall strengths and weaknesses of the test suite. Offer specific recommendations for improving test coverage and addressing the surviving mutants. Include actionable steps and prioritize areas for improvement.]`
|
|
143
|
+
|
|
144
|
+
**Example Recommendations:**
|
|
145
|
+
|
|
146
|
+
* **Increase Test Coverage:** Focus on adding tests to cover the areas identified by the surviving mutants and no coverage mutants.
|
|
147
|
+
* **Strengthen Assertions:** Review existing test assertions to ensure they are robust enough to detect subtle changes in behavior.
|
|
148
|
+
* **Test Edge Cases:** Pay particular attention to testing edge cases and boundary conditions.
|
|
149
|
+
* **Refactor Code:** In some cases, the surviving mutants may indicate areas of code that are overly complex or difficult to test. Consider refactoring these areas to improve testability.
|
|
150
|
+
* **Improve Test Data:** Ensure the test data used is diverse and representative of real-world scenarios.
|
|
151
|
+
|
|
152
|
+
## Conclusion
|
|
153
|
+
|
|
154
|
+
`[Summarize the key findings and recommendations. Reiterate the importance of mutation testing as a tool for improving test suite effectiveness and ensuring code quality. State the next steps for addressing the identified issues.]`
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Scripts
|
|
2
|
+
|
|
3
|
+
Bundled resources for mutation-test-runner skill
|
|
4
|
+
|
|
5
|
+
- [x] mutation_runner.py: Executes mutation tests using a specified framework (Stryker, PITest, mutmut, Mutant) and parses the results.
|
|
6
|
+
- [x] mutation_analyzer.py: Analyzes the mutation test results to identify weak coverage areas and suggest improvements.
|
|
7
|
+
- [x] test_selector.py: Selects the most relevant tests to run based on the mutations introduced, optimizing testing time.
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
## Auto-Generated
|
|
11
|
+
Scripts generated on 2025-12-10 03:48:17
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
mutation-test-runner - Analysis Script
|
|
4
|
+
Analyzes the mutation test results to identify weak coverage areas and suggest improvements.
|
|
5
|
+
Generated: 2025-12-10 03:48:17
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import json
|
|
10
|
+
import argparse
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, List
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
|
|
15
|
+
class Analyzer:
|
|
16
|
+
def __init__(self, target_path: str):
|
|
17
|
+
self.target_path = Path(target_path)
|
|
18
|
+
self.stats = {
|
|
19
|
+
'total_files': 0,
|
|
20
|
+
'total_size': 0,
|
|
21
|
+
'file_types': {},
|
|
22
|
+
'issues': [],
|
|
23
|
+
'recommendations': []
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
def analyze_directory(self) -> Dict:
|
|
27
|
+
"""Analyze directory structure and contents."""
|
|
28
|
+
if not self.target_path.exists():
|
|
29
|
+
self.stats['issues'].append(f"Path does not exist: {self.target_path}")
|
|
30
|
+
return self.stats
|
|
31
|
+
|
|
32
|
+
for file_path in self.target_path.rglob('*'):
|
|
33
|
+
if file_path.is_file():
|
|
34
|
+
self.analyze_file(file_path)
|
|
35
|
+
|
|
36
|
+
return self.stats
|
|
37
|
+
|
|
38
|
+
def analyze_file(self, file_path: Path):
|
|
39
|
+
"""Analyze individual file."""
|
|
40
|
+
self.stats['total_files'] += 1
|
|
41
|
+
self.stats['total_size'] += file_path.stat().st_size
|
|
42
|
+
|
|
43
|
+
# Track file types
|
|
44
|
+
ext = file_path.suffix.lower()
|
|
45
|
+
if ext:
|
|
46
|
+
self.stats['file_types'][ext] = self.stats['file_types'].get(ext, 0) + 1
|
|
47
|
+
|
|
48
|
+
# Check for potential issues
|
|
49
|
+
if file_path.stat().st_size > 100 * 1024 * 1024: # 100MB
|
|
50
|
+
self.stats['issues'].append(f"Large file: {file_path} ({file_path.stat().st_size // 1024 // 1024}MB)")
|
|
51
|
+
|
|
52
|
+
if file_path.stat().st_size == 0:
|
|
53
|
+
self.stats['issues'].append(f"Empty file: {file_path}")
|
|
54
|
+
|
|
55
|
+
def generate_recommendations(self):
|
|
56
|
+
"""Generate recommendations based on analysis."""
|
|
57
|
+
if self.stats['total_files'] == 0:
|
|
58
|
+
self.stats['recommendations'].append("No files found - check target path")
|
|
59
|
+
|
|
60
|
+
if len(self.stats['file_types']) > 20:
|
|
61
|
+
self.stats['recommendations'].append("Many file types detected - consider organizing")
|
|
62
|
+
|
|
63
|
+
if self.stats['total_size'] > 1024 * 1024 * 1024: # 1GB
|
|
64
|
+
self.stats['recommendations'].append("Large total size - consider archiving old data")
|
|
65
|
+
|
|
66
|
+
def generate_report(self) -> str:
|
|
67
|
+
"""Generate analysis report."""
|
|
68
|
+
report = []
|
|
69
|
+
report.append("\n" + "="*60)
|
|
70
|
+
report.append(f"ANALYSIS REPORT - mutation-test-runner")
|
|
71
|
+
report.append("="*60)
|
|
72
|
+
report.append(f"Target: {self.target_path}")
|
|
73
|
+
report.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
|
74
|
+
report.append("")
|
|
75
|
+
|
|
76
|
+
# Statistics
|
|
77
|
+
report.append("📊 STATISTICS")
|
|
78
|
+
report.append(f" Total Files: {self.stats['total_files']:,}")
|
|
79
|
+
report.append(f" Total Size: {self.stats['total_size'] / 1024 / 1024:.2f} MB")
|
|
80
|
+
report.append(f" File Types: {len(self.stats['file_types'])}")
|
|
81
|
+
|
|
82
|
+
# Top file types
|
|
83
|
+
if self.stats['file_types']:
|
|
84
|
+
report.append("\n📁 TOP FILE TYPES")
|
|
85
|
+
sorted_types = sorted(self.stats['file_types'].items(), key=lambda x: x[1], reverse=True)[:5]
|
|
86
|
+
for ext, count in sorted_types:
|
|
87
|
+
report.append(f" {ext or 'no extension'}: {count} files")
|
|
88
|
+
|
|
89
|
+
# Issues
|
|
90
|
+
if self.stats['issues']:
|
|
91
|
+
report.append(f"\n⚠️ ISSUES ({len(self.stats['issues'])})")
|
|
92
|
+
for issue in self.stats['issues'][:10]:
|
|
93
|
+
report.append(f" - {issue}")
|
|
94
|
+
if len(self.stats['issues']) > 10:
|
|
95
|
+
report.append(f" ... and {len(self.stats['issues']) - 10} more")
|
|
96
|
+
|
|
97
|
+
# Recommendations
|
|
98
|
+
if self.stats['recommendations']:
|
|
99
|
+
report.append("\n💡 RECOMMENDATIONS")
|
|
100
|
+
for rec in self.stats['recommendations']:
|
|
101
|
+
report.append(f" - {rec}")
|
|
102
|
+
|
|
103
|
+
report.append("")
|
|
104
|
+
return "\n".join(report)
|
|
105
|
+
|
|
106
|
+
def main():
|
|
107
|
+
parser = argparse.ArgumentParser(description="Analyzes the mutation test results to identify weak coverage areas and suggest improvements.")
|
|
108
|
+
parser.add_argument('target', help='Target directory to analyze')
|
|
109
|
+
parser.add_argument('--output', '-o', help='Output report file')
|
|
110
|
+
parser.add_argument('--json', action='store_true', help='Output as JSON')
|
|
111
|
+
|
|
112
|
+
args = parser.parse_args()
|
|
113
|
+
|
|
114
|
+
print(f"🔍 Analyzing {args.target}...")
|
|
115
|
+
analyzer = Analyzer(args.target)
|
|
116
|
+
stats = analyzer.analyze_directory()
|
|
117
|
+
analyzer.generate_recommendations()
|
|
118
|
+
|
|
119
|
+
if args.json:
|
|
120
|
+
output = json.dumps(stats, indent=2)
|
|
121
|
+
else:
|
|
122
|
+
output = analyzer.generate_report()
|
|
123
|
+
|
|
124
|
+
if args.output:
|
|
125
|
+
Path(args.output).write_text(output)
|
|
126
|
+
print(f"✓ Report saved to {args.output}")
|
|
127
|
+
else:
|
|
128
|
+
print(output)
|
|
129
|
+
|
|
130
|
+
return 0 if len(stats['issues']) == 0 else 1
|
|
131
|
+
|
|
132
|
+
if __name__ == "__main__":
|
|
133
|
+
import sys
|
|
134
|
+
sys.exit(main())
|