mutaruby 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +30 -0
- data/LICENSE +21 -0
- data/README.md +398 -0
- data/exe/mutaruby +6 -0
- data/lib/mutaruby/cli.rb +159 -0
- data/lib/mutaruby/command.rb +101 -0
- data/lib/mutaruby/generator.rb +336 -0
- data/lib/mutaruby/ignore_file.rb +131 -0
- data/lib/mutaruby/mutation.rb +53 -0
- data/lib/mutaruby/parsing.rb +48 -0
- data/lib/mutaruby/reporters.rb +103 -0
- data/lib/mutaruby/runner.rb +181 -0
- data/lib/mutaruby/sandbox.rb +82 -0
- data/lib/mutaruby/selection.rb +107 -0
- data/lib/mutaruby/splicer.rb +27 -0
- data/lib/mutaruby/version.rb +5 -0
- data/lib/mutaruby.rb +19 -0
- metadata +84 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: ed2be021810bf506056045c9de11c188af27c0f77ba28c9cef6c124265b13aaf
|
|
4
|
+
data.tar.gz: a0789ce2a2707894ed251c13641f689d6d2456e067ce830f366d5bdb4c89cdbe
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: df30932c81a94875c7aa082c6842d78593e253febc1eade648780946266cc5207ca1293a7e0b507919ead9be542600a15a6764be45fad7034504bed593a27aaf
|
|
7
|
+
data.tar.gz: 0f9a9fc9d2a3a1eb68e6c442dcf65ff9b3f3cf692358ca1af18010e53d93a1cef8630d7652af7182cf31cd6e783910f08ed7ffe90fb028bfe1c015f864a1eaf6
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [Semantic Versioning](https://semver.org/).
|
|
5
|
+
|
|
6
|
+
## [0.1.0] - 2026-10-11
|
|
7
|
+
|
|
8
|
+
First release.
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- AST-based mutant generation with the `parser` gem and 36 documented operators: arithmetic, comparison and
|
|
13
|
+
bitwise swaps; `&&`/`||` swap and operand drops; negation and unary-minus removal; condition negate/force true/force
|
|
14
|
+
false (including loops); integer, float, string, symbol, boolean, nil, array and hash literals; range swap; return
|
|
15
|
+
value and method-body replacement; statement deletion; argument removal; call-to-receiver and call-to-argument;
|
|
16
|
+
predicate forcing; safe-navigation removal; `raise` removal; compound, `||=` and `&&=` assignments.
|
|
17
|
+
- Any shell command as the test oracle (exit 0 = survived, non-zero = killed).
|
|
18
|
+
- Each mutant runs in a fresh process inside a per-worker copy of the project; the working tree is never written.
|
|
19
|
+
- Baseline check: the unmodified code must pass the command first.
|
|
20
|
+
- Per-mutant timeout (default derived from the baseline), counted as a kill and reported separately; the whole
|
|
21
|
+
process group is killed.
|
|
22
|
+
- Splicing at character offsets with a self-check that aborts on any mismatch between the replaced slice and the
|
|
23
|
+
node's own source.
|
|
24
|
+
- Parallel workers (`--jobs`), file and glob targets, `--since REF` for changed files, include/exclude by method and
|
|
25
|
+
by line, `--operators`/`--exclude-operators`, `--shard I/N`, and `# mutaruby:disable` comments.
|
|
26
|
+
- Ignore file for equivalent mutants, keyed by site (file, operator, line text, occurrence, method), with a required
|
|
27
|
+
reason, ambiguity and staleness checks, `--strict-ignore` and `--suggest-ignore`.
|
|
28
|
+
- Human, JSON and JUnit XML reports, a mutation score and `--fail-under` for CI.
|
|
29
|
+
|
|
30
|
+
[0.1.0]: https://github.com/Atlas-Authorization/mutaruby/releases/tag/v0.1.0
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Atlas Authorization
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
# mutaruby
|
|
2
|
+
|
|
3
|
+
Free, MIT-licensed mutation testing for Ruby.
|
|
4
|
+
|
|
5
|
+
mutaruby makes small, deliberate changes to your code, one at a time, and runs your test command against each change.
|
|
6
|
+
If your tests still pass, the change (the *mutant*) **survived**, and that points to code your tests do not really check.
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
$ mutaruby -t "bundle exec rspec"
|
|
10
|
+
.......S.........T......e....
|
|
11
|
+
|
|
12
|
+
Surviving mutants (not detected by the test command):
|
|
13
|
+
|
|
14
|
+
lib/pricing.rb:14 in Pricing#discount [#7]
|
|
15
|
+
comparison: > => >=
|
|
16
|
+
- return 0 if total > 100
|
|
17
|
+
+ return 0 if total >= 100
|
|
18
|
+
|
|
19
|
+
mutaruby summary
|
|
20
|
+
files: 3
|
|
21
|
+
mutants: 29
|
|
22
|
+
killed: 27 (1 by timeout)
|
|
23
|
+
survived: 1
|
|
24
|
+
equivalent: 1 (ignore file)
|
|
25
|
+
score: 96.43% = killed / (mutants - equivalent)
|
|
26
|
+
time: 11.2s (baseline 0.84s, timeout 8s, jobs 8)
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The survivor above tells you that no test checks a total of exactly 100.
|
|
30
|
+
|
|
31
|
+
## Contents
|
|
32
|
+
|
|
33
|
+
- [What mutation testing is](#what-mutation-testing-is)
|
|
34
|
+
- [Install](#install)
|
|
35
|
+
- [Quickstart](#quickstart)
|
|
36
|
+
- [CLI reference](#cli-reference)
|
|
37
|
+
- [Mutation operators](#mutation-operators)
|
|
38
|
+
- [Equivalent mutants: the ignore file](#equivalent-mutants-the-ignore-file)
|
|
39
|
+
- [Using it in CI](#using-it-in-ci)
|
|
40
|
+
- [Output formats](#output-formats)
|
|
41
|
+
- [How it compares with mutant](#how-it-compares-with-mutant)
|
|
42
|
+
- [Design notes](#design-notes)
|
|
43
|
+
- [Limitations](#limitations)
|
|
44
|
+
- [Contributing, security, licence](#contributing-security-licence)
|
|
45
|
+
|
|
46
|
+
## What mutation testing is
|
|
47
|
+
|
|
48
|
+
Line coverage tells you which code your tests *ran*. It does not tell you whether they would *notice* if that code were
|
|
49
|
+
wrong. A test that calls `discount(150)` and only checks that it returns something covers the `>` comparison but would
|
|
50
|
+
pass just as well with `>=`, `<` or `true` in its place.
|
|
51
|
+
|
|
52
|
+
Mutation testing measures that directly:
|
|
53
|
+
|
|
54
|
+
1. Run the test suite once on the unmodified code. It must pass (the **baseline**).
|
|
55
|
+
2. For every mutant, a copy of the code with one small change such as `>` to `>=`, `&&` to `||`, `x + 1` to
|
|
56
|
+
`x - 1` or a deleted statement, run the tests again.
|
|
57
|
+
3. If the tests fail, the mutant is **killed**: your tests detect that change. If they pass, it **survived**.
|
|
58
|
+
4. The **mutation score** is `killed / (mutants - equivalent)`.
|
|
59
|
+
|
|
60
|
+
Some mutants cannot be killed because they do not change behaviour (for example `x < 0` vs `x <= 0` when `x == 0`
|
|
61
|
+
produces the same result either way). They are **equivalent** mutants; you list them in an
|
|
62
|
+
[ignore file](#equivalent-mutants-the-ignore-file) with a reason, and they no longer count against the score.
|
|
63
|
+
|
|
64
|
+
## Install
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
gem install mutaruby
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
or in a Gemfile:
|
|
71
|
+
|
|
72
|
+
```ruby
|
|
73
|
+
group :development, :test do
|
|
74
|
+
gem 'mutaruby', require: false
|
|
75
|
+
end
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
mutaruby needs Ruby 3.1 or newer and depends only on the [`parser`](https://github.com/whitequark/parser) gem. It runs
|
|
79
|
+
on Linux and macOS (it uses process groups and `/bin/sh`).
|
|
80
|
+
|
|
81
|
+
## Quickstart
|
|
82
|
+
|
|
83
|
+
mutaruby runs any shell command as the test oracle: **exit status 0 means the mutant survived, anything else means it
|
|
84
|
+
was killed**. By default it mutates `lib/**/*.rb`.
|
|
85
|
+
|
|
86
|
+
### With RSpec
|
|
87
|
+
|
|
88
|
+
```sh
|
|
89
|
+
mutaruby -t "bundle exec rspec"
|
|
90
|
+
# only one file, and run only the spec that covers it (much faster):
|
|
91
|
+
mutaruby -t "bundle exec rspec spec/pricing_spec.rb" lib/pricing.rb
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### With minitest
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
mutaruby -t "bundle exec rake test"
|
|
98
|
+
# or without rake:
|
|
99
|
+
mutaruby -t "ruby -Ilib -Itest -e 'Dir[\"test/**/*_test.rb\"].each { |f| require File.expand_path(f) }'"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Any command
|
|
103
|
+
|
|
104
|
+
```sh
|
|
105
|
+
mutaruby -t "bin/rails test test/models" "app/models/**/*.rb"
|
|
106
|
+
mutaruby -t "make check" lib/parser.rb
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Tips:
|
|
110
|
+
|
|
111
|
+
- **Make the test command fast and fail fast.** It runs once per mutant. `rspec --fail-fast` stops at the first
|
|
112
|
+
failure, which is all mutaruby needs.
|
|
113
|
+
- **Point it at the tests that matter** for the files you mutate.
|
|
114
|
+
- **Start small**: one file, or `--since main` to mutate only what your branch changed.
|
|
115
|
+
- Use `--list` to see the mutants without running anything.
|
|
116
|
+
|
|
117
|
+
## CLI reference
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
mutaruby --test COMMAND [options] [FILE|GLOB ...]
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Positional arguments are files or globs relative to the project root. The default is `lib/**/*.rb`.
|
|
124
|
+
|
|
125
|
+
### Test oracle
|
|
126
|
+
|
|
127
|
+
| Option | Meaning |
|
|
128
|
+
|---|---|
|
|
129
|
+
| `-t`, `--test COMMAND` | **Required.** Shell command (run with `/bin/sh -c`) that runs your tests. Exit 0 means the mutant survived, non-zero means killed. |
|
|
130
|
+
| `-C`, `--root DIR` | Project root (default: the current directory). This is the directory copied into each sandbox, and the command runs there. |
|
|
131
|
+
| `-j`, `--jobs N` | Parallel workers (default: the number of CPUs). Each worker has its own copy of the project. |
|
|
132
|
+
| `--timeout SECONDS` | Per-mutant time limit. A mutant that exceeds it (an infinite loop, say) is killed together with every process it started and counted as **killed, by timeout**. The default is `baseline time x factor + 5` seconds. |
|
|
133
|
+
| `--timeout-factor F` | The factor for the default timeout (default: 3). |
|
|
134
|
+
| `--copy-exclude GLOB` | Do not copy matching paths into the sandboxes (repeatable). `.git` and `coverage` are never copied. Matches against the relative path and against the base name, e.g. `--copy-exclude node_modules --copy-exclude 'log/*'`. |
|
|
135
|
+
|
|
136
|
+
The command gets these environment variables: `MUTARUBY=1`, and while a mutant runs `MUTARUBY_MUTANT_ID` (its id) and
|
|
137
|
+
`MUTARUBY_FILE` (the mutated file, relative to the root).
|
|
138
|
+
|
|
139
|
+
### Selection
|
|
140
|
+
|
|
141
|
+
| Option | Meaning |
|
|
142
|
+
|---|---|
|
|
143
|
+
| `--since REF` | Only mutate files changed since a git ref: committed, staged and unstaged changes plus new untracked files (`git diff --name-only REF` and `git ls-files --others --exclude-standard`), intersected with the file arguments. |
|
|
144
|
+
| `--method PATTERN` | Only mutants inside matching methods (repeatable). The pattern is a glob matched against the qualified name (`Pricing#discount` for an instance method, `Pricing.build` for a singleton method) or the bare name (`discount`). `Pricing#*` selects every instance method of `Pricing`. |
|
|
145
|
+
| `--exclude-method PATTERN` | Skip mutants inside matching methods (repeatable). |
|
|
146
|
+
| `--line SPEC` | Only mutants starting on these lines (repeatable). `SPEC` is `[FILE:]LINE` or `[FILE:]FROM-TO`, e.g. `lib/a.rb:10-20` or `42`. |
|
|
147
|
+
| `--exclude-line SPEC` | Skip mutants on these lines (repeatable). |
|
|
148
|
+
| `--operators LIST` | Only these operators (comma-separated ids; globs allowed, e.g. `condition-*,comparison`). |
|
|
149
|
+
| `--exclude-operators LIST` | Skip these operators. |
|
|
150
|
+
| `--shard I/N` | Run only shard `I` of `N` (1-based). Mutants are numbered in a stable order (files sorted, then source order) and mutant `k` belongs to shard `k mod N + 1`, so the shards of one run are disjoint and together cover every mutant. |
|
|
151
|
+
|
|
152
|
+
A line containing the comment `mutaruby:disable` is never mutated:
|
|
153
|
+
|
|
154
|
+
```ruby
|
|
155
|
+
LOG_PREFIX = '[app]' # mutaruby:disable
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### Equivalent mutants
|
|
159
|
+
|
|
160
|
+
| Option | Meaning |
|
|
161
|
+
|---|---|
|
|
162
|
+
| `--ignore-file PATH` | The ignore file (default: `.mutaruby-ignore.yml` in the root, when it exists). |
|
|
163
|
+
| `--strict-ignore` | Exit 1 if an ignore entry matches no surviving mutant (it is stale; see below). |
|
|
164
|
+
| `--suggest-ignore` | After the run, print ignore-file entries for every survivor, with an empty `reason` for you to fill in. Keep only real equivalents. |
|
|
165
|
+
|
|
166
|
+
### Output
|
|
167
|
+
|
|
168
|
+
| Option | Meaning |
|
|
169
|
+
|---|---|
|
|
170
|
+
| `--json PATH` | Write a JSON report. `-` writes it to stdout (the human summary then goes to stderr). |
|
|
171
|
+
| `--junit PATH` | Write a JUnit-style XML report (`-` for stdout). |
|
|
172
|
+
| `--fail-under PERCENT` | Exit 1 when the mutation score is below `PERCENT`. |
|
|
173
|
+
| `--list` | Print the selected mutants (id, location, method, change) and exit without running anything. Needs no `--test`. |
|
|
174
|
+
| `--list-operators` | Print every operator id with a description. |
|
|
175
|
+
| `-q`, `--quiet` | No progress marks. Progress marks go to stderr: `.` killed, `T` killed by timeout, `S` survived, `e` equivalent. |
|
|
176
|
+
| `-v`, `--verbose` | Also list timeouts and equivalent mutants in the summary. |
|
|
177
|
+
| `--version`, `-h`/`--help` | |
|
|
178
|
+
|
|
179
|
+
### Exit status
|
|
180
|
+
|
|
181
|
+
| Status | Meaning |
|
|
182
|
+
|---|---|
|
|
183
|
+
| 0 | The run completed (and met `--fail-under`, if given). |
|
|
184
|
+
| 1 | The score is below `--fail-under`, or `--strict-ignore` found stale entries. |
|
|
185
|
+
| 2 | Usage error, invalid ignore file, or the **baseline failed**: the test command does not pass on the unmodified code. |
|
|
186
|
+
| 130 | Interrupted. |
|
|
187
|
+
|
|
188
|
+
## Mutation operators
|
|
189
|
+
|
|
190
|
+
Every mutant is one edit at one place. Mutants that would not parse, and mutants whose text equals an earlier
|
|
191
|
+
mutant's, are dropped before anything runs. `--list-operators` prints this list.
|
|
192
|
+
|
|
193
|
+
| Operator | Change | Example |
|
|
194
|
+
|---|---|---|
|
|
195
|
+
| `arithmetic` | swaps `+ -`, `* /`, `%` to `*`, `**` to `*` | `a + b` to `a - b` |
|
|
196
|
+
| `comparison` | boundary and equality swaps: `==` and `!=`, `<` to `<=`, `<=` to `<`, `>` to `>=`, `>=` to `>`, `<=>` to `==`, `===` to `!=`, `=~` and `!~` | `n > 0` to `n >= 0` |
|
|
197
|
+
| `bitwise` | swaps `&` and `\|`, `^` to `&`, `<<` and `>>` | `flags & MASK` to `flags \| MASK` |
|
|
198
|
+
| `logical-swap` | `&&` and `\|\|`, `and` and `or` | `a && b` to `a \|\| b` |
|
|
199
|
+
| `logical-drop-left` | keeps only the right operand | `a && b` to `b` |
|
|
200
|
+
| `logical-drop-right` | keeps only the left operand | `a && b` to `a` |
|
|
201
|
+
| `negation-removal` | removes `!` / `not` | `!valid` to `valid` |
|
|
202
|
+
| `unary-minus-removal` | removes unary minus | `-delta` to `delta` |
|
|
203
|
+
| `condition-negate` | negates the condition of `if`, `unless`, `elsif`, ternaries and modifiers | `if ok` to `if !(ok)` |
|
|
204
|
+
| `condition-true` | forces the condition true | `if ok` to `if true` |
|
|
205
|
+
| `condition-false` | forces the condition false | `if ok` to `if false` |
|
|
206
|
+
| `loop-condition-negate` | negates a `while`/`until` condition | `while i < n` to `while !(i < n)` |
|
|
207
|
+
| `loop-condition-false` | a `while` loop never runs | `while i < n` to `while false` |
|
|
208
|
+
| `literal-int` | an integer becomes n+1, n-1, 0 and 1 (each distinct value once) | `retries = 3` to `4`, `2`, `0`, `1` |
|
|
209
|
+
| `literal-float` | a float becomes 0.0 and x+1.0 | `0.5` to `0.0`, `1.5` |
|
|
210
|
+
| `literal-string` | a quoted string becomes `""`; an empty one becomes `"mutaruby"` | `'admin'` to `''` |
|
|
211
|
+
| `literal-symbol` | a `:symbol` becomes `:__mutaruby__` | `:read` to `:__mutaruby__` |
|
|
212
|
+
| `literal-boolean` | `true` and `false` | `strict = true` to `strict = false` |
|
|
213
|
+
| `literal-nil` | `nil` becomes `true` | `cache = nil` to `cache = true` |
|
|
214
|
+
| `literal-array` | a non-empty array literal becomes `[]` | `[1, 2]` to `[]` |
|
|
215
|
+
| `literal-hash` | a non-empty hash literal becomes `{}` | `{ a: 1 }` to `{}` |
|
|
216
|
+
| `range-swap` | inclusive and exclusive ranges (not endless ranges, where both mean the same) | `0..n` to `0...n` |
|
|
217
|
+
| `return-nil` | an explicit return value becomes `nil` | `return total` to `return nil` |
|
|
218
|
+
| `method-body-nil` | a method body becomes `nil` | `def total = a + b` to `def total = nil` |
|
|
219
|
+
| `method-body-bool` | the body of a predicate method (`def x?`) becomes `true`, and `false` | |
|
|
220
|
+
| `statement-delete` | one statement of a multi-statement body becomes `nil` (class, module and method definitions and bare `private`-style calls are not deleted) | `log(x)` to `nil` |
|
|
221
|
+
| `argument-removal` | one argument is removed from a call (calls with up to 6 arguments; declarations such as `require`, `attr_reader`, `include` and `raise` are left alone) | `f(a, b)` to `f(b)` and `f(a)` |
|
|
222
|
+
| `call-to-receiver` | a method call is replaced by its receiver | `name.strip` to `name` |
|
|
223
|
+
| `call-to-argument` | a method call is replaced by its first argument | `list.fetch(i)` to `i` |
|
|
224
|
+
| `predicate-true` | a predicate call becomes `true` (except `block_given?`, `nil?`, `frozen?`, `respond_to?`) | `user.admin?` to `true` |
|
|
225
|
+
| `predicate-false` | a predicate call becomes `false` | `user.admin?` to `false` |
|
|
226
|
+
| `safe-navigation` | `&.` becomes `.` | `user&.name` to `user.name` |
|
|
227
|
+
| `raise-removal` | a `raise` / `fail` becomes `nil` | `raise ArgumentError unless ok` to `nil unless ok` |
|
|
228
|
+
| `op-assign` | swaps compound assignments: `+=` and `-=`, `*=` and `/=`, `\|=` and `&=`, `<<=` and `>>=` | `i += 1` to `i -= 1` |
|
|
229
|
+
| `or-assign` | `\|\|=` becomes `=` | `@x \|\|= load` to `@x = load` |
|
|
230
|
+
| `and-assign` | `&&=` becomes `=` | `a &&= b` to `a = b` |
|
|
231
|
+
|
|
232
|
+
Strings inside interpolations, heredocs, `%w[]` arrays and the arguments of `require`-style declarations are not
|
|
233
|
+
mutated, and neither are symbols used as hash keys (`a: 1`).
|
|
234
|
+
|
|
235
|
+
## Equivalent mutants: the ignore file
|
|
236
|
+
|
|
237
|
+
`.mutaruby-ignore.yml` (or the file given to `--ignore-file`) is a YAML list. Each entry names one **mutation site** and
|
|
238
|
+
must say **why** the mutant is equivalent:
|
|
239
|
+
|
|
240
|
+
```yaml
|
|
241
|
+
- file: lib/pricing.rb # relative to the project root
|
|
242
|
+
operator: comparison # operator id
|
|
243
|
+
line: "return 0 if total < 0" # the source line, leading/trailing whitespace ignored
|
|
244
|
+
occurrence: 2 # the 2nd line with exactly this text in the file; required when the text is not unique
|
|
245
|
+
method: "Pricing#discount" # optional: the enclosing method
|
|
246
|
+
mutation: "< => <=" # optional: only this change at the site (as printed by --list / the JSON "change")
|
|
247
|
+
lineno: 41 # optional: a hint for humans, never used for matching
|
|
248
|
+
reason: "total == 0 returns 0 on both paths" # required, non-empty
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
The key is **file + operator + line text + occurrence + method**, not the line number. Line numbers move whenever
|
|
252
|
+
code above them changes; the text of the line does not. So an entry keeps working while you edit other code, and it
|
|
253
|
+
stops matching when the line itself changes, which is exactly when the "equivalent" verdict needs another look.
|
|
254
|
+
|
|
255
|
+
- An entry without `mutation` covers every mutant of that operator at that site. Add `mutation` to cover one change only.
|
|
256
|
+
- An entry whose line text appears more than once in the file must have an `occurrence`, otherwise mutaruby refuses
|
|
257
|
+
the file as **ambiguous** (exit 2). An entry whose line no longer exists is refused as **stale** (exit 2).
|
|
258
|
+
- After a full run (no `--method`, `--line`, `--operators` or `--shard` filter), an entry that matched no surviving
|
|
259
|
+
mutant is reported as **stale**: the mutant is now killed or the code changed. `--strict-ignore` turns that into
|
|
260
|
+
exit 1.
|
|
261
|
+
- An ignored mutant is still run. If it is killed it counts as killed; if it survives it is reported as
|
|
262
|
+
`equivalent` and left out of the score.
|
|
263
|
+
|
|
264
|
+
`--suggest-ignore` prints a ready-made entry for every survivor; fill in `reason` for the true equivalents and write
|
|
265
|
+
tests for the rest.
|
|
266
|
+
|
|
267
|
+
## Using it in CI
|
|
268
|
+
|
|
269
|
+
```yaml
|
|
270
|
+
# .github/workflows/mutation.yml
|
|
271
|
+
name: Mutation testing
|
|
272
|
+
on: [pull_request]
|
|
273
|
+
jobs:
|
|
274
|
+
mutation:
|
|
275
|
+
runs-on: ubuntu-latest
|
|
276
|
+
steps:
|
|
277
|
+
- uses: actions/checkout@v4
|
|
278
|
+
with:
|
|
279
|
+
fetch-depth: 0 # --since needs the base branch
|
|
280
|
+
- uses: ruby/setup-ruby@v1
|
|
281
|
+
with:
|
|
282
|
+
bundler-cache: true
|
|
283
|
+
- name: Mutation test the changed files
|
|
284
|
+
run: >
|
|
285
|
+
bundle exec mutaruby
|
|
286
|
+
-t "bundle exec rspec --fail-fast"
|
|
287
|
+
--since origin/${{ github.base_ref }}
|
|
288
|
+
--fail-under 85 --strict-ignore
|
|
289
|
+
--junit mutaruby.xml --json mutaruby.json
|
|
290
|
+
- uses: actions/upload-artifact@v4
|
|
291
|
+
if: always()
|
|
292
|
+
with:
|
|
293
|
+
name: mutaruby
|
|
294
|
+
path: mutaruby.*
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
For a large code base, split the work across machines with `--shard`:
|
|
298
|
+
|
|
299
|
+
```yaml
|
|
300
|
+
strategy:
|
|
301
|
+
matrix:
|
|
302
|
+
shard: [1, 2, 3, 4]
|
|
303
|
+
steps:
|
|
304
|
+
# ...
|
|
305
|
+
- run: bundle exec mutaruby -t "bundle exec rspec --fail-fast" --shard ${{ matrix.shard }}/4 --json shard-${{ matrix.shard }}.json
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Each shard reports its own score; to get the overall score, add up `counts` across the JSON files. Note that
|
|
309
|
+
`--fail-under` and stale-entry detection apply per shard.
|
|
310
|
+
|
|
311
|
+
## Output formats
|
|
312
|
+
|
|
313
|
+
**Human** (stdout): every survivor with its location, enclosing method, operator and a two-line diff, then the
|
|
314
|
+
summary.
|
|
315
|
+
|
|
316
|
+
**JSON** (`--json`): one document with `score`, `counts` (`mutants`, `killed`, `timeout`, `survived`, `equivalent`),
|
|
317
|
+
`complete` (true when no filter limited the run), the `files`, `baseline_seconds`, `timeout`, `stale_ignores`,
|
|
318
|
+
`warnings`, and one object per mutant:
|
|
319
|
+
|
|
320
|
+
```json
|
|
321
|
+
{
|
|
322
|
+
"id": 7, "file": "lib/pricing.rb", "lineno": 14, "operator": "comparison", "method": "Pricing#discount",
|
|
323
|
+
"line_text": "return 0 if total > 100", "occurrence": 1, "orig": ">", "repl": ">=", "change": "> => >=",
|
|
324
|
+
"label": "comparison: > => >=", "status": "survived", "seconds": 0.412, "ignore_reason": null
|
|
325
|
+
}
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
`status` is `killed`, `timeout` (also a kill), `survived` or `equivalent`.
|
|
329
|
+
|
|
330
|
+
**JUnit XML** (`--junit`): one `testcase` per mutant. A survivor is a `failure` (with the diff), an equivalent mutant
|
|
331
|
+
is `skipped` (with the reason) and the score is a suite `property`. CI systems that read JUnit reports show the
|
|
332
|
+
survivors as failed tests.
|
|
333
|
+
|
|
334
|
+
## How it compares with mutant
|
|
335
|
+
|
|
336
|
+
[mutant](https://github.com/mbj/mutant) is the established mutation tester for Ruby, and it is more mature and more
|
|
337
|
+
capable than mutaruby: it has many more operators, semantic knowledge of Ruby's core library, selection of only the
|
|
338
|
+
tests that cover each subject, and years of use on large code bases. It is commercially licensed (free for open
|
|
339
|
+
source projects).
|
|
340
|
+
|
|
341
|
+
mutaruby aims to be **MIT-licensed and simple**:
|
|
342
|
+
|
|
343
|
+
- It works with **any test command** (rspec, minitest, rake, a Makefile target), because the oracle is just an exit
|
|
344
|
+
status. There is no framework integration to configure, and no per-test selection either, so it is slower than
|
|
345
|
+
mutant on big suites unless you point the command at the relevant tests.
|
|
346
|
+
- It is small enough to read in an afternoon, and it has no dependency besides `parser`.
|
|
347
|
+
- It runs each mutant in a **separate process** in a **copy of the project**, so it cannot leave your working tree
|
|
348
|
+
modified.
|
|
349
|
+
|
|
350
|
+
If you can use mutant, it will usually find more. If you need an MIT-licensed tool, or want something you can point
|
|
351
|
+
at any project with one command, mutaruby is for you.
|
|
352
|
+
|
|
353
|
+
## Design notes
|
|
354
|
+
|
|
355
|
+
**A fresh process per mutant.** Every run, the baseline and each mutant, is a new `/bin/sh -c COMMAND` with its
|
|
356
|
+
own Ruby. Nothing loaded by one run can leak into the next. This matters more than it looks. When mutants are
|
|
357
|
+
evaluated inside one long-lived process, deleting a `require` cannot be detected because the library is already loaded,
|
|
358
|
+
and a deleted method or constant can quietly keep its old definition. Those mutants then "survive" and report test
|
|
359
|
+
gaps that do not exist. The cost is process start-up time per mutant; `--jobs` and a narrow test command are how you
|
|
360
|
+
pay it back.
|
|
361
|
+
|
|
362
|
+
**A sandbox, never the working tree.** At start-up each worker copies the project root (minus `.git`, `coverage` and
|
|
363
|
+
any `--copy-exclude`) into a temporary directory. A mutant is written into the worker's copy, the command runs there,
|
|
364
|
+
and the original bytes are written back before the next mutant. Your files are never opened for writing, so a crash,
|
|
365
|
+
a timeout or Ctrl-C cannot leave a mutant behind. If you run mutaruby under `bundle exec`, `BUNDLE_GEMFILE` is
|
|
366
|
+
pointed at the sandbox's Gemfile so that a `gemspec` line loads the sandbox's `lib/`, not your checkout's.
|
|
367
|
+
|
|
368
|
+
**Splicing at character offsets, with a self-check.** mutaruby does not regenerate source from the AST (that would
|
|
369
|
+
reformat the file and lose comments). Each mutant replaces exactly one source range, using the offsets that `parser`
|
|
370
|
+
reports for the node or token. Those offsets count **characters**, not bytes. Splicing by bytes moves every mutant
|
|
371
|
+
that comes after the first multi-byte character in a file (an accented letter in a comment is enough) onto the wrong
|
|
372
|
+
text. Before every splice, mutaruby checks that the slice it is about to replace is exactly the node's own source
|
|
373
|
+
text. If it is not, the run aborts rather than testing a mutant in the wrong place.
|
|
374
|
+
|
|
375
|
+
**The baseline must pass.** Before any mutant runs, the command runs once on the unmodified code. If it fails, every
|
|
376
|
+
mutant would be "killed" and the score would mean nothing, so mutaruby stops with exit status 2 and shows the tail of
|
|
377
|
+
the output. The baseline's duration also sets the default timeout.
|
|
378
|
+
|
|
379
|
+
**Timeouts kill the whole process group.** Each run is the leader of its own process group. On timeout the group gets
|
|
380
|
+
`SIGTERM`, then `SIGKILL` two seconds later, so a hung child (a forked server, a parallel test worker) does not
|
|
381
|
+
outlive its mutant.
|
|
382
|
+
|
|
383
|
+
**Site keys instead of line numbers** for the ignore file; see [above](#equivalent-mutants-the-ignore-file).
|
|
384
|
+
|
|
385
|
+
## Limitations
|
|
386
|
+
|
|
387
|
+
- Linux and macOS only (process groups, `/bin/sh`). Windows is not supported.
|
|
388
|
+
- The test command is run as is for every mutant; mutaruby does not work out which tests cover which code.
|
|
389
|
+
- The parser grammar is chosen to match the running Ruby, up to the newest grammar the installed `parser` gem knows.
|
|
390
|
+
Files that `parser` cannot read (for example syntax newer than it supports) are skipped with a warning.
|
|
391
|
+
- Mutants are syntactic. Some are equivalent by nature; that is what the ignore file is for.
|
|
392
|
+
- Files are copied into each worker's sandbox at start-up, so very large repositories should use `--copy-exclude`
|
|
393
|
+
for directories the tests do not need.
|
|
394
|
+
|
|
395
|
+
## Contributing, security, licence
|
|
396
|
+
|
|
397
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) and [SECURITY.md](SECURITY.md). mutaruby is released under the
|
|
398
|
+
[MIT licence](LICENSE).
|
data/exe/mutaruby
ADDED
data/lib/mutaruby/cli.rb
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'optparse'
|
|
4
|
+
require 'etc'
|
|
5
|
+
|
|
6
|
+
module Mutaruby
|
|
7
|
+
# Command line interface. Exit status: 0 success, 1 score below --fail-under (or stale ignore entries with
|
|
8
|
+
# --strict-ignore), 2 usage error, unparsable ignore file or failing baseline.
|
|
9
|
+
class CLI
|
|
10
|
+
EXIT_OK = 0
|
|
11
|
+
EXIT_FAIL = 1
|
|
12
|
+
EXIT_ERROR = 2
|
|
13
|
+
|
|
14
|
+
def initialize(argv, stdout: $stdout, stderr: $stderr)
|
|
15
|
+
@argv = argv.dup
|
|
16
|
+
@stdout = stdout
|
|
17
|
+
@stderr = stderr
|
|
18
|
+
@o = Runner::Options.new(jobs: Etc.nprocessors, patterns: [], methods: [], exclude_methods: [], lines: [],
|
|
19
|
+
exclude_lines: [], copy_excludes: [], progress: true)
|
|
20
|
+
@include_ops = []
|
|
21
|
+
@exclude_ops = []
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def run
|
|
25
|
+
catch(:done) { return execute }
|
|
26
|
+
EXIT_OK # --help / --version
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def execute
|
|
32
|
+
parser.parse!(@argv)
|
|
33
|
+
@o.patterns = @argv
|
|
34
|
+
return list_operators if @list_operators
|
|
35
|
+
|
|
36
|
+
@o.operators = operators
|
|
37
|
+
raise Error, 'missing --test COMMAND (the command that runs your tests; exit 0 = pass)' if @o.command.nil? && !@list
|
|
38
|
+
|
|
39
|
+
runner = Runner.new(@o, out: @stderr)
|
|
40
|
+
return list(runner.plan) if @list
|
|
41
|
+
|
|
42
|
+
report = runner.run
|
|
43
|
+
emit(report)
|
|
44
|
+
status(report)
|
|
45
|
+
rescue OptionParser::ParseError, Error, IgnoreFile::Invalid, SpliceError => e
|
|
46
|
+
@stderr.puts "mutaruby: #{e.message}"
|
|
47
|
+
EXIT_ERROR
|
|
48
|
+
rescue Interrupt
|
|
49
|
+
@stderr.puts "\nmutaruby: interrupted"
|
|
50
|
+
130
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def parser
|
|
54
|
+
OptionParser.new do |o|
|
|
55
|
+
o.banner = 'Usage: mutaruby --test COMMAND [options] [FILE|GLOB ...] (default: lib/**/*.rb)'
|
|
56
|
+
o.separator ''
|
|
57
|
+
o.separator 'Test oracle:'
|
|
58
|
+
o.on('-t', '--test COMMAND', 'Shell command that runs the tests; exit 0 = mutant survived, non-zero = killed') { |v| @o.command = v }
|
|
59
|
+
o.on('-C', '--root DIR', 'Project root (copied for every worker; default: current directory)') { |v| @o.root = v }
|
|
60
|
+
o.on('-j', '--jobs N', Integer, "Parallel workers (default: #{Etc.nprocessors})") { |v| @o.jobs = v }
|
|
61
|
+
o.on('--timeout SECONDS', Float, 'Per-mutant timeout (default: baseline time x factor + 5s)') { |v| @o.timeout = v }
|
|
62
|
+
o.on('--timeout-factor F', Float, 'Multiplier on the baseline time for the default timeout (default: 3)') { |v| @o.timeout_factor = v }
|
|
63
|
+
o.on('--copy-exclude GLOB', 'Do not copy matching paths into the sandboxes (repeatable; .git and coverage are always skipped)') { |v| @o.copy_excludes << v }
|
|
64
|
+
o.separator ''
|
|
65
|
+
o.separator 'Selection:'
|
|
66
|
+
o.on('--since REF', 'Only files changed since a git ref (committed, staged, unstaged, untracked)') { |v| @o.since = v }
|
|
67
|
+
o.on('--method PATTERN', 'Only mutants inside matching methods ("Calc#add", "Calc#*", "add"; repeatable)') { |v| @o.methods << v }
|
|
68
|
+
o.on('--exclude-method PATTERN', 'Skip mutants inside matching methods (repeatable)') { |v| @o.exclude_methods << v }
|
|
69
|
+
o.on('--line SPEC', 'Only mutants on these lines: [FILE:]LINE or [FILE:]FROM-TO (repeatable)') { |v| @o.lines << Selection.parse_line_spec(v) }
|
|
70
|
+
o.on('--exclude-line SPEC', 'Skip mutants on these lines (repeatable)') { |v| @o.exclude_lines << Selection.parse_line_spec(v) }
|
|
71
|
+
o.on('--operators LIST', Array, 'Only these operators (comma-separated; globs allowed, e.g. condition-*)') { |v| @include_ops.concat(v) }
|
|
72
|
+
o.on('--exclude-operators LIST', Array, 'Skip these operators (comma-separated; globs allowed)') { |v| @exclude_ops.concat(v) }
|
|
73
|
+
o.on('--shard I/N', 'Run only shard I of N (1-based) of the selected mutants') { |v| @o.shard = Selection.parse_shard(v) }
|
|
74
|
+
o.separator ''
|
|
75
|
+
o.separator 'Equivalent mutants:'
|
|
76
|
+
o.on('--ignore-file PATH', 'Ignore file (default: .mutaruby-ignore.yml in the root, if present)') { |v| @o.ignore_file = v }
|
|
77
|
+
o.on('--strict-ignore', 'Fail (exit 1) when an ignore entry matches no surviving mutant') { @strict = true }
|
|
78
|
+
o.on('--suggest-ignore', 'Print ignore-file stubs for the surviving mutants (fill in each reason)') { @suggest = true }
|
|
79
|
+
o.separator ''
|
|
80
|
+
o.separator 'Output:'
|
|
81
|
+
o.on('--json PATH', 'Write a JSON report ("-" = stdout)') { |v| @json = v }
|
|
82
|
+
o.on('--junit PATH', 'Write a JUnit-style XML report ("-" = stdout)') { |v| @junit = v }
|
|
83
|
+
o.on('--fail-under PERCENT', Float, 'Exit 1 when the mutation score is below PERCENT') { |v| @fail_under = v }
|
|
84
|
+
o.on('--list', 'List the selected mutants without running anything') { @list = true }
|
|
85
|
+
o.on('--list-operators', 'List the mutation operators') { @list_operators = true }
|
|
86
|
+
o.on('-q', '--quiet', 'No progress marks') { @o.progress = false }
|
|
87
|
+
o.on('-v', '--verbose', 'Also list timeouts and equivalent mutants') { @verbose = true }
|
|
88
|
+
o.on('--version', 'Print the version') do
|
|
89
|
+
@stdout.puts "mutaruby #{VERSION}"
|
|
90
|
+
throw :done
|
|
91
|
+
end
|
|
92
|
+
o.on('-h', '--help', 'Show this help') do
|
|
93
|
+
@stdout.puts o
|
|
94
|
+
throw :done
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def operators
|
|
102
|
+
return nil if @include_ops.empty? && @exclude_ops.empty?
|
|
103
|
+
|
|
104
|
+
all = Generator::OPERATORS.keys
|
|
105
|
+
(@include_ops + @exclude_ops).each do |pat|
|
|
106
|
+
raise Error, "unknown operator #{pat.inspect} (see --list-operators)" if all.none? { |id| File.fnmatch?(pat, id) }
|
|
107
|
+
end
|
|
108
|
+
chosen = @include_ops.empty? ? all : all.select { |id| @include_ops.any? { |p| File.fnmatch?(p, id) } }
|
|
109
|
+
chosen.reject { |id| @exclude_ops.any? { |p| File.fnmatch?(p, id) } }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def list_operators
|
|
113
|
+
w = Generator::OPERATORS.keys.map(&:size).max
|
|
114
|
+
Generator::OPERATORS.each { |id, d| @stdout.puts "#{id.ljust(w)} #{d}" }
|
|
115
|
+
EXIT_OK
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def list(plan)
|
|
119
|
+
plan[:mutations].each do |m|
|
|
120
|
+
@stdout.puts "#{m.id}\t#{m.location}\t#{m.method || '-'}\t#{m.label}"
|
|
121
|
+
end
|
|
122
|
+
@stderr.puts "#{plan[:mutations].size} mutants in #{plan[:files].size} files"
|
|
123
|
+
EXIT_OK
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def emit(report)
|
|
127
|
+
human_io = [@json, @junit].include?('-') ? @stderr : @stdout
|
|
128
|
+
Reporters.human(report, human_io, verbose: @verbose)
|
|
129
|
+
write(@json) { |io| Reporters.json(report, io) } if @json
|
|
130
|
+
write(@junit) { |io| Reporters.junit(report, io) } if @junit
|
|
131
|
+
return unless @suggest
|
|
132
|
+
|
|
133
|
+
survivors = report.outcomes.select { |o| o.status == :survived }.map(&:mutation)
|
|
134
|
+
@stderr.puts '# ignore-file stubs (add a reason to each; keep only true equivalents):'
|
|
135
|
+
@stderr.puts IgnoreFile.stub_for(survivors, report.sources) if survivors.any?
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def write(path, &block)
|
|
139
|
+
if path == '-'
|
|
140
|
+
block.call(@stdout)
|
|
141
|
+
else
|
|
142
|
+
File.open(path, 'w', &block)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def status(report)
|
|
147
|
+
code = EXIT_OK
|
|
148
|
+
if @fail_under && (report.score || 100.0) < @fail_under
|
|
149
|
+
@stderr.puts format('mutaruby: score %s is below --fail-under %.2f%%', Reporters.score_text(report), @fail_under)
|
|
150
|
+
code = EXIT_FAIL
|
|
151
|
+
end
|
|
152
|
+
if @strict && report.stale.any?
|
|
153
|
+
@stderr.puts "mutaruby: #{report.stale.size} stale ignore entr#{report.stale.size == 1 ? 'y' : 'ies'} (--strict-ignore)"
|
|
154
|
+
code = EXIT_FAIL
|
|
155
|
+
end
|
|
156
|
+
code
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|