active_mutator 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +244 -0
- data/exe/active_mutator +4 -0
- data/lib/active_mutator/accepted_ledger.rb +45 -0
- data/lib/active_mutator/analysis.rb +3 -0
- data/lib/active_mutator/atomic_file.rb +15 -0
- data/lib/active_mutator/baseline.rb +125 -0
- data/lib/active_mutator/baseline_delta.rb +57 -0
- data/lib/active_mutator/baseline_hooks.rb +68 -0
- data/lib/active_mutator/cli.rb +53 -0
- data/lib/active_mutator/config.rb +8 -0
- data/lib/active_mutator/coverage_map.rb +55 -0
- data/lib/active_mutator/edit.rb +5 -0
- data/lib/active_mutator/engine.rb +75 -0
- data/lib/active_mutator/fingerprint.rb +21 -0
- data/lib/active_mutator/inserter.rb +18 -0
- data/lib/active_mutator/mutation.rb +11 -0
- data/lib/active_mutator/operators/base.rb +25 -0
- data/lib/active_mutator/operators/call_swap.rb +28 -0
- data/lib/active_mutator/operators/condition_forcing.rb +17 -0
- data/lib/active_mutator/operators/conditional_boundary.rb +16 -0
- data/lib/active_mutator/operators/early_return.rb +16 -0
- data/lib/active_mutator/operators/literal.rb +37 -0
- data/lib/active_mutator/operators/logical_operator.rb +24 -0
- data/lib/active_mutator/operators/negation_removal.rb +11 -0
- data/lib/active_mutator/operators/statement_deletion.rb +15 -0
- data/lib/active_mutator/reporter/json.rb +40 -0
- data/lib/active_mutator/reporter/terminal.rb +50 -0
- data/lib/active_mutator/result.rb +6 -0
- data/lib/active_mutator/runner.rb +143 -0
- data/lib/active_mutator/scheduler.rb +125 -0
- data/lib/active_mutator/since_filter.rb +48 -0
- data/lib/active_mutator/splicer.rb +13 -0
- data/lib/active_mutator/subject.rb +7 -0
- data/lib/active_mutator/subject_finder.rb +56 -0
- data/lib/active_mutator/version.rb +3 -0
- data/lib/active_mutator/work_item.rb +4 -0
- data/lib/active_mutator/worker.rb +64 -0
- data/lib/active_mutator.rb +42 -0
- metadata +132 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: c401b5edbafa0a072b6672e088a9016f918d9e76ce6b264a9e2a054b1b538cc6
|
|
4
|
+
data.tar.gz: d2a1908e950d8687e68524184efc2bce200d595be3b25f31e3a0e92a95325b3e
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 9d42e2e4a207ab1c6ced606eeeee342389b503da5055e207d282df220f112537f90b5f967079dfd76060d740f710d5441c8422565e7bdad314fa4bc9cf5d07c6
|
|
7
|
+
data.tar.gz: 3e226d88dc71b5b0b698f5e2a1148a38c7f6b12e0cd47411033368a0bf7d18a90d0ca8243fa3d2841ebd1b1be4cfb554c41732b4e1eacc2961f6477ddb692cc7
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Daniel John
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
# active_mutator
|
|
2
|
+
|
|
3
|
+
Mutation testing for Ruby, built on [Prism](https://github.com/ruby/prism).
|
|
4
|
+
Open source, RSpec-integrated, Rails-first.
|
|
5
|
+
|
|
6
|
+
active_mutator mutates your code one small change at a time (`>` → `>=`,
|
|
7
|
+
`&&` → `||`, delete a statement, force a condition…), runs exactly the
|
|
8
|
+
examples that cover the mutated line, and reports every mutant your suite
|
|
9
|
+
fails to kill. A surviving mutant is a behavior change no test notices —
|
|
10
|
+
a precise, machine-verified test gap.
|
|
11
|
+
|
|
12
|
+
## A surviving mutant, in one example
|
|
13
|
+
|
|
14
|
+
```ruby
|
|
15
|
+
def discount(total)
|
|
16
|
+
return 0 if total < 100
|
|
17
|
+
total / 10
|
|
18
|
+
end
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
```ruby
|
|
22
|
+
it { expect(calc.discount(50)).to eq(0) }
|
|
23
|
+
it { expect(calc.discount(200)).to eq(20) }
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Both examples pass. Line coverage on `discount` is 100%. Run active_mutator
|
|
27
|
+
and one mutant survives anyway:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
Surviving mutants:
|
|
31
|
+
|
|
32
|
+
Calculator#discount (lib/calculator.rb:11)
|
|
33
|
+
replace `<` with `<=`
|
|
34
|
+
- total < 100
|
|
35
|
+
+ total <= 100
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Nothing in the test suite calls `discount(100)` — the one input where `<`
|
|
39
|
+
and `<=` disagree. The tests pass, coverage is green, and the boundary is
|
|
40
|
+
still unverified. That gap is invisible to coverage and obvious to
|
|
41
|
+
mutation testing. Add `it { expect(calc.discount(100)).to eq(0) }` and the
|
|
42
|
+
mutant is killed.
|
|
43
|
+
|
|
44
|
+
## What is mutation testing?
|
|
45
|
+
|
|
46
|
+
Coverage answers "did a test run this line?" Mutation testing answers "would
|
|
47
|
+
a test *notice* if this line were wrong?" — a materially different, and
|
|
48
|
+
usually more useful, question.
|
|
49
|
+
|
|
50
|
+
active_mutator applies one small, syntactically valid change to your code
|
|
51
|
+
(a "mutant") and re-runs only the examples that cover it. If a test fails,
|
|
52
|
+
the mutant is **killed**: your tests correctly reject that wrong behavior.
|
|
53
|
+
If every covering test still passes, the mutant **survived**: something
|
|
54
|
+
changed and nothing noticed. A survivor is not a hypothetical — it's the
|
|
55
|
+
exact line, the exact before/after diff, and proof that no assertion
|
|
56
|
+
depends on the difference.
|
|
57
|
+
|
|
58
|
+
Mutation score is `(killed + timeout) / (killed + timeout + survived)`.
|
|
59
|
+
100% is usually not the right target — some mutants are behaviorally
|
|
60
|
+
*equivalent* to the original and can never be killed by any test — which is
|
|
61
|
+
why active_mutator has a committed acceptance ledger for closing survivors
|
|
62
|
+
out with a stated reason instead of chasing an unreachable score.
|
|
63
|
+
|
|
64
|
+
Full primer, including the origin of the technique and further reading:
|
|
65
|
+
**[`docs/guides/what-is-mutation-testing.md`](docs/guides/what-is-mutation-testing.md)**.
|
|
66
|
+
|
|
67
|
+
## Install
|
|
68
|
+
|
|
69
|
+
```ruby
|
|
70
|
+
# Gemfile
|
|
71
|
+
group :development, :test do
|
|
72
|
+
gem "active_mutator"
|
|
73
|
+
end
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Requires Ruby ≥ 3.2, RSpec, and a green suite. Linux/macOS (MRI fork).
|
|
77
|
+
|
|
78
|
+
## Quick start
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
bundle install
|
|
82
|
+
bundle exec active_mutator app/models/calculator.rb
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
First run performs an instrumented baseline of your suite to build the
|
|
86
|
+
coverage map (cached in `.active_mutator/`, refreshed incrementally after
|
|
87
|
+
that — see [`docs/guides/how-it-works.md`](docs/guides/how-it-works.md)).
|
|
88
|
+
Then each mutant runs in its own fork against only its covering examples.
|
|
89
|
+
|
|
90
|
+
### Reading the output
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
$ bundle exec active_mutator app/models/calculator.rb
|
|
94
|
+
|
|
95
|
+
.....S..T...U..A....
|
|
96
|
+
|
|
97
|
+
killed: 14
|
|
98
|
+
survived: 1
|
|
99
|
+
timeout: 1
|
|
100
|
+
error: 0
|
|
101
|
+
uncovered: 1
|
|
102
|
+
accepted: 1
|
|
103
|
+
invalid (discarded): 2
|
|
104
|
+
Mutation score: 93.8%
|
|
105
|
+
|
|
106
|
+
Surviving mutants:
|
|
107
|
+
|
|
108
|
+
Calculator#discount (app/models/calculator.rb:9)
|
|
109
|
+
replace `<` with `<=`
|
|
110
|
+
- total < 100
|
|
111
|
+
+ total <= 100
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Each character on the progress line is one mutant, printed as it finishes:
|
|
115
|
+
|
|
116
|
+
| Char | Status | Meaning |
|
|
117
|
+
|---|---|---|
|
|
118
|
+
| `.` | `killed` | a covering test failed — good, the mutant is dead |
|
|
119
|
+
| `S` | `survived` | every covering test passed — a test gap |
|
|
120
|
+
| `T` | `timeout` | ran past its time budget — counted as detected (likely an infinite loop) |
|
|
121
|
+
| `E` | `error` | the worker crashed, or the mutated code raised outside a test assertion |
|
|
122
|
+
| `U` | `uncovered` | no test executes the mutated line at all — coverage debt, worse than a survivor |
|
|
123
|
+
| `A` | `accepted` | matches a known-equivalent entry in the acceptance ledger — excluded from the score |
|
|
124
|
+
|
|
125
|
+
`invalid` mutants (edits that don't even re-parse as valid Ruby) are
|
|
126
|
+
discarded before scheduling and reported as a count only. Exit code is `1`
|
|
127
|
+
iff unaccepted survivors exist — `0` otherwise, including when there are
|
|
128
|
+
only `uncovered`/`accepted`/`error` results.
|
|
129
|
+
|
|
130
|
+
## How it works, compactly
|
|
131
|
+
|
|
132
|
+
1. **Subject discovery** — a Prism visitor finds every method (`def`) in
|
|
133
|
+
your target files.
|
|
134
|
+
2. **Source-span edits** — each operator emits byte-range text edits
|
|
135
|
+
against the original file, not a rewritten AST; every mutant is
|
|
136
|
+
re-parsed with Prism and discarded (`invalid`) if the edit produced
|
|
137
|
+
something that doesn't parse. No unparser is ever built or maintained.
|
|
138
|
+
3. **Coverage-mapped test selection** — one instrumented baseline run maps
|
|
139
|
+
every source line to the examples that cover it; incremental runs
|
|
140
|
+
refresh only what changed instead of re-running the whole suite.
|
|
141
|
+
4. **Fork-per-mutant kill runs** — the parent preloads your app and spec
|
|
142
|
+
helper once; each mutant is inserted and exercised in its own fork
|
|
143
|
+
against just its covering examples, so results can't bleed state
|
|
144
|
+
between mutants.
|
|
145
|
+
|
|
146
|
+
Full architecture, including the coverage-cache format, the fork pipeline,
|
|
147
|
+
the serial lane for browser specs, timeout budgets, and every status —
|
|
148
|
+
**[`docs/guides/how-it-works.md`](docs/guides/how-it-works.md)**.
|
|
149
|
+
|
|
150
|
+
## Usage
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
active_mutator # mutate app/ and lib/, full run
|
|
154
|
+
active_mutator app/models # scope by path
|
|
155
|
+
active_mutator --changed # uncommitted work only (dev loop)
|
|
156
|
+
active_mutator --since origin/main # PR scope (CI)
|
|
157
|
+
active_mutator --subject 'Foo::Bar#baz' # one method
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Statuses: `killed` (test failed — good), `survived` (test gap), `timeout`
|
|
161
|
+
(counts as detected), `uncovered` (no covering example — coverage debt),
|
|
162
|
+
`accepted` (known-equivalent, see ledger), `error`, `invalid` (discarded).
|
|
163
|
+
Exit code 1 iff unaccepted survivors exist.
|
|
164
|
+
|
|
165
|
+
Score = (killed + timeout) / (killed + timeout + survived).
|
|
166
|
+
|
|
167
|
+
## The dev loop
|
|
168
|
+
|
|
169
|
+
TDD until green, then verify the tests constrain the behavior:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
bundle exec active_mutator --changed --format json
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Kill survivors by writing the missing tests. For genuine equivalent mutants:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
bundle exec active_mutator --changed --accept-survivors # records to ledger
|
|
179
|
+
git add .active_mutator_accepted.json # committed state
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Acceptance takes effect on the NEXT run (the accepting run still exits 1).
|
|
183
|
+
Agent workflow: see [`docs/skills/mutation-check.md`](docs/skills/mutation-check.md).
|
|
184
|
+
|
|
185
|
+
## CI recipe
|
|
186
|
+
|
|
187
|
+
- Per-PR: `active_mutator --since origin/main` (minutes)
|
|
188
|
+
- Nightly: `active_mutator --force-baseline` (full run; also recovers the
|
|
189
|
+
incremental baseline's newly-covering-example blind spot)
|
|
190
|
+
|
|
191
|
+
## Flags
|
|
192
|
+
|
|
193
|
+
| Flag | Default | Meaning |
|
|
194
|
+
|---|---|---|
|
|
195
|
+
| `--jobs N` | half the cores | fork-pool width |
|
|
196
|
+
| `--changed` | — | mutate uncommitted + untracked work |
|
|
197
|
+
| `--since REF` | — | mutate methods changed since REF |
|
|
198
|
+
| `--subject NAME` | — | one subject, e.g. `Foo#bar` |
|
|
199
|
+
| `--format terminal\|json` | terminal | report format |
|
|
200
|
+
| `--accept-survivors` | off | record survivors to the acceptance ledger |
|
|
201
|
+
| `--force-baseline` | off | ignore cached coverage map |
|
|
202
|
+
| `--preload-helper FILE` / `--no-preload-helper` | auto-detect | parent spec-helper preload |
|
|
203
|
+
| `--serial-pattern PAT` | `spec/system/`, `spec/features/` | covering-path prefixes forced serial |
|
|
204
|
+
| `--browser-boot-seconds S` | 15 | serial-lane timeout bump |
|
|
205
|
+
| `--timeout-factor F` / `--timeout-floor S` | 8 / 10 | mutation timeout budget |
|
|
206
|
+
| `--require FILE` | — | preload files (repeatable) |
|
|
207
|
+
|
|
208
|
+
Every active_mutator process sets `ENV["ACTIVE_MUTATOR"] = "1"` — use it to
|
|
209
|
+
guard SimpleCov or other tooling in your spec helper:
|
|
210
|
+
|
|
211
|
+
```ruby
|
|
212
|
+
SimpleCov.start "rails" unless ENV["ACTIVE_MUTATOR"]
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
## Known limits (v1.1)
|
|
216
|
+
|
|
217
|
+
Method bodies only (no class-macro/constant mutation) · RSpec only ·
|
|
218
|
+
heredoc strings not mutated · `class << self` bodies and nested defs
|
|
219
|
+
skipped · incremental baseline can miss examples that only cover changed
|
|
220
|
+
code after the change (nightly `--force-baseline` recovers).
|
|
221
|
+
|
|
222
|
+
## Guides
|
|
223
|
+
|
|
224
|
+
- [What is mutation testing?](docs/guides/what-is-mutation-testing.md) —
|
|
225
|
+
the concepts: kill/survive, score, equivalent mutants, further reading.
|
|
226
|
+
- [How it works](docs/guides/how-it-works.md) — architecture: subject
|
|
227
|
+
discovery, source-span edits, the coverage map, the fork pipeline, and
|
|
228
|
+
honest limits.
|
|
229
|
+
- [Operator reference](docs/guides/operators.md) — every mutation
|
|
230
|
+
active_mutator can generate, with before/after examples and what a
|
|
231
|
+
survivor of each one means.
|
|
232
|
+
- [Mutation-check skill](docs/skills/mutation-check.md) — the agent-facing
|
|
233
|
+
workflow: run, read survivors, strengthen tests, or accept with a reason.
|
|
234
|
+
|
|
235
|
+
## Contributing
|
|
236
|
+
|
|
237
|
+
Issues and pull requests welcome. Run `bundle exec rspec` before sending a
|
|
238
|
+
change; `bundle exec active_mutator --changed` on your own diff before
|
|
239
|
+
sending a change that touches `lib/` is a good idea for the same reason
|
|
240
|
+
you'd want it run on any other codebase.
|
|
241
|
+
|
|
242
|
+
## License
|
|
243
|
+
|
|
244
|
+
[MIT](LICENSE).
|
data/exe/active_mutator
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "set"
|
|
3
|
+
|
|
4
|
+
module ActiveMutator
|
|
5
|
+
# Committed, repo-root ledger of accepted (equivalent) survivors.
|
|
6
|
+
# Deliberately NOT inside .active_mutator/ — that dir is gitignored and
|
|
7
|
+
# disposable, while acceptance decisions are durable team/CI state.
|
|
8
|
+
class AcceptedLedger
|
|
9
|
+
FILENAME = ".active_mutator_accepted.json"
|
|
10
|
+
|
|
11
|
+
def self.load(root)
|
|
12
|
+
path = File.join(root, FILENAME)
|
|
13
|
+
entries = File.exist?(path) ? JSON.parse(File.read(path)) : []
|
|
14
|
+
new(path, entries.map { |e| from_hash(e) })
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.from_hash(hash)
|
|
18
|
+
Fingerprint.new(file: hash.fetch("file"), subject: hash.fetch("subject"),
|
|
19
|
+
description: hash.fetch("description"),
|
|
20
|
+
original_snippet: hash.fetch("original_snippet"),
|
|
21
|
+
ordinal: hash.fetch("ordinal"))
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def initialize(path, entries)
|
|
25
|
+
@path = path
|
|
26
|
+
@entries = entries
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def accepted?(fingerprint) = @entries.include?(fingerprint)
|
|
30
|
+
|
|
31
|
+
def stale_entries(all_current_fingerprints)
|
|
32
|
+
current = all_current_fingerprints.to_set
|
|
33
|
+
@entries.reject { |e| current.include?(e) }
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Union new acceptances in, prune anything no longer matching a current
|
|
37
|
+
# mutant, write atomically.
|
|
38
|
+
def accept!(new_fingerprints, all_current_fingerprints)
|
|
39
|
+
current = all_current_fingerprints.to_set
|
|
40
|
+
@entries = (@entries + new_fingerprints).uniq.select { |e| current.include?(e) }
|
|
41
|
+
AtomicFile.write(@path, JSON.pretty_generate(@entries.map(&:to_h)))
|
|
42
|
+
nil
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
module ActiveMutator
|
|
2
|
+
# flock-guarded write-to-temp + rename. Concurrent runs in one repo (an
|
|
3
|
+
# agent plus a human — the dev-loop case) must not corrupt cache or ledger.
|
|
4
|
+
module AtomicFile
|
|
5
|
+
def self.write(path, content)
|
|
6
|
+
File.open("#{path}.lock", File::CREAT | File::RDWR, 0o644) do |lock|
|
|
7
|
+
lock.flock(File::LOCK_EX)
|
|
8
|
+
tmp = "#{path}.tmp#{Process.pid}"
|
|
9
|
+
File.write(tmp, content)
|
|
10
|
+
File.rename(tmp, path)
|
|
11
|
+
end
|
|
12
|
+
nil
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
require "digest"
|
|
2
|
+
require "fileutils"
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module ActiveMutator
|
|
6
|
+
# Runs the host suite once, instrumented, in a subprocess. Produces and
|
|
7
|
+
# caches the CoverageMap. Invalidation is coarse: any digest change in
|
|
8
|
+
# {app,lib,spec}/**/*.rb triggers a full re-run.
|
|
9
|
+
class Baseline
|
|
10
|
+
def initialize(root:, cache_dir: File.join(root, ".active_mutator"))
|
|
11
|
+
@root = root
|
|
12
|
+
@cache_dir = cache_dir
|
|
13
|
+
@out_path = File.join(cache_dir, "coverage.json")
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
attr_reader :last_refresh
|
|
17
|
+
|
|
18
|
+
def coverage_map(force: false)
|
|
19
|
+
digests = current_digests
|
|
20
|
+
if !force && File.exist?(@out_path)
|
|
21
|
+
map = CoverageMap.load(@out_path)
|
|
22
|
+
if map.fresh?(digests)
|
|
23
|
+
@last_refresh = :cached
|
|
24
|
+
return map
|
|
25
|
+
end
|
|
26
|
+
if map.version == 2
|
|
27
|
+
delta = BaselineDelta.compute(old_digests: stored_digests(map), new_digests: digests,
|
|
28
|
+
coverage_map: map, root: @root)
|
|
29
|
+
unless delta.full?
|
|
30
|
+
run_partial!(delta)
|
|
31
|
+
stamp_digests(digests)
|
|
32
|
+
@last_refresh = :partial
|
|
33
|
+
return CoverageMap.load(@out_path)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
run_baseline!
|
|
38
|
+
stamp_digests(digests)
|
|
39
|
+
@last_refresh = :full
|
|
40
|
+
CoverageMap.load(@out_path)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def run_baseline!
|
|
46
|
+
FileUtils.mkdir_p(@cache_dir)
|
|
47
|
+
env = baseline_env(@out_path)
|
|
48
|
+
# out: :err — the subprocess suite's progress output must not pollute
|
|
49
|
+
# our stdout (breaks `--format json` consumers).
|
|
50
|
+
ok = system(env, "bundle", "exec", "rspec", chdir: @root, out: :err)
|
|
51
|
+
raise BaselineFailed, "baseline suite failed — fix the suite before mutating" unless ok
|
|
52
|
+
raise BaselineFailed, "baseline produced no coverage output" unless File.exist?(@out_path)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def baseline_env(out_path)
|
|
56
|
+
{
|
|
57
|
+
"ACTIVE_MUTATOR" => "1",
|
|
58
|
+
"ACTIVE_MUTATOR_ROOT" => @root,
|
|
59
|
+
"ACTIVE_MUTATOR_BASELINE_OUT" => out_path,
|
|
60
|
+
# RUBYOPT, not `rspec --require`: project .rspec requires (spec_helper
|
|
61
|
+
# → app code) run before command-line requires, and Coverage misses
|
|
62
|
+
# everything loaded before Coverage.start. -r fires before rspec boots.
|
|
63
|
+
#
|
|
64
|
+
# Absolute path, not the gem-relative "active_mutator/baseline_hooks":
|
|
65
|
+
# `bundle exec` appends its own "-rbundler/setup" to RUBYOPT AFTER
|
|
66
|
+
# whatever RUBYOPT already held, so a bare gem-relative require here
|
|
67
|
+
# would run before Bundler has put this gem's lib/ on $LOAD_PATH and
|
|
68
|
+
# raise LoadError. An absolute path bypasses $LOAD_PATH entirely.
|
|
69
|
+
"RUBYOPT" => "-r#{File.expand_path("baseline_hooks", __dir__)}"
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def stored_digests(map)
|
|
74
|
+
JSON.parse(File.read(@out_path)).fetch("digests", {})
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def run_partial!(delta)
|
|
78
|
+
targets = delta.rerun_spec_files + delta.rerun_example_ids
|
|
79
|
+
partial_out = File.join(@cache_dir, "partial.json")
|
|
80
|
+
if targets.any?
|
|
81
|
+
env = baseline_env(partial_out)
|
|
82
|
+
ok = system(env, "bundle", "exec", "rspec", *targets, chdir: @root, out: :err)
|
|
83
|
+
raise BaselineFailed, "partial baseline run failed — fix the suite before mutating" unless ok
|
|
84
|
+
raise BaselineFailed, "partial baseline produced no output" unless File.exist?(partial_out)
|
|
85
|
+
end
|
|
86
|
+
merge_partial!(partial_out, delta)
|
|
87
|
+
ensure
|
|
88
|
+
FileUtils.rm_f(partial_out) if partial_out
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def merge_partial!(partial_out, delta)
|
|
92
|
+
cache = JSON.parse(File.read(@out_path))
|
|
93
|
+
part = File.exist?(partial_out) ? JSON.parse(File.read(partial_out)) : { "records" => {}, "times" => {} }
|
|
94
|
+
|
|
95
|
+
rerun_prefixes = delta.rerun_spec_files.map { |rel| "#{rel}[" }
|
|
96
|
+
obsolete = lambda do |example_id|
|
|
97
|
+
bare = example_id.sub(%r{\A\./}, "")
|
|
98
|
+
delta.rerun_example_ids.include?(example_id) ||
|
|
99
|
+
delta.drop_example_ids.include?(example_id) ||
|
|
100
|
+
rerun_prefixes.any? { |p| bare.start_with?(p) }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
cache["records"].reject! { |id, _| obsolete.call(id) }
|
|
104
|
+
cache["times"].reject! { |id, _| obsolete.call(id) }
|
|
105
|
+
cache["records"].each_value do |hits|
|
|
106
|
+
hits.reject! { |(path, _)| delta.drop_source_files.include?(path) }
|
|
107
|
+
end
|
|
108
|
+
cache["records"].merge!(part.fetch("records", {}))
|
|
109
|
+
cache["times"].merge!(part.fetch("times", {}))
|
|
110
|
+
AtomicFile.write(@out_path, JSON.generate(cache))
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def stamp_digests(digests)
|
|
114
|
+
data = JSON.parse(File.read(@out_path))
|
|
115
|
+
data["digests"] = digests
|
|
116
|
+
AtomicFile.write(@out_path, JSON.generate(data))
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def current_digests
|
|
120
|
+
files = Dir[File.join(@root, "{app,lib,spec}/**/*.rb")].sort
|
|
121
|
+
files += [File.join(@root, "Gemfile.lock"), File.join(@root, ".rspec")].select { |f| File.exist?(f) }
|
|
122
|
+
files.to_h { |f| [f.delete_prefix("#{@root}/"), Digest::SHA256.file(f).hexdigest] }
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
module ActiveMutator
|
|
2
|
+
# Decides how to refresh a stale coverage cache: surgically (re-run only
|
|
3
|
+
# affected spec files / examples) or fully (the safe fallback). Rules per
|
|
4
|
+
# the v1.1 spec's delta table; anything ambiguous prefers full.
|
|
5
|
+
class BaselineDelta
|
|
6
|
+
Delta = Data.define(:full, :rerun_spec_files, :rerun_example_ids,
|
|
7
|
+
:drop_example_ids, :drop_source_files) do
|
|
8
|
+
def full? = full
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
FULL = Delta.new(full: true, rerun_spec_files: [], rerun_example_ids: [],
|
|
12
|
+
drop_example_ids: [], drop_source_files: [])
|
|
13
|
+
|
|
14
|
+
def self.compute(old_digests:, new_digests:, coverage_map:, root:)
|
|
15
|
+
changed = (old_digests.keys | new_digests.keys)
|
|
16
|
+
.reject { |k| old_digests[k] == new_digests[k] }
|
|
17
|
+
return FULL if changed.any? { |k| full_trigger?(k) }
|
|
18
|
+
|
|
19
|
+
rerun_spec_files = []
|
|
20
|
+
rerun_example_ids = []
|
|
21
|
+
drop_example_ids = []
|
|
22
|
+
drop_source_files = []
|
|
23
|
+
|
|
24
|
+
changed.each do |rel|
|
|
25
|
+
added = !old_digests.key?(rel)
|
|
26
|
+
deleted = !new_digests.key?(rel)
|
|
27
|
+
if rel.start_with?("spec/")
|
|
28
|
+
owned = coverage_map.examples_for_spec_file(rel)
|
|
29
|
+
if deleted
|
|
30
|
+
# A deleted spec file with no records is support-like: other spec
|
|
31
|
+
# files may require it, and partial re-runs would explode. Full.
|
|
32
|
+
return FULL if owned.empty?
|
|
33
|
+
drop_example_ids.concat(owned)
|
|
34
|
+
elsif !added && owned.empty?
|
|
35
|
+
return FULL # pre-existing spec file owning no examples: support-like
|
|
36
|
+
else
|
|
37
|
+
rerun_spec_files << rel
|
|
38
|
+
end
|
|
39
|
+
else
|
|
40
|
+
abs = File.join(root, rel)
|
|
41
|
+
drop_source_files << abs if deleted
|
|
42
|
+
rerun_example_ids.concat(coverage_map.examples_covering_file(abs)) unless added
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
Delta.new(full: false,
|
|
47
|
+
rerun_spec_files: rerun_spec_files.uniq.sort,
|
|
48
|
+
rerun_example_ids: rerun_example_ids.uniq.sort,
|
|
49
|
+
drop_example_ids: drop_example_ids.uniq.sort,
|
|
50
|
+
drop_source_files: drop_source_files.uniq.sort)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def self.full_trigger?(rel)
|
|
54
|
+
rel.start_with?("spec/support/") || !rel.end_with?(".rb")
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Loaded standalone via RUBYOPT=-ractive_mutator/baseline_hooks in the host
|
|
2
|
+
# project's suite — before rspec boots, so Coverage instruments everything
|
|
3
|
+
# the suite loads (including code loaded by spec_helper). Records per-example
|
|
4
|
+
# coverage diffs and writes the inverted map to ACTIVE_MUTATOR_BASELINE_OUT.
|
|
5
|
+
require "json"
|
|
6
|
+
|
|
7
|
+
module ActiveMutator
|
|
8
|
+
module BaselineHooks
|
|
9
|
+
RECORDS = {}
|
|
10
|
+
TIMES = {}
|
|
11
|
+
|
|
12
|
+
def self.diff_coverage(before, after, root)
|
|
13
|
+
hits = []
|
|
14
|
+
after.each do |path, data|
|
|
15
|
+
next unless path.start_with?(root)
|
|
16
|
+
|
|
17
|
+
# Relative to root, not a global substring check: `root` itself may
|
|
18
|
+
# contain "/spec/" (e.g. a fixture nested under this gem's own
|
|
19
|
+
# spec/fixtures/ tree), which would otherwise falsely exclude every
|
|
20
|
+
# file under it.
|
|
21
|
+
relative = path.delete_prefix(root)
|
|
22
|
+
next if relative.start_with?("/spec/")
|
|
23
|
+
|
|
24
|
+
before_lines = before.dig(path, :lines)
|
|
25
|
+
data[:lines].each_with_index do |count, idx|
|
|
26
|
+
next if count.nil?
|
|
27
|
+
|
|
28
|
+
previous = before_lines ? before_lines[idx].to_i : 0
|
|
29
|
+
hits << [path, idx + 1] if count > previous
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
hits
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.build_payload(records, times)
|
|
36
|
+
{ "version" => 2, "records" => records, "times" => times }
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
if ENV["ACTIVE_MUTATOR_BASELINE_OUT"]
|
|
42
|
+
require "coverage"
|
|
43
|
+
Coverage.start(lines: true)
|
|
44
|
+
require "rspec/core" # loaded via RUBYOPT, so rspec isn't up yet
|
|
45
|
+
|
|
46
|
+
RSpec.configure do |config|
|
|
47
|
+
config.around(:each) do |example|
|
|
48
|
+
before = Coverage.peek_result
|
|
49
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
50
|
+
example.run
|
|
51
|
+
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
52
|
+
after = Coverage.peek_result
|
|
53
|
+
root = ENV.fetch("ACTIVE_MUTATOR_ROOT")
|
|
54
|
+
ActiveMutator::BaselineHooks::RECORDS[example.id] =
|
|
55
|
+
ActiveMutator::BaselineHooks.diff_coverage(before, after, root)
|
|
56
|
+
# NOT example.execution_result.run_time — that is nil until after
|
|
57
|
+
# around hooks complete.
|
|
58
|
+
ActiveMutator::BaselineHooks::TIMES[example.id] = elapsed
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
config.after(:suite) do
|
|
62
|
+
payload = ActiveMutator::BaselineHooks.build_payload(
|
|
63
|
+
ActiveMutator::BaselineHooks::RECORDS, ActiveMutator::BaselineHooks::TIMES
|
|
64
|
+
)
|
|
65
|
+
File.write(ENV.fetch("ACTIVE_MUTATOR_BASELINE_OUT"), JSON.generate(payload))
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
require "optparse"
|
|
2
|
+
|
|
3
|
+
module ActiveMutator
|
|
4
|
+
module CLI
|
|
5
|
+
def self.run(argv)
|
|
6
|
+
Runner.new(parse(argv)).call
|
|
7
|
+
rescue OptionParser::ParseError, Error => e
|
|
8
|
+
warn "active_mutator: #{e.message}"
|
|
9
|
+
2
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def self.parse(argv)
|
|
13
|
+
options = {
|
|
14
|
+
# Half the cores, not all of them: each worker pays full RSpec setup,
|
|
15
|
+
# and system-spec workers boot a browser + app server. Full-core
|
|
16
|
+
# oversubscription starves workers of CPU and turns slow-but-honest
|
|
17
|
+
# kills into false timeouts (observed on a real Rails monolith).
|
|
18
|
+
since: nil, subject_filter: nil, jobs: [Etc.nprocessors / 2, 1].max,
|
|
19
|
+
format: :terminal,
|
|
20
|
+
# Budgets derive from baseline times measured warm and unloaded; the
|
|
21
|
+
# factor must absorb parallel-load slowdown, and the floor the fork's
|
|
22
|
+
# boot cost (RSpec setup + spec file loading).
|
|
23
|
+
requires: [], timeout_factor: 8.0, timeout_floor: 10.0, force_baseline: false,
|
|
24
|
+
preload_helper: nil, serial_patterns: ["spec/system/", "spec/features/"],
|
|
25
|
+
browser_boot_seconds: 15.0, accept_survivors: false
|
|
26
|
+
}
|
|
27
|
+
paths = OptionParser.new do |o|
|
|
28
|
+
o.banner = "Usage: active_mutator [paths] [options]"
|
|
29
|
+
o.on("--since REF", "Mutate only methods changed since git REF") { |v| options[:since] = v }
|
|
30
|
+
o.on("--changed", "Mutate uncommitted work (alias for --since HEAD, plus untracked files)") { options[:since] = "HEAD" }
|
|
31
|
+
o.on("--subject NAME", "Mutate only the named subject, e.g. Foo::Bar#baz") { |v| options[:subject_filter] = v }
|
|
32
|
+
o.on("--jobs N", Integer, "Concurrent workers (default: half the CPU count)") { |v| options[:jobs] = v }
|
|
33
|
+
o.on("--format FMT", %w[terminal json], "Output format") { |v| options[:format] = v.to_sym }
|
|
34
|
+
o.on("--require FILE", "File to require before mutating (repeatable)") { |v| options[:requires] << v }
|
|
35
|
+
o.on("--force-baseline", "Ignore cached coverage map") { options[:force_baseline] = true }
|
|
36
|
+
o.on("--timeout-factor F", Float, "Timeout = baseline time * F + floor") { |v| options[:timeout_factor] = v }
|
|
37
|
+
o.on("--timeout-floor S", Float, "Minimum timeout seconds") { |v| options[:timeout_floor] = v }
|
|
38
|
+
o.on("--preload-helper FILE", "Spec helper to preload in the parent (default: auto-detect)") { |v| options[:preload_helper] = v }
|
|
39
|
+
o.on("--no-preload-helper", "Skip spec-helper preload") { options[:preload_helper] = :none }
|
|
40
|
+
o.on("--serial-pattern PAT", "Covering-path prefix that forces the serial lane (repeatable; replaces defaults on first use)") do |v|
|
|
41
|
+
options[:serial_patterns] = [] unless options[:serial_patterns_replaced]
|
|
42
|
+
options[:serial_patterns_replaced] = true
|
|
43
|
+
options[:serial_patterns] << v
|
|
44
|
+
end
|
|
45
|
+
o.on("--browser-boot-seconds S", Float, "Extra timeout budget for serial-lane mutants") { |v| options[:browser_boot_seconds] = v }
|
|
46
|
+
o.on("--accept-survivors", "Record surviving mutants into the acceptance ledger") { options[:accept_survivors] = true }
|
|
47
|
+
end.parse(argv)
|
|
48
|
+
options.delete(:serial_patterns_replaced)
|
|
49
|
+
|
|
50
|
+
Config.new(paths: paths, root: Dir.pwd, **options)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
require "etc"
|
|
2
|
+
|
|
3
|
+
module ActiveMutator
|
|
4
|
+
Config = Data.define(:paths, :since, :subject_filter, :jobs, :format, :requires,
|
|
5
|
+
:timeout_factor, :timeout_floor, :force_baseline, :root,
|
|
6
|
+
:preload_helper, :serial_patterns, :browser_boot_seconds,
|
|
7
|
+
:accept_survivors)
|
|
8
|
+
end
|