mutineer 1.6.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +680 -0
- data/README.md +210 -499
- data/lib/mutineer/baseline.rb +1 -1
- data/lib/mutineer/changed_lines.rb +2 -1
- data/lib/mutineer/child_coverage.rb +53 -0
- data/lib/mutineer/cli.rb +293 -42
- data/lib/mutineer/concern_blocks.rb +243 -0
- data/lib/mutineer/config.rb +154 -8
- data/lib/mutineer/coverage_map.rb +265 -27
- data/lib/mutineer/daemon_backend.rb +244 -75
- data/lib/mutineer/daemon_client.rb +73 -10
- data/lib/mutineer/daemon_server.rb +226 -42
- data/lib/mutineer/external_backend.rb +159 -15
- data/lib/mutineer/init.rb +124 -0
- data/lib/mutineer/isolation.rb +78 -24
- data/lib/mutineer/job_plan.rb +234 -44
- data/lib/mutineer/kill_channel.rb +24 -14
- data/lib/mutineer/migrate.rb +707 -0
- data/lib/mutineer/minitest_integration/in_test_database_error.rb +184 -0
- data/lib/mutineer/minitest_integration/kill_recorder.rb +25 -17
- data/lib/mutineer/minitest_integration.rb +44 -14
- data/lib/mutineer/mutator_registry.rb +6 -2
- data/lib/mutineer/mutators/array_literal.rb +4 -2
- data/lib/mutineer/mutators/base.rb +36 -7
- data/lib/mutineer/mutators/call_to_receiver.rb +148 -0
- data/lib/mutineer/mutators/chain_link.rb +6 -6
- data/lib/mutineer/mutators/condition_forcing.rb +6 -3
- data/lib/mutineer/mutators/operand_removal.rb +4 -2
- data/lib/mutineer/mutators/range_literal.rb +105 -4
- data/lib/mutineer/mutators/return_nil.rb +1 -0
- data/lib/mutineer/orphan_guard.rb +84 -0
- data/lib/mutineer/pairing.rb +62 -1
- data/lib/mutineer/project.rb +302 -14
- data/lib/mutineer/rails_run_env.rb +73 -0
- data/lib/mutineer/rails_worker_db.rb +1596 -57
- data/lib/mutineer/redaction.rb +199 -0
- data/lib/mutineer/reporter.rb +173 -31
- data/lib/mutineer/result.rb +52 -15
- data/lib/mutineer/runner.rb +148 -28
- data/lib/mutineer/statement_lines.rb +24 -11
- data/lib/mutineer/subject.rb +30 -4
- data/lib/mutineer/test_runners/minitest.rb +2 -1
- data/lib/mutineer/test_runners/rspec.rb +15 -3
- data/lib/mutineer/test_runners.rb +5 -2
- data/lib/mutineer/triage.rb +111 -0
- data/lib/mutineer/version.rb +1 -1
- data/lib/mutineer/worker_pool.rb +2 -1
- data/lib/mutineer.rb +4 -0
- data/skills/mutineer/SKILL.md +84 -0
- metadata +13 -1
data/lib/mutineer/baseline.rb
CHANGED
|
@@ -50,7 +50,7 @@ module Mutineer
|
|
|
50
50
|
# @return [Mutineer::Baseline] baseline object.
|
|
51
51
|
# @raise [Mutineer::ConfigError] when the file is missing or invalid.
|
|
52
52
|
def self.load(path)
|
|
53
|
-
doc = JSON.parse(File.read(path))
|
|
53
|
+
doc = JSON.parse(File.read(path, encoding: "UTF-8"))
|
|
54
54
|
unless doc.is_a?(Hash) && doc["schema_version"] && doc["survivors"].is_a?(Array)
|
|
55
55
|
raise ConfigError, "not a Mutineer JSON report: #{path}"
|
|
56
56
|
end
|
|
@@ -73,7 +73,8 @@ module Mutineer
|
|
|
73
73
|
# @return [String] diff text, or `""` on failure (after a stderr warning).
|
|
74
74
|
def git_diff(ref, abs_file, project_root)
|
|
75
75
|
out, _err, status = Open3.capture3(
|
|
76
|
-
"git", "--literal-pathspecs", "-C", project_root, "diff", "--unified=0", ref, "--", abs_file
|
|
76
|
+
"git", "--literal-pathspecs", "-C", project_root, "diff", "--unified=0", ref, "--", abs_file,
|
|
77
|
+
binmode: true # the hunk regex must not trip on a changed line's non-ASCII bytes
|
|
77
78
|
)
|
|
78
79
|
if status.success?
|
|
79
80
|
return out if indexed?(abs_file, project_root)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Mutineer
|
|
4
|
+
# Lets a test helper start Coverage in a forked mutant child (#248).
|
|
5
|
+
#
|
|
6
|
+
# Once the coverage map is built, the parent suspends Coverage so mutant
|
|
7
|
+
# children do not pay for it (#228). A suspended Coverage reports
|
|
8
|
+
# `running? == false`, so a test helper that starts it when it is off
|
|
9
|
+
# (SimpleCov's `SimpleCov.start`) calls `Coverage.start`, which raises
|
|
10
|
+
# "coverage measurement is already setup" while Coverage is suspended. Every
|
|
11
|
+
# mutant then errors before a test runs.
|
|
12
|
+
#
|
|
13
|
+
# In the child, `Coverage.start` and `Coverage.setup` first stop the
|
|
14
|
+
# suspended Coverage, then run as usual. Stopping builds and drops the whole
|
|
15
|
+
# result, which walks the heap, so only a child whose helper starts Coverage
|
|
16
|
+
# pays for it. Only files loaded after that start are measured. The parent's
|
|
17
|
+
# Coverage is untouched.
|
|
18
|
+
#
|
|
19
|
+
# Stdlib-only, so the app-side daemon can load it.
|
|
20
|
+
module ChildCoverage
|
|
21
|
+
# Prepended to Coverage's singleton class in the child: stops a suspended
|
|
22
|
+
# Coverage before a start or setup.
|
|
23
|
+
module StopBeforeStart
|
|
24
|
+
# Stops a suspended Coverage, then sets it up.
|
|
25
|
+
#
|
|
26
|
+
# @return [nil]
|
|
27
|
+
def setup(...)
|
|
28
|
+
result(stop: true, clear: true) if state == :suspended
|
|
29
|
+
super
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Stops a suspended Coverage, then starts it.
|
|
33
|
+
#
|
|
34
|
+
# @return [nil]
|
|
35
|
+
def start(...)
|
|
36
|
+
result(stop: true, clear: true) if state == :suspended
|
|
37
|
+
super
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Makes a later `Coverage.start` or `Coverage.setup` work when Coverage is
|
|
42
|
+
# suspended. A running or idle Coverage is left alone.
|
|
43
|
+
#
|
|
44
|
+
# Call it only in a forked child, before any test loads.
|
|
45
|
+
#
|
|
46
|
+
# @return [void]
|
|
47
|
+
def self.allow_start
|
|
48
|
+
return unless defined?(::Coverage) && ::Coverage.state == :suspended
|
|
49
|
+
|
|
50
|
+
::Coverage.singleton_class.prepend(StopBeforeStart)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
data/lib/mutineer/cli.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "fileutils"
|
|
3
4
|
require "optparse"
|
|
4
5
|
require "open3"
|
|
5
6
|
require_relative "version"
|
|
@@ -13,7 +14,11 @@ require_relative "job_plan"
|
|
|
13
14
|
require_relative "reporter"
|
|
14
15
|
require_relative "kill_matrix"
|
|
15
16
|
require_relative "baseline"
|
|
17
|
+
require_relative "migrate"
|
|
16
18
|
require_relative "mutator_registry"
|
|
19
|
+
require_relative "triage"
|
|
20
|
+
require_relative "init"
|
|
21
|
+
require_relative "redaction"
|
|
17
22
|
|
|
18
23
|
module Mutineer
|
|
19
24
|
# Command-line entry point. `start` is the single public method called by
|
|
@@ -21,7 +26,8 @@ module Mutineer
|
|
|
21
26
|
#
|
|
22
27
|
# Exit codes:
|
|
23
28
|
# 0 success / requested output (--version, --help, score >= threshold)
|
|
24
|
-
# 1 survivors below threshold,
|
|
29
|
+
# 1 survivors below threshold, a runtime error, or an ignore id
|
|
30
|
+
# `migrate` could not map
|
|
25
31
|
# 2 usage / flag error (unknown subcommand, invalid flag, unknown operator,
|
|
26
32
|
# out-of-range threshold)
|
|
27
33
|
class CLI
|
|
@@ -30,12 +36,23 @@ module Mutineer
|
|
|
30
36
|
Usage: mutineer [options] <command> [args]
|
|
31
37
|
|
|
32
38
|
Commands:
|
|
33
|
-
run [options] <source...>
|
|
39
|
+
run [options] <source...> [--test <test>...]
|
|
34
40
|
Mutate, run, and report
|
|
35
41
|
run --dry-run [options] <source...> Print candidate mutations only
|
|
42
|
+
triage REPORT.json --reason TEXT
|
|
43
|
+
(--id ID ... | --all) Print ignore: entries for
|
|
44
|
+
survivors. Does not edit files.
|
|
45
|
+
migrate [options] <source...> Rewrite old ignore ids in
|
|
46
|
+
.mutineer.yml. --dry-run prints
|
|
47
|
+
each change and writes nothing
|
|
48
|
+
init [--rails] [--force] Write .mutineer.yml and print
|
|
49
|
+
the first run command.
|
|
50
|
+
--force replaces an existing file.
|
|
36
51
|
|
|
37
52
|
Run options:
|
|
38
|
-
--test FILE Test file covering the sources (one per flag; repeat it)
|
|
53
|
+
--test FILE Test file covering the sources (one per flag; repeat it).
|
|
54
|
+
Omit to pair each source with its test by convention
|
|
55
|
+
(not under --dry-run)
|
|
39
56
|
--operators LIST Comma-separated operator names (default: Tier 1 set)
|
|
40
57
|
--threshold FLOAT Fail (exit 1) when score < FLOAT (default: 0 = off)
|
|
41
58
|
--baseline FILE Fail (exit 1) on NEW survivors / score drop vs a prior
|
|
@@ -55,9 +72,12 @@ module Mutineer
|
|
|
55
72
|
and the clean check (default: 120; not with --test-command)
|
|
56
73
|
--cache-dir DIR Directory for the coverage cache (default: .mutineer;
|
|
57
74
|
not with --test-command)
|
|
75
|
+
--coverage-from FILE Reuse the coverage.json of a --rails/--boot capture over
|
|
76
|
+
these sources or more; runs alone only the tests that
|
|
77
|
+
cover these sources
|
|
58
78
|
--framework NAME minitest or rspec (default: auto-detect from --test names)
|
|
59
79
|
--boot FILE Require FILE once in the parent to boot the app env, then
|
|
60
|
-
fork per mutant (Rails apps;
|
|
80
|
+
fork per mutant (Rails apps; needs a test, given or found by convention)
|
|
61
81
|
--rails Boot config/environment; without --daemon, defaults to
|
|
62
82
|
redefine and serial execution
|
|
63
83
|
--test-command CMD Run the target suite in the app's own runtime as a
|
|
@@ -66,7 +86,9 @@ module Mutineer
|
|
|
66
86
|
on the mutineer command (not as KEY=val inside CMD)
|
|
67
87
|
--daemon Boot the app ONCE in a persistent daemon and fork per
|
|
68
88
|
mutant, with per-worker DB isolation so --jobs N is safe
|
|
69
|
-
under Rails (needs --rails/--boot; not with --test-command)
|
|
89
|
+
under Rails (needs --rails/--boot; not with --test-command).
|
|
90
|
+
--no-daemon beats a .mutineer.yml daemon: (2.0 will default
|
|
91
|
+
--rails to --daemon; --no-daemon keeps it serial)
|
|
70
92
|
--format human|json|html Report format (default: human)
|
|
71
93
|
--output FILE Write the report to FILE instead of stdout
|
|
72
94
|
--dry-run List mutations without executing
|
|
@@ -77,7 +99,11 @@ module Mutineer
|
|
|
77
99
|
killers (in-process only; not with --daemon, --test-command,
|
|
78
100
|
--fail-fast or --dry-run; --no-matrix beats a .mutineer.yml
|
|
79
101
|
matrix:)
|
|
80
|
-
--
|
|
102
|
+
--allow-empty A run with no mutants is expected; do not warn (--no-allow-empty
|
|
103
|
+
beats a .mutineer.yml allow_empty:)
|
|
104
|
+
--verbose Show full error details: the backtrace and causes of a load
|
|
105
|
+
failure, the real error of a failed fork capture, and the
|
|
106
|
+
--test-command output (alias: --debug)
|
|
81
107
|
|
|
82
108
|
Options:
|
|
83
109
|
--list-operators List available operators (default vs optional) and exit
|
|
@@ -90,6 +116,14 @@ module Mutineer
|
|
|
90
116
|
# @param argv [Array<String>] raw command-line arguments.
|
|
91
117
|
# @return [void]
|
|
92
118
|
def self.start(argv)
|
|
119
|
+
if argv.first == "triage"
|
|
120
|
+
exit Triage.run(argv[1..])
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
if argv.first == "init"
|
|
124
|
+
exit Init.run(argv[1..])
|
|
125
|
+
end
|
|
126
|
+
|
|
93
127
|
# The CLI layer holds only the fields the user typed, so Config.resolve
|
|
94
128
|
# can tell a typed `false`/`nil` from an absent flag by whether the key exists.
|
|
95
129
|
opts = {}
|
|
@@ -111,6 +145,7 @@ module Mutineer
|
|
|
111
145
|
# conflicts with another flag can be turned off for one run.
|
|
112
146
|
o.on("--[no-]fail-fast") { |on| opts[:fail_fast] = on }
|
|
113
147
|
o.on("--[no-]matrix") { |on| opts[:matrix] = on }
|
|
148
|
+
o.on("--[no-]allow-empty") { |on| opts[:allow_empty] = on }
|
|
114
149
|
o.on("--only NAME") { |v| opts[:only] = v }
|
|
115
150
|
o.on("--since REF") { |v| opts[:since] = Config.parse(:since, v) }
|
|
116
151
|
# A typed "no" must beat a .mutineer.yml `since:` key: the key is present
|
|
@@ -125,6 +160,7 @@ module Mutineer
|
|
|
125
160
|
o.on("--timeout SECONDS") { |v| opts[:timeout] = Config.parse(:timeout, v) }
|
|
126
161
|
o.on("--capture-timeout SECONDS") { |v| opts[:capture_timeout] = Config.parse(:capture_timeout, v) }
|
|
127
162
|
o.on("--cache-dir DIR") { |v| opts[:cache_dir] = Config.parse(:cache_dir, v) }
|
|
163
|
+
o.on("--coverage-from FILE") { |v| opts[:coverage_from] = Config.parse(:coverage_from, v) }
|
|
128
164
|
o.on("--strategy STRAT") { |v| opts[:strategy] = Config.parse(:strategy, v) }
|
|
129
165
|
o.on("--framework NAME") { |v| opts[:framework] = Config.parse(:framework, v) }
|
|
130
166
|
o.on("--boot FILE") { |v| opts[:boot] = v }
|
|
@@ -141,7 +177,9 @@ module Mutineer
|
|
|
141
177
|
o.on("--test-command CMD") { |v| opts[:test_command] = v }
|
|
142
178
|
# Boot the app ONCE in a persistent daemon and fork per mutant, with
|
|
143
179
|
# per-worker DB isolation so --jobs N is safe under Rails.
|
|
144
|
-
|
|
180
|
+
# A typed --no-daemon beats a .mutineer.yml `daemon: true`, and pins the
|
|
181
|
+
# serial --rails path before 2.0 makes the daemon the default.
|
|
182
|
+
o.on("--[no-]daemon") { |on| opts[:daemon] = on }
|
|
145
183
|
end
|
|
146
184
|
|
|
147
185
|
begin
|
|
@@ -175,7 +213,9 @@ module Mutineer
|
|
|
175
213
|
case argv.first
|
|
176
214
|
when "run"
|
|
177
215
|
warn_config_root_mismatch(file_path, config.project_root) if file_path
|
|
178
|
-
tests_as_sources = argv[1..].
|
|
216
|
+
tests_as_sources = argv[1..].select do |arg|
|
|
217
|
+
Pairing.test_given_as_source?(arg, project_root: config.project_root, tests: config.tests)
|
|
218
|
+
end
|
|
179
219
|
if config.explicit?(:tests) && tests_as_sources.any?
|
|
180
220
|
warn "mutineer: #{tests_as_sources.join(', ')} looks like a test file, not a source. " \
|
|
181
221
|
"--test takes one file; repeat it for each test file (--test a_test.rb --test b_test.rb)"
|
|
@@ -185,12 +225,99 @@ module Mutineer
|
|
|
185
225
|
# project becomes its root-relative path (#104). Test inference (when --test is omitted) happens in validate!.
|
|
186
226
|
config.sources = Pairing.expand_sources(argv[1..], project_root: config.project_root)
|
|
187
227
|
run(config)
|
|
228
|
+
when "migrate"
|
|
229
|
+
warn_config_root_mismatch(file_path, config.project_root) if file_path
|
|
230
|
+
unless file_path
|
|
231
|
+
warn "mutineer: migrate needs a .mutineer.yml (none found from #{Dir.pwd})"
|
|
232
|
+
exit 2
|
|
233
|
+
end
|
|
234
|
+
if argv[1..].empty?
|
|
235
|
+
warn "mutineer: migrate requires at least one source file"
|
|
236
|
+
exit 2
|
|
237
|
+
end
|
|
238
|
+
config.sources = Pairing.expand_sources(argv[1..], project_root: config.project_root)
|
|
239
|
+
migrate(config, file_path)
|
|
188
240
|
else
|
|
189
241
|
warn "mutineer: unknown command '#{argv.first}'"
|
|
190
242
|
exit 2
|
|
191
243
|
end
|
|
192
244
|
end
|
|
193
245
|
|
|
246
|
+
# Rewrites old-format ids in the loaded `.mutineer.yml` `ignore:` list.
|
|
247
|
+
# Prints each change. With `--dry-run`, writes nothing. Exits 1 when any
|
|
248
|
+
# id matched no mutant in these sources and operators, or when an id in a
|
|
249
|
+
# `{ id:, reason: }` entry now matches several mutants (split it by hand).
|
|
250
|
+
# Those ids stay in the file. Exits 0 when every id is current or was
|
|
251
|
+
# rewritten.
|
|
252
|
+
# An unsupported `ignore:` shape exits 2 and leaves the file unchanged.
|
|
253
|
+
#
|
|
254
|
+
# @param config [Mutineer::Config] run configuration, sources already set.
|
|
255
|
+
# @param file_path [String] the `.mutineer.yml` to edit.
|
|
256
|
+
# @return [void]
|
|
257
|
+
def self.migrate(config, file_path)
|
|
258
|
+
current, old_to_new = Migrate.id_maps(config)
|
|
259
|
+
text = File.read(file_path, encoding: "UTF-8")
|
|
260
|
+
outcome = Migrate.rewrite(text, current, old_to_new)
|
|
261
|
+
outcome.replacements.each do |rep|
|
|
262
|
+
puts "[mutineer] #{rep.old_id} -> #{rep.new_ids.join(', ')}"
|
|
263
|
+
end
|
|
264
|
+
outcome.unmapped.each do |id|
|
|
265
|
+
warn "[mutineer] ignore entry #{id} matched no mutant in these sources and operators. " \
|
|
266
|
+
"It was left in place."
|
|
267
|
+
end
|
|
268
|
+
outcome.split.each do |rep|
|
|
269
|
+
warn "[mutineer] ignore entry #{rep.old_id} has a reason and now matches several mutants " \
|
|
270
|
+
"(#{rep.new_ids.join(', ')}). Split it into one entry per id by hand. It was left in place."
|
|
271
|
+
end
|
|
272
|
+
warn "[mutineer] dry run: .mutineer.yml was not changed" if config.dry_run
|
|
273
|
+
replace_file(file_path, outcome.text) if !config.dry_run && outcome.text != text
|
|
274
|
+
exit(outcome.unmapped.empty? && outcome.split.empty? ? 0 : 1)
|
|
275
|
+
rescue ArgumentError => e
|
|
276
|
+
warn "mutineer: #{e.message}"
|
|
277
|
+
exit 2
|
|
278
|
+
rescue Mutineer::ParseError => e
|
|
279
|
+
warn "mutineer: error reading: #{e.message}"
|
|
280
|
+
exit 1
|
|
281
|
+
rescue SystemCallError => e
|
|
282
|
+
warn "mutineer: #{e.message}"
|
|
283
|
+
exit 2
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# Replaces `path` with `contents` without truncating the original first.
|
|
287
|
+
# The new bytes go to a temporary file in the same directory. That file
|
|
288
|
+
# is closed before it is renamed over the target, and it keeps the
|
|
289
|
+
# target's mode. A symlink is followed, so the target file changes and
|
|
290
|
+
# the link stays. If the write fails, the temporary file is removed and
|
|
291
|
+
# the original file is left as it was.
|
|
292
|
+
#
|
|
293
|
+
# @api private
|
|
294
|
+
# @param path [String] the configuration file to replace.
|
|
295
|
+
# @param contents [String] the full new text.
|
|
296
|
+
# @return [void]
|
|
297
|
+
def self.replace_file(path, contents)
|
|
298
|
+
target = File.symlink?(path) ? File.realpath(path) : path
|
|
299
|
+
directory = File.dirname(target)
|
|
300
|
+
base = File.basename(target)
|
|
301
|
+
temp = nil
|
|
302
|
+
100.times do
|
|
303
|
+
candidate = File.join(directory, ".#{base}.migrate-#{Process.pid}-#{rand(1_000_000_000)}.tmp")
|
|
304
|
+
next if File.exist?(candidate)
|
|
305
|
+
|
|
306
|
+
temp = candidate
|
|
307
|
+
break
|
|
308
|
+
end
|
|
309
|
+
raise Errno::EEXIST, target unless temp
|
|
310
|
+
|
|
311
|
+
mode = File.stat(target).mode & 0o7777
|
|
312
|
+
# `perm` is still masked by the umask, so chmod restores the exact mode.
|
|
313
|
+
File.write(temp, contents, perm: mode)
|
|
314
|
+
File.chmod(mode, temp)
|
|
315
|
+
File.rename(temp, target)
|
|
316
|
+
rescue StandardError
|
|
317
|
+
File.unlink(temp) if temp && File.exist?(temp)
|
|
318
|
+
raise
|
|
319
|
+
end
|
|
320
|
+
|
|
194
321
|
# Lists available operators.
|
|
195
322
|
#
|
|
196
323
|
# @return [void]
|
|
@@ -212,39 +339,60 @@ module Mutineer
|
|
|
212
339
|
exit 2
|
|
213
340
|
end
|
|
214
341
|
validate!(config)
|
|
342
|
+
Config.warn_rails_default_change(config)
|
|
215
343
|
|
|
216
344
|
config.dry_run ? dry_run(config) : execute(config)
|
|
217
345
|
rescue ArgumentError => e
|
|
218
346
|
# Unknown --operators value surfaces here; no backtrace reaches the user.
|
|
347
|
+
warn "mutineer: #{Redaction.text(e.message)}"
|
|
348
|
+
exit 2
|
|
349
|
+
rescue Mutineer::ConfigError => e
|
|
350
|
+
# A config value that only the booted app can check (`shared_databases`
|
|
351
|
+
# naming the primary database). A usage error, raised before any mutant runs.
|
|
219
352
|
warn "mutineer: #{e.message}"
|
|
220
353
|
exit 2
|
|
221
354
|
rescue SystemCallError => e
|
|
222
355
|
# A missing/unreadable path reaches here as Errno::ENOENT etc. A plain
|
|
223
356
|
# message and usage exit, never a raw backtrace.
|
|
224
|
-
warn "mutineer: #{e.message}"
|
|
357
|
+
warn "mutineer: #{Redaction.text(e.message)}"
|
|
225
358
|
exit 2
|
|
226
359
|
rescue SyntaxError => e
|
|
227
|
-
#
|
|
228
|
-
#
|
|
229
|
-
|
|
360
|
+
# Boot, source and require files report a SyntaxError as ProjectLoadError
|
|
361
|
+
# below. This covers a file loaded on any other path; report it cleanly
|
|
362
|
+
# rather than dumping a backtrace.
|
|
363
|
+
warn "mutineer: cannot load source: #{Redaction.text(e.message)}"
|
|
230
364
|
exit 1
|
|
231
365
|
rescue Mutineer::ParseError => e
|
|
232
|
-
warn "mutineer: error reading: #{e.message}"
|
|
366
|
+
warn "mutineer: error reading: #{Redaction.text(e.message)}"
|
|
233
367
|
exit 1
|
|
234
368
|
rescue Mutineer::SmokeCheckError => e
|
|
235
369
|
# The unmutated suite is not green under --test-command: a broken
|
|
236
370
|
# environment, not weak tests. Runtime error (exit 1), not usage (exit 2).
|
|
237
|
-
warn "mutineer: #{e.message}"
|
|
371
|
+
warn "mutineer: #{Redaction.text(e.message)}"
|
|
238
372
|
exit 1
|
|
239
373
|
rescue Mutineer::ConcurrentRunError => e
|
|
240
374
|
# Another process owns a source file. Runtime error (exit 1), not a
|
|
241
375
|
# backtrace: the working tree is still the other run's responsibility.
|
|
242
|
-
warn "mutineer: #{e.message}"
|
|
376
|
+
warn "mutineer: #{Redaction.text(e.message)}"
|
|
243
377
|
exit 1
|
|
378
|
+
rescue Mutineer::CoverageFromError => e
|
|
379
|
+
warn "mutineer: #{Redaction.text(e.message)}"
|
|
380
|
+
exit 2
|
|
244
381
|
rescue Mutineer::DaemonBootError => e
|
|
245
382
|
# The daemon is gone for good, so the run ended rather than scoring the rest
|
|
246
383
|
# against it. A deliberate stop deserves a message, not a raw backtrace.
|
|
384
|
+
warn "mutineer: #{Redaction.text(e.message)}"
|
|
385
|
+
exit 1
|
|
386
|
+
rescue Mutineer::ProjectLoadError => e
|
|
387
|
+
# A boot file, source or --require file failed to load. The message is
|
|
388
|
+
# already redacted; --verbose adds the backtrace and the cause chain.
|
|
247
389
|
warn "mutineer: #{e.message}"
|
|
390
|
+
if config.verbose
|
|
391
|
+
(e.backtrace || []).each { |frame| warn "\t#{Redaction.text(frame)}" }
|
|
392
|
+
e.causes.each { |cause| warn "mutineer: caused by: #{cause}" }
|
|
393
|
+
else
|
|
394
|
+
warn "mutineer: re-run with --verbose for the backtrace and causes"
|
|
395
|
+
end
|
|
248
396
|
exit 1
|
|
249
397
|
end
|
|
250
398
|
|
|
@@ -260,8 +408,9 @@ module Mutineer
|
|
|
260
408
|
validate_test_command!(config) if config.test_command
|
|
261
409
|
|
|
262
410
|
validate_since!(config) if config.since
|
|
263
|
-
preflight_output!(config.output) if config.output
|
|
411
|
+
preflight_output!(config.output, config) if config.output
|
|
264
412
|
preflight_baseline!(config.baseline) if config.baseline
|
|
413
|
+
preflight_same_file!(config) if config.output && config.baseline && !config.dry_run
|
|
265
414
|
|
|
266
415
|
# When --test is omitted, infer each source's test by convention. Autopair
|
|
267
416
|
# also re-detects framework from inferred tests when --framework was not set.
|
|
@@ -271,6 +420,7 @@ module Mutineer
|
|
|
271
420
|
# cannot bypass the RSpec rejection (framework would still be minitest if we
|
|
272
421
|
# validated before discovery).
|
|
273
422
|
validate_daemon!(config) if config.daemon
|
|
423
|
+
warn_unused_coverage_from(config)
|
|
274
424
|
|
|
275
425
|
# Boot mode needs at least one --test file (nothing to select from otherwise).
|
|
276
426
|
if config.boot && config.tests.empty?
|
|
@@ -318,9 +468,10 @@ module Mutineer
|
|
|
318
468
|
end
|
|
319
469
|
|
|
320
470
|
# --daemon selects the persistent-daemon backend (boot once, fork per mutant,
|
|
321
|
-
# per-worker DB isolation on SQLite). Usage errors exit 2:
|
|
322
|
-
# --test-command; requires --rails or --boot; minitest only;
|
|
323
|
-
# (redefine needs a shared VM surgical path the daemon
|
|
471
|
+
# per-worker DB isolation on SQLite, Postgres, and MySQL). Usage errors exit 2:
|
|
472
|
+
# cannot combine with --test-command; requires --rails or --boot; minitest only;
|
|
473
|
+
# reload strategy only (redefine needs a shared VM surgical path the daemon
|
|
474
|
+
# does not ship).
|
|
324
475
|
#
|
|
325
476
|
# @api private
|
|
326
477
|
# @param config [Mutineer::Config] run configuration.
|
|
@@ -352,9 +503,24 @@ module Mutineer
|
|
|
352
503
|
UNUSED_SETTING_SCOPE = {
|
|
353
504
|
timeout: "in-process runs only",
|
|
354
505
|
capture_timeout: "in-process and --daemon coverage capture",
|
|
355
|
-
cache_dir: "the in-process and --daemon coverage cache"
|
|
506
|
+
cache_dir: "the in-process and --daemon coverage cache",
|
|
507
|
+
coverage_from: "in-process --rails/--boot runs"
|
|
356
508
|
}.freeze
|
|
357
509
|
|
|
510
|
+
# Warns when `--coverage-from` is set on a run that captures no coverage.
|
|
511
|
+
#
|
|
512
|
+
# @api private
|
|
513
|
+
# @param config [Mutineer::Config] run configuration.
|
|
514
|
+
# @return [void]
|
|
515
|
+
def self.warn_unused_coverage_from(config)
|
|
516
|
+
mode = if config.test_command then "--test-command"
|
|
517
|
+
elsif config.daemon then "--daemon"
|
|
518
|
+
elsif config.dry_run then "--dry-run"
|
|
519
|
+
elsif !config.boot then "a run without --rails/--boot"
|
|
520
|
+
end
|
|
521
|
+
warn_unused_settings(config, mode, %i[coverage_from]) if mode
|
|
522
|
+
end
|
|
523
|
+
|
|
358
524
|
# A backend that never reads a setting the user set would silently do
|
|
359
525
|
# nothing with it, so say so. `--test-command` builds no coverage map and
|
|
360
526
|
# has its own runtime, so it reads none of {UNUSED_SETTING_SCOPE}; the
|
|
@@ -496,13 +662,50 @@ module Mutineer
|
|
|
496
662
|
exit 2
|
|
497
663
|
end
|
|
498
664
|
|
|
499
|
-
#
|
|
665
|
+
# #281: a report written over its own baseline turns a failed gate into the
|
|
666
|
+
# next baseline. Same check as the Action's guard (action.yml); the message
|
|
667
|
+
# is that guard's text adapted for the CLI, so keep the two in step. The
|
|
668
|
+
# output is expanded the way Reporter#report writes it, so `x/../b.json`
|
|
669
|
+
# with a missing `x/` still matches `b.json`. File.identical? is false when
|
|
670
|
+
# either file is missing, so a new output path never matches.
|
|
671
|
+
# Unlike the Action's guard, a dry run skips this check: it writes no report.
|
|
672
|
+
#
|
|
673
|
+
# @api private
|
|
674
|
+
# @param config [Mutineer::Config] run configuration (output and baseline set).
|
|
675
|
+
# @return [void]
|
|
676
|
+
def self.preflight_same_file!(config)
|
|
677
|
+
output = File.expand_path(config.output)
|
|
678
|
+
return unless File.identical?(output, config.baseline)
|
|
679
|
+
|
|
680
|
+
warn "mutineer: --output and #{config.origin(:baseline)} are the same file ('#{File.realpath(output)}'). " \
|
|
681
|
+
"A failing run would replace the baseline. Write the report to a separate path. " \
|
|
682
|
+
"Copy that file onto the baseline only when the run exits 0."
|
|
683
|
+
exit 2
|
|
684
|
+
end
|
|
685
|
+
|
|
686
|
+
# Preflights an output path. A missing parent is a usage error (exit 2),
|
|
687
|
+
# except the cache directory: the documented `--output .mutineer/run.json`
|
|
688
|
+
# must work on a fresh checkout. It is created here, before the run, so a
|
|
689
|
+
# failure to create it exits 2 up front. A later failed check can leave the
|
|
690
|
+
# empty cache directory behind (a real run creates it anyway). A dry run
|
|
691
|
+
# writes no report, so it creates nothing.
|
|
500
692
|
#
|
|
501
693
|
# @api private
|
|
502
694
|
# @param path [String] output file path.
|
|
695
|
+
# @param config [Mutineer::Config] run configuration (for the cache directory).
|
|
503
696
|
# @return [void]
|
|
504
|
-
def self.preflight_output!(path)
|
|
697
|
+
def self.preflight_output!(path, config)
|
|
505
698
|
dir = File.dirname(File.expand_path(path))
|
|
699
|
+
if !File.exist?(dir) && dir == File.expand_path(config.cache_dir, config.project_root)
|
|
700
|
+
return if config.dry_run
|
|
701
|
+
|
|
702
|
+
begin
|
|
703
|
+
FileUtils.mkdir_p(dir)
|
|
704
|
+
rescue SystemCallError => e
|
|
705
|
+
warn "mutineer: cannot write to #{path}: #{e.message}"
|
|
706
|
+
exit 2
|
|
707
|
+
end
|
|
708
|
+
end
|
|
506
709
|
return if File.directory?(dir) && File.writable?(dir)
|
|
507
710
|
|
|
508
711
|
reason = File.directory?(dir) ? "directory is not writable" : "no such directory"
|
|
@@ -543,17 +746,24 @@ module Mutineer
|
|
|
543
746
|
baseline: delta ? delta.legacy_matches : 0 }
|
|
544
747
|
reporter.report(out: $stdout, err: $stderr, threshold: config.threshold,
|
|
545
748
|
format: config.format, output: config.output, baseline: delta,
|
|
546
|
-
scoped: !config.since.nil?, legacy_id_matches: legacy_id_matches
|
|
749
|
+
scoped: !config.since.nil?, legacy_id_matches: legacy_id_matches,
|
|
750
|
+
declined: extras.fetch(:declined, {}))
|
|
751
|
+
warn_empty_run(config, extras) if aggregate.total.zero?
|
|
547
752
|
|
|
548
753
|
# Warn (stderr, so it never pollutes json/html) that an external run's score
|
|
549
754
|
# is not comparable to an in-process run: no coverage narrowing (uncovered
|
|
550
|
-
# mutants count as survivors), and an infra failure
|
|
551
|
-
# (
|
|
552
|
-
#
|
|
755
|
+
# mutants count as survivors), and an infra failure that exits 1 can still
|
|
756
|
+
# score as a kill (#292): with the summary check on, when its output names a
|
|
757
|
+
# failing test; with it off (the clean run printed no Minitest or RSpec
|
|
758
|
+
# summary line), on any exit 1, a load crash included. Daemon coverage
|
|
759
|
+
# fallback warnings are emitted from the runner only when the map is
|
|
760
|
+
# unavailable, not on every --daemon run.
|
|
553
761
|
if config.test_command
|
|
554
762
|
warn "[mutineer] --test-command score is an upper bound, not comparable to an " \
|
|
555
|
-
"in-process run: no coverage narrowing (uncovered mutants count as survivors) " \
|
|
556
|
-
"
|
|
763
|
+
"in-process run: no coverage narrowing (uncovered mutants count as survivors). " \
|
|
764
|
+
"Only exit 1 counts as a kill, or a SimpleCov gate exit whose summary shows a failing " \
|
|
765
|
+
"test (an RSpec --failure-exit-code other than 1 makes every kill an error). When the clean run printed a Minitest or RSpec summary line, " \
|
|
766
|
+
"Mutineer also reads it to tell a crash from a kill. See https://github.com/davidteren/mutineer/blob/main/docs/rails-setup.md#apps-on-ruby--34"
|
|
557
767
|
end
|
|
558
768
|
|
|
559
769
|
# Nudge toward the opt-in tier-2 operators (human report only: never
|
|
@@ -568,6 +778,33 @@ module Mutineer
|
|
|
568
778
|
exit [reporter.exit_code(threshold: config.threshold), baseline_exit].max
|
|
569
779
|
end
|
|
570
780
|
|
|
781
|
+
# Says why a run has no mutants, on stderr for every format. A --since run
|
|
782
|
+
# whose sources had mutants before scoping is a stated success: the changes
|
|
783
|
+
# held no mutable Ruby code. Any other empty run is a likely mistake (wrong
|
|
784
|
+
# path, operators that never match), which 2.0 fails unless --allow-empty
|
|
785
|
+
# says it is expected.
|
|
786
|
+
#
|
|
787
|
+
# @param config [Mutineer::Config] run configuration.
|
|
788
|
+
# @param extras [Hash] the run extras; `:unscoped_mutants` counts the mutants (run and
|
|
789
|
+
# suppressed) before --since, and `:declined` the candidates each operator did not mutate.
|
|
790
|
+
# @return [void]
|
|
791
|
+
def self.warn_empty_run(config, extras)
|
|
792
|
+
# Mutants existed before --since narrowed them: the changes held none.
|
|
793
|
+
if config.since && extras[:unscoped_mutants].to_i.positive?
|
|
794
|
+
warn "mutineer: nothing to test in the changes since #{config.since}"
|
|
795
|
+
elsif !config.allow_empty
|
|
796
|
+
declined = extras.fetch(:declined, {}).values.sum
|
|
797
|
+
cause = if declined.positive?
|
|
798
|
+
"#{declined} #{declined == 1 ? 'candidate was' : 'candidates were'} declined " \
|
|
799
|
+
"(--dry-run shows the count per operator)."
|
|
800
|
+
else
|
|
801
|
+
"Check that the sources contain code the operators mutate."
|
|
802
|
+
end
|
|
803
|
+
warn "mutineer: no mutants were generated. #{cause} The run exits 0, and #{BECOMES_ERROR_IN_2_0}. " \
|
|
804
|
+
"Pass --allow-empty (or allow_empty: true) if an empty run is expected."
|
|
805
|
+
end
|
|
806
|
+
end
|
|
807
|
+
|
|
571
808
|
# The tier-2 operators not in the active set, as a one-line hint (or nil when
|
|
572
809
|
# they are all already enabled). `active` nil means the default (Tier-1) set.
|
|
573
810
|
#
|
|
@@ -624,7 +861,8 @@ module Mutineer
|
|
|
624
861
|
warn "[mutineer] ignore entry #{old} uses the old id format, which did not include the " \
|
|
625
862
|
"file path. It matched these new ids: #{listed}. This list covers only mutants in " \
|
|
626
863
|
"this run's sources and operators; a run over every source gives the complete " \
|
|
627
|
-
"replacement. #{advice}"
|
|
864
|
+
"replacement. #{advice} To update the file, run `mutineer migrate`. " \
|
|
865
|
+
"#{BECOMES_ERROR_IN_2_0}."
|
|
628
866
|
end
|
|
629
867
|
end
|
|
630
868
|
|
|
@@ -635,12 +873,12 @@ module Mutineer
|
|
|
635
873
|
# @return [void]
|
|
636
874
|
def self.warn_legacy_baseline
|
|
637
875
|
warn "[mutineer] the baseline uses the old id format, which did not include the file " \
|
|
638
|
-
"path, so survivors were matched on their old ids and files.
|
|
639
|
-
"
|
|
640
|
-
"
|
|
876
|
+
"path, so survivors were matched on their old ids and files. Please generate a new " \
|
|
877
|
+
"baseline with `--format json`, but only after every gate that reads it runs this " \
|
|
878
|
+
"mutineer version or later. #{BECOMES_ERROR_IN_2_0}."
|
|
641
879
|
end
|
|
642
880
|
|
|
643
|
-
# Runs dry-run mode. Reuses JobPlan.collect_jobs (+
|
|
881
|
+
# Runs dry-run mode. Reuses JobPlan.collect_jobs (+ scope_since) so the
|
|
644
882
|
# candidate list cannot drift from a real run's job selection.
|
|
645
883
|
#
|
|
646
884
|
# @param config [Mutineer::Config] run configuration.
|
|
@@ -649,14 +887,10 @@ module Mutineer
|
|
|
649
887
|
operator_classes = MutatorRegistry.resolve(config.operators || MutatorRegistry::DEFAULT_NAMES)
|
|
650
888
|
jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
|
|
651
889
|
warn_legacy_ignore_matches(extras[:legacy_ignore_matches])
|
|
652
|
-
# Narrow jobs and ignored the same way
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
ignored = JobPlan.filter_since(ignored_jobs, source_map, config).size
|
|
657
|
-
else
|
|
658
|
-
ignored = ignored_results.size
|
|
659
|
-
end
|
|
890
|
+
# Narrow jobs and ignored the same way a real run does, so the summary
|
|
891
|
+
# matches the printed list.
|
|
892
|
+
jobs, ignored_results, = JobPlan.scope_since(jobs, ignored_results, source_map, config)
|
|
893
|
+
ignored = ignored_results.size
|
|
660
894
|
|
|
661
895
|
per_operator = Hash.new(0)
|
|
662
896
|
skipped = 0
|
|
@@ -678,8 +912,25 @@ module Mutineer
|
|
|
678
912
|
breakdown = per_operator.map { |op, n| "#{op}: #{n}" }.join(", ")
|
|
679
913
|
summary = breakdown.empty? ? "" : "#{breakdown} — "
|
|
680
914
|
puts "#{summary}#{total} mutations (dry run, not executed); " \
|
|
681
|
-
"#{skipped} skipped (invalid); #{ignored} ignored (suppressed)"
|
|
915
|
+
"#{skipped} skipped (invalid); #{ignored} ignored (suppressed); " \
|
|
916
|
+
"#{declined_summary(extras[:declined], since: config.since)}"
|
|
682
917
|
exit 0
|
|
683
918
|
end
|
|
919
|
+
|
|
920
|
+
# Formats the dry run's declined count: the total, then each operator's
|
|
921
|
+
# count when the total is above zero. `--since` does not narrow the count,
|
|
922
|
+
# so under `--since` the text says so.
|
|
923
|
+
#
|
|
924
|
+
# @param declined [Hash{String => Integer}] operator name => declined count.
|
|
925
|
+
# @param since [String, nil] the `--since` ref, when set.
|
|
926
|
+
# @return [String] e.g. `2 declined (range: 2)`.
|
|
927
|
+
def self.declined_summary(declined, since:)
|
|
928
|
+
total = declined.values.sum
|
|
929
|
+
text = "#{total} declined"
|
|
930
|
+
text += " in the scanned files (not narrowed by --since)" if since
|
|
931
|
+
return text if total.zero?
|
|
932
|
+
|
|
933
|
+
"#{text} (#{declined.map { |op, n| "#{op}: #{n}" }.join(', ')})"
|
|
934
|
+
end
|
|
684
935
|
end
|
|
685
936
|
end
|